PGMAllBth.h@ 19087

Last change on this file since 19087 was 19015, checked in by vboxsync, 16 years ago
Split up TRPM. (guest SMP)
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 189.5 KB

Line
1	/* $Id: PGMAllBth.h 19015 2009-04-20 07:54:29Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
43	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
44	__END_DECLS
45
46
47	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
48	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
49	# error "Invalid combination; PAE guest implies PAE shadow"
50	#endif
51
52	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
53	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
54	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
55	#endif
56
57	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
58	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
59	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
60	#endif
61
62	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
63	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
64	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
65	#endif
66
67	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
68	# define PGM_WITHOUT_MAPPINGS
69	#endif
70
71
72	#ifndef IN_RING3
73	/**
74	* #PF Handler for raw-mode guest execution.
75	*
76	* @returns VBox status code (appropriate for trap handling and GC return).
77	*
78	* @param pVCpu VMCPU Handle.
79	* @param uErr The trap error code.
80	* @param pRegFrame Trap register frame.
81	* @param pvFault The fault address.
82	*/
83	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
84	{
85	PVM pVM = pVCpu->CTX_SUFF(pVM);
86
87	# if defined(IN_RC) && defined(VBOX_STRICT)
88	PGMDynCheckLocks(pVM);
89	# endif
90
91	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
92	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
93	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
94
95	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
96	/*
97	* Hide the instruction fetch trap indicator for now.
98	*/
99	/** @todo NXE will change this and we must fix NXE in the switcher too! */
100	if (uErr & X86_TRAP_PF_ID)
101	{
102	uErr &= ~X86_TRAP_PF_ID;
103	TRPMSetErrorCode(pVCpu, uErr);
104	}
105	# endif
106
107	/*
108	* Get PDs.
109	*/
110	int rc;
111	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
112	# if PGM_GST_TYPE == PGM_TYPE_32BIT
113	const unsigned iPDSrc = pvFault >> GST_PD_SHIFT;
114	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
115
116	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
117
118	# if PGM_GST_TYPE == PGM_TYPE_PAE
119	unsigned iPDSrc;
120	X86PDPE PdpeSrc;
121	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, pvFault, &iPDSrc, &PdpeSrc);
122
123	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
124	unsigned iPDSrc;
125	PX86PML4E pPml4eSrc;
126	X86PDPE PdpeSrc;
127	PGSTPD pPDSrc;
128
129	pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
130	Assert(pPml4eSrc);
131	# endif
132
133	/* Quick check for a valid guest trap. (PAE & AMD64) */
134	if (!pPDSrc)
135	{
136	# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
137	LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)((pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
138	# else
139	LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
140	# endif
141	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
142	TRPMSetErrorCode(pVCpu, uErr);
143	return VINF_EM_RAW_GUEST_TRAP;
144	}
145	# endif
146
147	# else /* !PGM_WITH_PAGING */
148	PGSTPD pPDSrc = NULL;
149	const unsigned iPDSrc = 0;
150	# endif /* !PGM_WITH_PAGING */
151
152
153	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
154	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
155	PX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
156
157	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
158	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
159
160	PX86PDPAE pPDDst;
161	# if PGM_GST_TYPE != PGM_TYPE_PAE
162	X86PDPE PdpeSrc;
163
164	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
165	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
166	# endif
167	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, &PdpeSrc, &pPDDst);
168	if (rc != VINF_SUCCESS)
169	{
170	AssertRC(rc);
171	return rc;
172	}
173	Assert(pPDDst);
174
175	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
176	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
177	PX86PDPAE pPDDst;
178	# if PGM_GST_TYPE == PGM_TYPE_PROT
179	/* AMD-V nested paging */
180	X86PML4E Pml4eSrc;
181	X86PDPE PdpeSrc;
182	PX86PML4E pPml4eSrc = &Pml4eSrc;
183
184	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
185	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
186	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
187	# endif
188
189	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
190	if (rc != VINF_SUCCESS)
191	{
192	AssertRC(rc);
193	return rc;
194	}
195	Assert(pPDDst);
196
197	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
198	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
199	PEPTPD pPDDst;
200
201	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
202	if (rc != VINF_SUCCESS)
203	{
204	AssertRC(rc);
205	return rc;
206	}
207	Assert(pPDDst);
208	# endif
209
210	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
211	/*
212	* If we successfully correct the write protection fault due to dirty bit
213	* tracking, or this page fault is a genuine one, then return immediately.
214	*/
215	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
216	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], pvFault);
217	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
218	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
219	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
220	{
221	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
222	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVCpu->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
223	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
224	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
225	}
226
227	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
228	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
229
230	/*
231	* A common case is the not-present error caused by lazy page table syncing.
232	*
233	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
234	* so we can safely assume that the shadow PT is present when calling SyncPage later.
235	*
236	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
237	* of mapping conflict and defer to SyncCR3 in R3.
238	* (Again, we do NOT support access handlers for non-present guest pages.)
239	*
240	*/
241	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
242	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
243	# else
244	GSTPDE PdeSrc;
245	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
246	PdeSrc.n.u1Present = 1;
247	PdeSrc.n.u1Write = 1;
248	PdeSrc.n.u1Accessed = 1;
249	PdeSrc.n.u1User = 1;
250	# endif
251	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
252	&& !pPDDst->a[iPDDst].n.u1Present
253	&& PdeSrc.n.u1Present
254	)
255
256	{
257	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2SyncPT; });
258	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
259	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
260	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, pvFault);
261	if (RT_SUCCESS(rc))
262	{
263	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
264	return rc;
265	}
266	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
267	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
268	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
269	return VINF_PGM_SYNC_CR3;
270	}
271
272	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
273	/*
274	* Check if this address is within any of our mappings.
275	*
276	* This is very fast and it's gonna save us a bit of effort below and prevent
277	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
278	* (BTW, it's impossible to have physical access handlers in a mapping.)
279	*/
280	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
281	{
282	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
283	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
284	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
285	{
286	if (pvFault < pMapping->GCPtr)
287	break;
288	if (pvFault - pMapping->GCPtr < pMapping->cb)
289	{
290	/*
291	* The first thing we check is if we've got an undetected conflict.
292	*/
293	if (!pVM->pgm.s.fMappingsFixed)
294	{
295	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
296	while (iPT-- > 0)
297	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
298	{
299	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eConflicts);
300	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
301	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
302	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
303	return VINF_PGM_SYNC_CR3;
304	}
305	}
306
307	/*
308	* Check if the fault address is in a virtual page access handler range.
309	*/
310	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
311	if ( pCur
312	&& pvFault - pCur->Core.Key < pCur->cb
313	&& uErr & X86_TRAP_PF_RW)
314	{
315	# ifdef IN_RC
316	STAM_PROFILE_START(&pCur->Stat, h);
317	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
318	STAM_PROFILE_STOP(&pCur->Stat, h);
319	# else
320	AssertFailed();
321	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
322	# endif
323	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersMapping);
324	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
325	return rc;
326	}
327
328	/*
329	* Pretend we're not here and let the guest handle the trap.
330	*/
331	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
332	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFMapping);
333	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
334	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
335	return VINF_EM_RAW_GUEST_TRAP;
336	}
337	}
338	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
339	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
340	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
341
342	/*
343	* Check if this fault address is flagged for special treatment,
344	* which means we'll have to figure out the physical address and
345	* check flags associated with it.
346	*
347	* ASSUME that we can limit any special access handling to pages
348	* in page tables which the guest believes to be present.
349	*/
350	if (PdeSrc.n.u1Present)
351	{
352	RTGCPHYS GCPhys = NIL_RTGCPHYS;
353
354	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
355	# if PGM_GST_TYPE == PGM_TYPE_AMD64
356	bool fBigPagesSupported = true;
357	# else
358	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
359	# endif
360	if ( PdeSrc.b.u1Size
361	&& fBigPagesSupported)
362	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc)
363	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
364	else
365	{
366	PGSTPT pPTSrc;
367	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
368	if (RT_SUCCESS(rc))
369	{
370	unsigned iPTESrc = (pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
371	if (pPTSrc->a[iPTESrc].n.u1Present)
372	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
373	}
374	}
375	# else
376	/* No paging so the fault address is the physical address */
377	GCPhys = (RTGCPHYS)(pvFault & ~PAGE_OFFSET_MASK);
378	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
379
380	/*
381	* If we have a GC address we'll check if it has any flags set.
382	*/
383	if (GCPhys != NIL_RTGCPHYS)
384	{
385	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
386
387	PPGMPAGE pPage;
388	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
389	if (RT_SUCCESS(rc)) /** just handle the failure immediate (it returns) and make things easier to read. */
390	{
391	if ( PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage)
392	\|\| PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage))
393	{
394	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
395	{
396	/*
397	* Physical page access handler.
398	*/
399	const RTGCPHYS GCPhysFault = GCPhys \| (pvFault & PAGE_OFFSET_MASK);
400	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
401	if (pCur)
402	{
403	# ifdef PGM_SYNC_N_PAGES
404	/*
405	* If the region is write protected and we got a page not present fault, then sync
406	* the pages. If the fault was caused by a read, then restart the instruction.
407	* In case of write access continue to the GC write handler.
408	*
409	* ASSUMES that there is only one handler per page or that they have similar write properties.
410	*/
411	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
412	&& !(uErr & X86_TRAP_PF_P))
413	{
414	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
415	if ( RT_FAILURE(rc)
416	\|\| !(uErr & X86_TRAP_PF_RW)
417	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
418	{
419	AssertRC(rc);
420	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
421	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
422	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
423	return rc;
424	}
425	}
426	# endif
427
428	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
429	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
430	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
431
432	# if defined(IN_RC) \|\| defined(IN_RING0)
433	if (pCur->CTX_SUFF(pfnHandler))
434	{
435	STAM_PROFILE_START(&pCur->Stat, h);
436	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pCur->CTX_SUFF(pvUser));
437	STAM_PROFILE_STOP(&pCur->Stat, h);
438	}
439	else
440	# endif
441	rc = VINF_EM_RAW_EMULATE_INSTR;
442	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersPhysical);
443	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
444	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndPhys; });
445	return rc;
446	}
447	}
448	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
449	else
450	{
451	# ifdef PGM_SYNC_N_PAGES
452	/*
453	* If the region is write protected and we got a page not present fault, then sync
454	* the pages. If the fault was caused by a read, then restart the instruction.
455	* In case of write access continue to the GC write handler.
456	*/
457	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
458	&& !(uErr & X86_TRAP_PF_P))
459	{
460	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
461	if ( RT_FAILURE(rc)
462	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
463	\|\| !(uErr & X86_TRAP_PF_RW))
464	{
465	AssertRC(rc);
466	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
467	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
468	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
469	return rc;
470	}
471	}
472	# endif
473	/*
474	* Ok, it's an virtual page access handler.
475	*
476	* Since it's faster to search by address, we'll do that first
477	* and then retry by GCPhys if that fails.
478	*/
479	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
480	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
481	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
482	*/
483	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
484	if (pCur)
485	{
486	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
487	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
488	\|\| !(uErr & X86_TRAP_PF_P)
489	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
490	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
491
492	if ( pvFault - pCur->Core.Key < pCur->cb
493	&& ( uErr & X86_TRAP_PF_RW
494	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
495	{
496	# ifdef IN_RC
497	STAM_PROFILE_START(&pCur->Stat, h);
498	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
499	STAM_PROFILE_STOP(&pCur->Stat, h);
500	# else
501	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
502	# endif
503	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtual);
504	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
505	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
506	return rc;
507	}
508	/* Unhandled part of a monitored page */
509	}
510	else
511	{
512	/* Check by physical address. */
513	PPGMVIRTHANDLER pCur;
514	unsigned iPage;
515	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + (pvFault & PAGE_OFFSET_MASK),
516	&pCur, &iPage);
517	Assert(RT_SUCCESS(rc) \|\| !pCur);
518	if ( pCur
519	&& ( uErr & X86_TRAP_PF_RW
520	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
521	{
522	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
523	# ifdef IN_RC
524	RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
525	Assert(off < pCur->cb);
526	STAM_PROFILE_START(&pCur->Stat, h);
527	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
528	STAM_PROFILE_STOP(&pCur->Stat, h);
529	# else
530	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
531	# endif
532	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualByPhys);
533	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
534	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
535	return rc;
536	}
537	}
538	}
539	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
540
541	/*
542	* There is a handled area of the page, but this fault doesn't belong to it.
543	* We must emulate the instruction.
544	*
545	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
546	* we first check if this was a page-not-present fault for a page with only
547	* write access handlers. Restart the instruction if it wasn't a write access.
548	*/
549	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersUnhandled);
550
551	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
552	&& !(uErr & X86_TRAP_PF_P))
553	{
554	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
555	if ( RT_FAILURE(rc)
556	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
557	\|\| !(uErr & X86_TRAP_PF_RW))
558	{
559	AssertRC(rc);
560	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
561	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
562	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
563	return rc;
564	}
565	}
566
567	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
568	* It's writing to an unhandled part of the LDT page several million times.
569	*/
570	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
571	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
572	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
573	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndUnhandled; });
574	return rc;
575	} /* if any kind of handler */
576
577	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
578	if (uErr & X86_TRAP_PF_P)
579	{
580	/*
581	* The page isn't marked, but it might still be monitored by a virtual page access handler.
582	* (ASSUMES no temporary disabling of virtual handlers.)
583	*/
584	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
585	* we should correct both the shadow page table and physical memory flags, and not only check for
586	* accesses within the handler region but for access to pages with virtual handlers. */
587	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
588	if (pCur)
589	{
590	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
591	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
592	\|\| !(uErr & X86_TRAP_PF_P)
593	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
594	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
595
596	if ( pvFault - pCur->Core.Key < pCur->cb
597	&& ( uErr & X86_TRAP_PF_RW
598	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
599	{
600	# ifdef IN_RC
601	STAM_PROFILE_START(&pCur->Stat, h);
602	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
603	STAM_PROFILE_STOP(&pCur->Stat, h);
604	# else
605	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
606	# endif
607	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualUnmarked);
608	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
609	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
610	return rc;
611	}
612	}
613	}
614	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
615	}
616	else
617	{
618	/*
619	* When the guest accesses invalid physical memory (e.g. probing
620	* of RAM or accessing a remapped MMIO range), then we'll fall
621	* back to the recompiler to emulate the instruction.
622	*/
623	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
624	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersInvalid);
625	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
626	return VINF_EM_RAW_EMULATE_INSTR;
627	}
628
629	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
630
631	# ifdef PGM_OUT_OF_SYNC_IN_GC /** @todo remove this bugger. */
632	/*
633	* We are here only if page is present in Guest page tables and
634	* trap is not handled by our handlers.
635	*
636	* Check it for page out-of-sync situation.
637	*/
638	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
639
640	if (!(uErr & X86_TRAP_PF_P))
641	{
642	/*
643	* Page is not present in our page tables.
644	* Try to sync it!
645	* BTW, fPageShw is invalid in this branch!
646	*/
647	if (uErr & X86_TRAP_PF_US)
648	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
649	else /* supervisor */
650	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
651
652	# if defined(LOG_ENABLED) && !defined(IN_RING0)
653	RTGCPHYS GCPhys;
654	uint64_t fPageGst;
655	PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
656	Log(("Page out of sync: %RGv eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%RGp scan=%d\n",
657	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
658	# endif /* LOG_ENABLED */
659
660	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
661	if (CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
662	{
663	uint64_t fPageGst;
664	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
665	if ( RT_SUCCESS(rc)
666	&& !(fPageGst & X86_PTE_US))
667	{
668	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
669	if ( pvFault == (RTGCPTR)pRegFrame->eip
670	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
671	# ifdef CSAM_DETECT_NEW_CODE_PAGES
672	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
673	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
674	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
675	)
676	{
677	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
678	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
679	if (rc != VINF_SUCCESS)
680	{
681	/*
682	* CSAM needs to perform a job in ring 3.
683	*
684	* Sync the page before going to the host context; otherwise we'll end up in a loop if
685	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
686	*/
687	LogFlow(("CSAM ring 3 job\n"));
688	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
689	AssertRC(rc2);
690
691	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
692	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2CSAM; });
693	return rc;
694	}
695	}
696	# ifdef CSAM_DETECT_NEW_CODE_PAGES
697	else if ( uErr == X86_TRAP_PF_RW
698	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
699	&& pRegFrame->ecx < 0x10000)
700	{
701	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
702	* to detect loading of new code pages.
703	*/
704
705	/*
706	* Decode the instruction.
707	*/
708	RTGCPTR PC;
709	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
710	if (rc == VINF_SUCCESS)
711	{
712	DISCPUSTATE Cpu;
713	uint32_t cbOp;
714	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, &Cpu, &cbOp);
715
716	/* For now we'll restrict this to rep movsw/d instructions */
717	if ( rc == VINF_SUCCESS
718	&& Cpu.pCurInstr->opcode == OP_MOVSWD
719	&& (Cpu.prefix & PREFIX_REP))
720	{
721	CSAMMarkPossibleCodePage(pVM, pvFault);
722	}
723	}
724	}
725	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
726
727	/*
728	* Mark this page as safe.
729	*/
730	/** @todo not correct for pages that contain both code and data!! */
731	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
732	CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
733	}
734	}
735	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
736	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
737	if (RT_SUCCESS(rc))
738	{
739	/* The page was successfully synced, return to the guest. */
740	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
741	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSync; });
742	return VINF_SUCCESS;
743	}
744	}
745	else /* uErr & X86_TRAP_PF_P: */
746	{
747	/*
748	* Write protected pages is make writable when the guest makes the first
749	* write to it. This happens for pages that are shared, write monitored
750	* and not yet allocated.
751	*
752	* Also, a side effect of not flushing global PDEs are out of sync pages due
753	* to physical monitored regions, that are no longer valid.
754	* Assume for now it only applies to the read/write flag.
755	*/
756	if (RT_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
757	{
758	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
759	{
760	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n",
761	GCPhys, pPage, pvFault, uErr));
762	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
763	if (rc != VINF_SUCCESS)
764	{
765	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
766	return rc;
767	}
768	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
769	return VINF_EM_NO_MEMORY;
770	}
771	/// @todo count the above case; else
772	if (uErr & X86_TRAP_PF_US)
773	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
774	else /* supervisor */
775	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
776
777	/*
778	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
779	* page is not present, which is not true in this case.
780	*/
781	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
782	if (RT_SUCCESS(rc))
783	{
784	/*
785	* Page was successfully synced, return to guest.
786	*/
787	# ifdef VBOX_STRICT
788	RTGCPHYS GCPhys;
789	uint64_t fPageGst;
790	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
791	Assert(RT_SUCCESS(rc) && fPageGst & X86_PTE_RW);
792	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
793
794	uint64_t fPageShw;
795	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
796	AssertMsg(RT_SUCCESS(rc) && fPageShw & X86_PTE_RW, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
797	# endif /* VBOX_STRICT */
798	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
799	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
800	return VINF_SUCCESS;
801	}
802
803	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
804	if ( CPUMGetGuestCPL(pVCpu, pRegFrame) == 0
805	&& ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG)
806	&& (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P))
807	{
808	uint64_t fPageGst;
809	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
810	if ( RT_SUCCESS(rc)
811	&& !(fPageGst & X86_PTE_RW))
812	{
813	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
814	if (RT_SUCCESS(rc))
815	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulInRZ);
816	else
817	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulToR3);
818	return rc;
819	}
820	AssertMsgFailed(("Unexpected r/w page %RGv flag=%x rc=%Rrc\n", pvFault, (uint32_t)fPageGst, rc));
821	}
822	}
823
824	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
825	# ifdef VBOX_STRICT
826	/*
827	* Check for VMM page flags vs. Guest page flags consistency.
828	* Currently only for debug purposes.
829	*/
830	if (RT_SUCCESS(rc))
831	{
832	/* Get guest page flags. */
833	uint64_t fPageGst;
834	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
835	if (RT_SUCCESS(rc))
836	{
837	uint64_t fPageShw;
838	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
839
840	/*
841	* Compare page flags.
842	* Note: we have AVL, A, D bits desynched.
843	*/
844	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
845	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
846	}
847	else
848	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
849	}
850	else
851	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
852	# endif /* VBOX_STRICT */
853	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
854	}
855	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
856	# endif /* PGM_OUT_OF_SYNC_IN_GC */
857	}
858	else /* GCPhys == NIL_RTGCPHYS */
859	{
860	/*
861	* Page not present in Guest OS or invalid page table address.
862	* This is potential virtual page access handler food.
863	*
864	* For the present we'll say that our access handlers don't
865	* work for this case - we've already discarded the page table
866	* not present case which is identical to this.
867	*
868	* When we perchance find we need this, we will probably have AVL
869	* trees (offset based) to operate on and we can measure their speed
870	* agains mapping a page table and probably rearrange this handling
871	* a bit. (Like, searching virtual ranges before checking the
872	* physical address.)
873	*/
874	}
875	}
876	/* else: !present (guest) */
877
878
879	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
880	/*
881	* Conclusion, this is a guest trap.
882	*/
883	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
884	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFUnh);
885	return VINF_EM_RAW_GUEST_TRAP;
886	# else
887	/* present, but not a monitored page; perhaps the guest is probing physical memory */
888	return VINF_EM_RAW_EMULATE_INSTR;
889	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
890
891
892	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
893
894	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
895	return VERR_INTERNAL_ERROR;
896	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
897	}
898	#endif /* !IN_RING3 */
899
900
901	/**
902	* Emulation of the invlpg instruction.
903	*
904	*
905	* @returns VBox status code.
906	*
907	* @param pVCpu The VMCPU handle.
908	* @param GCPtrPage Page to invalidate.
909	*
910	* @remark ASSUMES that the guest is updating before invalidating. This order
911	* isn't required by the CPU, so this is speculative and could cause
912	* trouble.
913	*
914	* @todo Flush page or page directory only if necessary!
915	* @todo Add a #define for simply invalidating the page.
916	*/
917	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
918	{
919	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
920	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
921	&& PGM_SHW_TYPE != PGM_TYPE_EPT
922	int rc;
923	PVM pVM = pVCpu->CTX_SUFF(pVM);
924
925	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
926	/*
927	* Get the shadow PD entry and skip out if this PD isn't present.
928	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
929	*/
930	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
931	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
932	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
933
934	/* Fetch the pgm pool shadow descriptor. */
935	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
936	Assert(pShwPde);
937
938	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
939	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
940	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
941
942	/* If the shadow PDPE isn't present, then skip the invalidate. */
943	if (!pPdptDst->a[iPdpt].n.u1Present)
944	{
945	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
946	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
947	return VINF_SUCCESS;
948	}
949
950	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
951	PPGMPOOLPAGE pShwPde = NULL;
952	PX86PDPAE pPDDst;
953
954	/* Fetch the pgm pool shadow descriptor. */
955	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
956	AssertRCSuccessReturn(rc, rc);
957	Assert(pShwPde);
958
959	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
960	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
961
962	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
963	/* PML4 */
964	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
965	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
966	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
967	PX86PDPAE pPDDst;
968	PX86PDPT pPdptDst;
969	PX86PML4E pPml4eDst;
970	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
971	if (rc != VINF_SUCCESS)
972	{
973	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
974	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
975	if (!VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
976	PGM_INVL_GUEST_TLBS();
977	return VINF_SUCCESS;
978	}
979	Assert(pPDDst);
980
981	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
982	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
983
984	if (!pPdpeDst->n.u1Present)
985	{
986	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
987	if (!VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
988	PGM_INVL_GUEST_TLBS();
989	return VINF_SUCCESS;
990	}
991
992	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
993
994	const SHWPDE PdeDst = *pPdeDst;
995	if (!PdeDst.n.u1Present)
996	{
997	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
998	return VINF_SUCCESS;
999	}
1000
1001	# if defined(IN_RC)
1002	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1003	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1004	# endif
1005
1006	/*
1007	* Get the guest PD entry and calc big page.
1008	*/
1009	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1010	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
1011	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1012	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1013	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1014	unsigned iPDSrc = 0;
1015	# if PGM_GST_TYPE == PGM_TYPE_PAE
1016	X86PDPE PdpeSrc;
1017	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
1018	# else /* AMD64 */
1019	PX86PML4E pPml4eSrc;
1020	X86PDPE PdpeSrc;
1021	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
1022	# endif
1023	GSTPDE PdeSrc;
1024
1025	if (pPDSrc)
1026	PdeSrc = pPDSrc->a[iPDSrc];
1027	else
1028	PdeSrc.u = 0;
1029	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1030
1031	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1032	const bool fIsBigPage = PdeSrc.b.u1Size;
1033	# else
1034	const bool fIsBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1035	# endif
1036
1037	# ifdef IN_RING3
1038	/*
1039	* If a CR3 Sync is pending we may ignore the invalidate page operation
1040	* depending on the kind of sync and if it's a global page or not.
1041	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1042	*/
1043	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1044	if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
1045	\|\| ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
1046	&& fIsBigPage
1047	&& PdeSrc.b.u1Global
1048	)
1049	)
1050	# else
1051	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1052	# endif
1053	{
1054	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1055	return VINF_SUCCESS;
1056	}
1057	# endif /* IN_RING3 */
1058
1059	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1060	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1061
1062	/* Fetch the pgm pool shadow descriptor. */
1063	PPGMPOOLPAGE pShwPdpt = pgmPoolGetPageByHCPhys(pVM, pPml4eDst->u & X86_PML4E_PG_MASK);
1064	Assert(pShwPdpt);
1065
1066	/* Fetch the pgm pool shadow descriptor. */
1067	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1068	Assert(pShwPde);
1069
1070	Assert(pPml4eDst->n.u1Present && (pPml4eDst->u & SHW_PDPT_MASK));
1071	RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1072
1073	if ( !pPml4eSrc->n.u1Present
1074	\|\| pShwPdpt->GCPhys != GCPhysPdpt)
1075	{
1076	LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1077	GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1078	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1079	pPml4eDst->u = 0;
1080	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1081	PGM_INVL_GUEST_TLBS();
1082	return VINF_SUCCESS;
1083	}
1084	if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
1085	\|\| (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
1086	{
1087	/*
1088	* Mark not present so we can resync the PML4E when it's used.
1089	*/
1090	LogFlow(("InvalidatePage: Out-of-sync PML4E at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1091	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1092	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1093	pPml4eDst->u = 0;
1094	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1095	PGM_INVL_GUEST_TLBS();
1096	}
1097	else if (!pPml4eSrc->n.u1Accessed)
1098	{
1099	/*
1100	* Mark not present so we can set the accessed bit.
1101	*/
1102	LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1103	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1104	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1105	pPml4eDst->u = 0;
1106	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1107	PGM_INVL_GUEST_TLBS();
1108	}
1109
1110	/* Check if the PDPT entry has changed. */
1111	Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1112	RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1113	if ( !PdpeSrc.n.u1Present
1114	\|\| pShwPde->GCPhys != GCPhysPd)
1115	{
1116	LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1117	GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1118	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1119	pPdpeDst->u = 0;
1120	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1121	PGM_INVL_GUEST_TLBS();
1122	return VINF_SUCCESS;
1123	}
1124	if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1125	\|\| (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1126	{
1127	/*
1128	* Mark not present so we can resync the PDPTE when it's used.
1129	*/
1130	LogFlow(("InvalidatePage: Out-of-sync PDPE at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1131	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1132	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1133	pPdpeDst->u = 0;
1134	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1135	PGM_INVL_GUEST_TLBS();
1136	}
1137	else if (!PdpeSrc.lm.u1Accessed)
1138	{
1139	/*
1140	* Mark not present so we can set the accessed bit.
1141	*/
1142	LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1143	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1144	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1145	pPdpeDst->u = 0;
1146	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1147	PGM_INVL_GUEST_TLBS();
1148	}
1149	# endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1150
1151
1152	/*
1153	* Deal with the Guest PDE.
1154	*/
1155	rc = VINF_SUCCESS;
1156	if (PdeSrc.n.u1Present)
1157	{
1158	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1159	{
1160	/*
1161	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1162	*/
1163	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1164	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1165	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1166	}
1167	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1168	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1169	{
1170	/*
1171	* Mark not present so we can resync the PDE when it's used.
1172	*/
1173	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1174	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1175	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1176	pPdeDst->u = 0;
1177	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1178	PGM_INVL_GUEST_TLBS();
1179	}
1180	else if (!PdeSrc.n.u1Accessed)
1181	{
1182	/*
1183	* Mark not present so we can set the accessed bit.
1184	*/
1185	LogFlow(("InvalidatePage: Out-of-sync (A) at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1186	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1187	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1188	pPdeDst->u = 0;
1189	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1190	PGM_INVL_GUEST_TLBS();
1191	}
1192	else if (!fIsBigPage)
1193	{
1194	/*
1195	* 4KB - page.
1196	*/
1197	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1198	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1199	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1200	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1201	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1202	# endif
1203	if (pShwPage->GCPhys == GCPhys)
1204	{
1205	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1206	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1207	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1208	if (pPT->a[iPTEDst].n.u1Present)
1209	{
1210	# ifdef PGMPOOL_WITH_USER_TRACKING
1211	/* This is very unlikely with caching/monitoring enabled. */
1212	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1213	# endif
1214	pPT->a[iPTEDst].u = 0;
1215	}
1216	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1217	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1218	if (RT_SUCCESS(rc))
1219	rc = VINF_SUCCESS;
1220	# endif
1221	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1222	PGM_INVL_PG(GCPtrPage);
1223	}
1224	else
1225	{
1226	/*
1227	* The page table address changed.
1228	*/
1229	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1230	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1231	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1232	pPdeDst->u = 0;
1233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1234	PGM_INVL_GUEST_TLBS();
1235	}
1236	}
1237	else
1238	{
1239	/*
1240	* 2/4MB - page.
1241	*/
1242	/* Before freeing the page, check if anything really changed. */
1243	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1244	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1245	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1246	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1247	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1248	# endif
1249	if ( pShwPage->GCPhys == GCPhys
1250	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1251	{
1252	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1253	/** @todo PAT */
1254	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1255	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1256	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1257	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1258	{
1259	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1260	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1261	# if defined(IN_RC)
1262	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1263	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1264	# endif
1265	return VINF_SUCCESS;
1266	}
1267	}
1268
1269	/*
1270	* Ok, the page table is present and it's been changed in the guest.
1271	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1272	* We could do this for some flushes in GC too, but we need an algorithm for
1273	* deciding which 4MB pages containing code likely to be executed very soon.
1274	*/
1275	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1276	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1277	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1278	pPdeDst->u = 0;
1279	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1280	PGM_INVL_BIG_PG(GCPtrPage);
1281	}
1282	}
1283	else
1284	{
1285	/*
1286	* Page directory is not present, mark shadow PDE not present.
1287	*/
1288	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1289	{
1290	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1291	pPdeDst->u = 0;
1292	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1293	PGM_INVL_PG(GCPtrPage);
1294	}
1295	else
1296	{
1297	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1298	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDMappings));
1299	}
1300	}
1301	# if defined(IN_RC)
1302	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1303	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1304	# endif
1305	return rc;
1306
1307	#else /* guest real and protected mode */
1308	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1309	return VINF_SUCCESS;
1310	#endif
1311	}
1312
1313
1314	#ifdef PGMPOOL_WITH_USER_TRACKING
1315	/**
1316	* Update the tracking of shadowed pages.
1317	*
1318	* @param pVCpu The VMCPU handle.
1319	* @param pShwPage The shadow page.
1320	* @param HCPhys The physical page we is being dereferenced.
1321	*/
1322	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1323	{
1324	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1325	PVM pVM = pVCpu->CTX_SUFF(pVM);
1326
1327	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1328	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1329
1330	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1331	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1332	* 2. write protect all shadowed pages. I.e. implement caching.
1333	*/
1334	/*
1335	* Find the guest address.
1336	*/
1337	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1338	pRam;
1339	pRam = pRam->CTX_SUFF(pNext))
1340	{
1341	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1342	while (iPage-- > 0)
1343	{
1344	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1345	{
1346	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1347	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1348	pShwPage->cPresent--;
1349	pPool->cPresent--;
1350	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1351	return;
1352	}
1353	}
1354	}
1355
1356	for (;;)
1357	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1358	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1359	pShwPage->cPresent--;
1360	pVM->pgm.s.CTX_SUFF(pPool)->cPresent--;
1361	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1362	}
1363
1364
1365	/**
1366	* Update the tracking of shadowed pages.
1367	*
1368	* @param pVCpu The VMCPU handle.
1369	* @param pShwPage The shadow page.
1370	* @param u16 The top 16-bit of the pPage->HCPhys.
1371	* @param pPage Pointer to the guest page. this will be modified.
1372	* @param iPTDst The index into the shadow table.
1373	*/
1374	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1375	{
1376	PVM pVM = pVCpu->CTX_SUFF(pVM);
1377	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1378	/*
1379	* Just deal with the simple first time here.
1380	*/
1381	if (!u16)
1382	{
1383	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1384	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1385	}
1386	else
1387	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1388
1389	/* write back */
1390	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1391	PGM_PAGE_SET_TRACKING(pPage, u16);
1392
1393	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1394
1395	/* update statistics. */
1396	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1397	pShwPage->cPresent++;
1398	if (pShwPage->iFirstPresent > iPTDst)
1399	pShwPage->iFirstPresent = iPTDst;
1400	}
1401	#endif /* PGMPOOL_WITH_USER_TRACKING */
1402
1403
1404	/**
1405	* Creates a 4K shadow page for a guest page.
1406	*
1407	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1408	* physical address. The PdeSrc argument only the flags are used. No page structured
1409	* will be mapped in this function.
1410	*
1411	* @param pVCpu The VMCPU handle.
1412	* @param pPteDst Destination page table entry.
1413	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1414	* Can safely assume that only the flags are being used.
1415	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1416	* @param pShwPage Pointer to the shadow page.
1417	* @param iPTDst The index into the shadow table.
1418	*
1419	* @remark Not used for 2/4MB pages!
1420	*/
1421	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1422	{
1423	if (PteSrc.n.u1Present)
1424	{
1425	PVM pVM = pVCpu->CTX_SUFF(pVM);
1426
1427	/*
1428	* Find the ram range.
1429	*/
1430	PPGMPAGE pPage;
1431	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1432	if (RT_SUCCESS(rc))
1433	{
1434	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1435	/* Try make the page writable if necessary. */
1436	if ( PteSrc.n.u1Write
1437	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1438	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1439	{
1440	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1441	AssertRC(rc);
1442	}
1443	#endif
1444
1445	/** @todo investiage PWT, PCD and PAT. */
1446	/*
1447	* Make page table entry.
1448	*/
1449	SHWPTE PteDst;
1450	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1451	{
1452	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1453	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1454	{
1455	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1456	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1457	PteDst.n.u1Present = 1;
1458	PteDst.n.u1Execute = 1;
1459	PteDst.n.u1IgnorePAT = 1;
1460	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1461	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1462	#else
1463	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1464	\| PGM_PAGE_GET_HCPHYS(pPage);
1465	#endif
1466	}
1467	else
1468	{
1469	LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", PGM_PAGE_GET_HCPHYS(pPage)));
1470	PteDst.u = 0;
1471	}
1472	/** @todo count these two kinds. */
1473	}
1474	else
1475	{
1476	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1477	/*
1478	* If the page or page directory entry is not marked accessed,
1479	* we mark the page not present.
1480	*/
1481	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1482	{
1483	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1484	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,AccessedPage));
1485	PteDst.u = 0;
1486	}
1487	else
1488	/*
1489	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1490	* when the page is modified.
1491	*/
1492	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1493	{
1494	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPage));
1495	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1496	\| PGM_PAGE_GET_HCPHYS(pPage)
1497	\| PGM_PTFLAGS_TRACK_DIRTY;
1498	}
1499	else
1500	#endif
1501	{
1502	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageSkipped));
1503	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1504	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1505	PteDst.n.u1Present = 1;
1506	PteDst.n.u1Write = 1;
1507	PteDst.n.u1Execute = 1;
1508	PteDst.n.u1IgnorePAT = 1;
1509	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1510	/* PteDst.n.u1Size = 0 */
1511	#else
1512	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1513	\| PGM_PAGE_GET_HCPHYS(pPage);
1514	#endif
1515	}
1516	}
1517
1518	/*
1519	* Make sure only allocated pages are mapped writable.
1520	*/
1521	if ( PteDst.n.u1Write
1522	&& PteDst.n.u1Present
1523	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1524	{
1525	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. */
1526	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1527	}
1528
1529	#ifdef PGMPOOL_WITH_USER_TRACKING
1530	/*
1531	* Keep user track up to date.
1532	*/
1533	if (PteDst.n.u1Present)
1534	{
1535	if (!pPteDst->n.u1Present)
1536	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1537	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1538	{
1539	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1540	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1541	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1542	}
1543	}
1544	else if (pPteDst->n.u1Present)
1545	{
1546	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1547	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1548	}
1549	#endif /* PGMPOOL_WITH_USER_TRACKING */
1550
1551	/*
1552	* Update statistics and commit the entry.
1553	*/
1554	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1555	if (!PteSrc.n.u1Global)
1556	pShwPage->fSeenNonGlobal = true;
1557	#endif
1558	*pPteDst = PteDst;
1559	}
1560	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1561	/** @todo count these. */
1562	}
1563	else
1564	{
1565	/*
1566	* Page not-present.
1567	*/
1568	LogFlow(("SyncPageWorker: page not present in Pte\n"));
1569	#ifdef PGMPOOL_WITH_USER_TRACKING
1570	/* Keep user track up to date. */
1571	if (pPteDst->n.u1Present)
1572	{
1573	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1574	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1575	}
1576	#endif /* PGMPOOL_WITH_USER_TRACKING */
1577	pPteDst->u = 0;
1578	/** @todo count these. */
1579	}
1580	}
1581
1582
1583	/**
1584	* Syncs a guest OS page.
1585	*
1586	* There are no conflicts at this point, neither is there any need for
1587	* page table allocations.
1588	*
1589	* @returns VBox status code.
1590	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1591	* @param pVCpu The VMCPU handle.
1592	* @param PdeSrc Page directory entry of the guest.
1593	* @param GCPtrPage Guest context page address.
1594	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1595	* @param uErr Fault error (X86_TRAP_PF_*).
1596	*/
1597	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1598	{
1599	PVM pVM = pVCpu->CTX_SUFF(pVM);
1600	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1601
1602	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1603	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1604	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1605	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1606	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1607
1608	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1609	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
1610	# endif
1611
1612	/*
1613	* Assert preconditions.
1614	*/
1615	Assert(PdeSrc.n.u1Present);
1616	Assert(cPages);
1617	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1618
1619	/*
1620	* Get the shadow PDE, find the shadow page table in the pool.
1621	*/
1622	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1623	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1624	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
1625
1626	/* Fetch the pgm pool shadow descriptor. */
1627	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1628	Assert(pShwPde);
1629
1630	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1631	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1632	PPGMPOOLPAGE pShwPde = NULL;
1633	PX86PDPAE pPDDst;
1634
1635	/* Fetch the pgm pool shadow descriptor. */
1636	int rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1637	AssertRCSuccessReturn(rc, rc);
1638	Assert(pShwPde);
1639
1640	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1641	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1642
1643	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1644	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1645	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1646	PX86PDPAE pPDDst;
1647	PX86PDPT pPdptDst;
1648
1649	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1650	AssertRCSuccessReturn(rc, rc);
1651	Assert(pPDDst && pPdptDst);
1652	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1653	# endif
1654
1655	SHWPDE PdeDst = *pPdeDst;
1656	AssertMsg(PdeDst.n.u1Present, ("%p=%llx\n", pPdeDst, (uint64_t)PdeDst.u));
1657	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1658
1659	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1660	/* Fetch the pgm pool shadow descriptor. */
1661	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1662	Assert(pShwPde);
1663	# endif
1664
1665	# if defined(IN_RC)
1666	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1667	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1668	# endif
1669
1670	/*
1671	* Check that the page is present and that the shadow PDE isn't out of sync.
1672	*/
1673	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1674	const bool fBigPage = PdeSrc.b.u1Size;
1675	# else
1676	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1677	# endif
1678	RTGCPHYS GCPhys;
1679	if (!fBigPage)
1680	{
1681	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1682	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1683	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1684	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1685	# endif
1686	}
1687	else
1688	{
1689	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1690	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1691	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1692	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1693	# endif
1694	}
1695	if ( pShwPage->GCPhys == GCPhys
1696	&& PdeSrc.n.u1Present
1697	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1698	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1699	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1700	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1701	# endif
1702	)
1703	{
1704	/*
1705	* Check that the PDE is marked accessed already.
1706	* Since we set the accessed bit before getting here on a #PF, this
1707	* check is only meant for dealing with non-#PF'ing paths.
1708	*/
1709	if (PdeSrc.n.u1Accessed)
1710	{
1711	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1712	if (!fBigPage)
1713	{
1714	/*
1715	* 4KB Page - Map the guest page table.
1716	*/
1717	PGSTPT pPTSrc;
1718	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1719	if (RT_SUCCESS(rc))
1720	{
1721	# ifdef PGM_SYNC_N_PAGES
1722	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1723	if ( cPages > 1
1724	&& !(uErr & X86_TRAP_PF_P)
1725	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1726	{
1727	/*
1728	* This code path is currently only taken when the caller is PGMTrap0eHandler
1729	* for non-present pages!
1730	*
1731	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1732	* deal with locality.
1733	*/
1734	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1735	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1736	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1737	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1738	# else
1739	const unsigned offPTSrc = 0;
1740	# endif
1741	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1742	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1743	iPTDst = 0;
1744	else
1745	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1746	for (; iPTDst < iPTDstEnd; iPTDst++)
1747	{
1748	if (!pPTDst->a[iPTDst].n.u1Present)
1749	{
1750	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1751	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1752	NOREF(GCPtrCurPage);
1753	#ifndef IN_RING0
1754	/*
1755	* Assuming kernel code will be marked as supervisor - and not as user level
1756	* and executed using a conforming code selector - And marked as readonly.
1757	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1758	*/
1759	PPGMPAGE pPage;
1760	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1761	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1762	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1763	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1764	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1765	)
1766	#endif /* else: CSAM not active */
1767	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1768	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1769	GCPtrCurPage, PteSrc.n.u1Present,
1770	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1771	PteSrc.n.u1User & PdeSrc.n.u1User,
1772	(uint64_t)PteSrc.u,
1773	(uint64_t)pPTDst->a[iPTDst].u,
1774	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1775	}
1776	}
1777	}
1778	else
1779	# endif /* PGM_SYNC_N_PAGES */
1780	{
1781	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1782	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1783	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1784	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1785	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1786	GCPtrPage, PteSrc.n.u1Present,
1787	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1788	PteSrc.n.u1User & PdeSrc.n.u1User,
1789	(uint64_t)PteSrc.u,
1790	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1791	}
1792	}
1793	else /* MMIO or invalid page: emulated in #PF handler. */
1794	{
1795	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1796	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1797	}
1798	}
1799	else
1800	{
1801	/*
1802	* 4/2MB page - lazy syncing shadow 4K pages.
1803	* (There are many causes of getting here, it's no longer only CSAM.)
1804	*/
1805	/* Calculate the GC physical address of this 4KB shadow page. */
1806	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1807	/* Find ram range. */
1808	PPGMPAGE pPage;
1809	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1810	if (RT_SUCCESS(rc))
1811	{
1812	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1813	/* Try make the page writable if necessary. */
1814	if ( PdeSrc.n.u1Write
1815	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1816	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1817	{
1818	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
1819	AssertRC(rc);
1820	}
1821	# endif
1822
1823	/*
1824	* Make shadow PTE entry.
1825	*/
1826	SHWPTE PteDst;
1827	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1828	\| PGM_PAGE_GET_HCPHYS(pPage);
1829	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1830	{
1831	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1832	PteDst.n.u1Write = 0;
1833	else
1834	PteDst.u = 0;
1835	}
1836	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1837	# ifdef PGMPOOL_WITH_USER_TRACKING
1838	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1839	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1840	# endif
1841	/* Make sure only allocated pages are mapped writable. */
1842	if ( PteDst.n.u1Write
1843	&& PteDst.n.u1Present
1844	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1845	{
1846	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1847	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1848	}
1849
1850	pPTDst->a[iPTDst] = PteDst;
1851
1852
1853	/*
1854	* If the page is not flagged as dirty and is writable, then make it read-only
1855	* at PD level, so we can set the dirty bit when the page is modified.
1856	*
1857	* ASSUMES that page access handlers are implemented on page table entry level.
1858	* Thus we will first catch the dirty access and set PDE.D and restart. If
1859	* there is an access handler, we'll trap again and let it work on the problem.
1860	*/
1861	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1862	* As for invlpg, it simply frees the whole shadow PT.
1863	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1864	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1865	{
1866	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
1867	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1868	PdeDst.n.u1Write = 0;
1869	}
1870	else
1871	{
1872	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1873	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1874	}
1875	*pPdeDst = PdeDst;
1876	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1877	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1878	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1879	}
1880	else
1881	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1882	}
1883	# if defined(IN_RC)
1884	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1885	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1886	# endif
1887	return VINF_SUCCESS;
1888	}
1889	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDNAs));
1890	}
1891	else
1892	{
1893	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1894	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1895	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1896	}
1897
1898	/*
1899	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1900	* Yea, I'm lazy.
1901	*/
1902	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1903	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1904
1905	pPdeDst->u = 0;
1906
1907	# if defined(IN_RC)
1908	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1909	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1910	# endif
1911	PGM_INVL_GUEST_TLBS();
1912	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1913
1914	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1915	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1916	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1917	&& !defined(IN_RC)
1918
1919	# ifdef PGM_SYNC_N_PAGES
1920	/*
1921	* Get the shadow PDE, find the shadow page table in the pool.
1922	*/
1923	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1924	X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
1925
1926	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1927	X86PDEPAE PdeDst = pgmShwGetPaePDE(&pVCpu->pgm.s, GCPtrPage);
1928
1929	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1930	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1931	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
1932	PX86PDPAE pPDDst;
1933	X86PDEPAE PdeDst;
1934	PX86PDPT pPdptDst;
1935
1936	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1937	AssertRCSuccessReturn(rc, rc);
1938	Assert(pPDDst && pPdptDst);
1939	PdeDst = pPDDst->a[iPDDst];
1940	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
1941	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1942	PEPTPD pPDDst;
1943	EPTPDE PdeDst;
1944
1945	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
1946	if (rc != VINF_SUCCESS)
1947	{
1948	AssertRC(rc);
1949	return rc;
1950	}
1951	Assert(pPDDst);
1952	PdeDst = pPDDst->a[iPDDst];
1953	# endif
1954	AssertMsg(PdeDst.n.u1Present, ("%#llx\n", (uint64_t)PdeDst.u));
1955	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1956	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1957
1958	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1959	if ( cPages > 1
1960	&& !(uErr & X86_TRAP_PF_P)
1961	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1962	{
1963	/*
1964	* This code path is currently only taken when the caller is PGMTrap0eHandler
1965	* for non-present pages!
1966	*
1967	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1968	* deal with locality.
1969	*/
1970	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1971	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1972	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1973	iPTDst = 0;
1974	else
1975	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1976	for (; iPTDst < iPTDstEnd; iPTDst++)
1977	{
1978	if (!pPTDst->a[iPTDst].n.u1Present)
1979	{
1980	GSTPTE PteSrc;
1981
1982	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
1983
1984	/* Fake the page table entry */
1985	PteSrc.u = GCPtrCurPage;
1986	PteSrc.n.u1Present = 1;
1987	PteSrc.n.u1Dirty = 1;
1988	PteSrc.n.u1Accessed = 1;
1989	PteSrc.n.u1Write = 1;
1990	PteSrc.n.u1User = 1;
1991
1992	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1993
1994	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1995	GCPtrCurPage, PteSrc.n.u1Present,
1996	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1997	PteSrc.n.u1User & PdeSrc.n.u1User,
1998	(uint64_t)PteSrc.u,
1999	(uint64_t)pPTDst->a[iPTDst].u,
2000	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2001
2002	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2003	break;
2004	}
2005	else
2006	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2007	}
2008	}
2009	else
2010	# endif /* PGM_SYNC_N_PAGES */
2011	{
2012	GSTPTE PteSrc;
2013	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2014	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2015
2016	/* Fake the page table entry */
2017	PteSrc.u = GCPtrCurPage;
2018	PteSrc.n.u1Present = 1;
2019	PteSrc.n.u1Dirty = 1;
2020	PteSrc.n.u1Accessed = 1;
2021	PteSrc.n.u1Write = 1;
2022	PteSrc.n.u1User = 1;
2023	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2024
2025	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2026	GCPtrPage, PteSrc.n.u1Present,
2027	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2028	PteSrc.n.u1User & PdeSrc.n.u1User,
2029	(uint64_t)PteSrc.u,
2030	(uint64_t)pPTDst->a[iPTDst].u,
2031	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2032	}
2033	return VINF_SUCCESS;
2034
2035	#else
2036	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2037	return VERR_INTERNAL_ERROR;
2038	#endif
2039	}
2040
2041
2042	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2043	/**
2044	* Investigate page fault and handle write protection page faults caused by
2045	* dirty bit tracking.
2046	*
2047	* @returns VBox status code.
2048	* @param pVCpu The VMCPU handle.
2049	* @param uErr Page fault error code.
2050	* @param pPdeDst Shadow page directory entry.
2051	* @param pPdeSrc Guest page directory entry.
2052	* @param GCPtrPage Guest context page address.
2053	*/
2054	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage)
2055	{
2056	bool fWriteProtect = !!(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP);
2057	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
2058	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
2059	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2060	bool fBigPagesSupported = true;
2061	# else
2062	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2063	# endif
2064	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2065	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2066	# endif
2067	unsigned uPageFaultLevel;
2068	int rc;
2069	PVM pVM = pVCpu->CTX_SUFF(pVM);
2070
2071	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2072	LogFlow(("CheckPageFault: GCPtrPage=%RGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
2073
2074	# if PGM_GST_TYPE == PGM_TYPE_PAE \
2075	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2076
2077	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2078	PX86PML4E pPml4eSrc;
2079	PX86PDPE pPdpeSrc;
2080
2081	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc);
2082	Assert(pPml4eSrc);
2083
2084	/*
2085	* Real page fault? (PML4E level)
2086	*/
2087	if ( (uErr & X86_TRAP_PF_RSVD)
2088	\|\| !pPml4eSrc->n.u1Present
2089	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
2090	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2091	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
2092	)
2093	{
2094	uPageFaultLevel = 0;
2095	goto l_UpperLevelPageFault;
2096	}
2097	Assert(pPdpeSrc);
2098
2099	# else /* PAE */
2100	PX86PDPE pPdpeSrc = pgmGstGetPaePDPEPtr(&pVCpu->pgm.s, GCPtrPage);
2101	# endif /* PAE */
2102
2103	/*
2104	* Real page fault? (PDPE level)
2105	*/
2106	if ( (uErr & X86_TRAP_PF_RSVD)
2107	\|\| !pPdpeSrc->n.u1Present
2108	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
2109	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
2110	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
2111	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
2112	# endif
2113	)
2114	{
2115	uPageFaultLevel = 1;
2116	goto l_UpperLevelPageFault;
2117	}
2118	# endif
2119
2120	/*
2121	* Real page fault? (PDE level)
2122	*/
2123	if ( (uErr & X86_TRAP_PF_RSVD)
2124	\|\| !pPdeSrc->n.u1Present
2125	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2126	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
2127	# endif
2128	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2129	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
2130	{
2131	uPageFaultLevel = 2;
2132	goto l_UpperLevelPageFault;
2133	}
2134
2135	/*
2136	* First check the easy case where the page directory has been marked read-only to track
2137	* the dirty bit of an emulated BIG page
2138	*/
2139	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2140	{
2141	/* Mark guest page directory as accessed */
2142	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2143	pPml4eSrc->n.u1Accessed = 1;
2144	pPdpeSrc->lm.u1Accessed = 1;
2145	# endif
2146	pPdeSrc->b.u1Accessed = 1;
2147
2148	/*
2149	* Only write protection page faults are relevant here.
2150	*/
2151	if (fWriteFault)
2152	{
2153	/* Mark guest page directory as dirty (BIG page only). */
2154	pPdeSrc->b.u1Dirty = 1;
2155
2156	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2157	{
2158	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2159
2160	Assert(pPdeSrc->b.u1Write);
2161
2162	pPdeDst->n.u1Write = 1;
2163	pPdeDst->n.u1Accessed = 1;
2164	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2165	PGM_INVL_BIG_PG(GCPtrPage);
2166	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2167	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2168	}
2169	}
2170	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2171	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2172	}
2173	/* else: 4KB page table */
2174
2175	/*
2176	* Map the guest page table.
2177	*/
2178	PGSTPT pPTSrc;
2179	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2180	if (RT_SUCCESS(rc))
2181	{
2182	/*
2183	* Real page fault?
2184	*/
2185	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2186	const GSTPTE PteSrc = *pPteSrc;
2187	if ( !PteSrc.n.u1Present
2188	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2189	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2190	# endif
2191	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2192	\|\| (fUserLevelFault && !PteSrc.n.u1User)
2193	)
2194	{
2195	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2196	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2197	LogFlow(("CheckPageFault: real page fault at %RGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2198
2199	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2200	* See the 2nd case above as well.
2201	*/
2202	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2203	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2204
2205	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2206	return VINF_EM_RAW_GUEST_TRAP;
2207	}
2208	LogFlow(("CheckPageFault: page fault at %RGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2209
2210	/*
2211	* Set the accessed bits in the page directory and the page table.
2212	*/
2213	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2214	pPml4eSrc->n.u1Accessed = 1;
2215	pPdpeSrc->lm.u1Accessed = 1;
2216	# endif
2217	pPdeSrc->n.u1Accessed = 1;
2218	pPteSrc->n.u1Accessed = 1;
2219
2220	/*
2221	* Only write protection page faults are relevant here.
2222	*/
2223	if (fWriteFault)
2224	{
2225	/* Write access, so mark guest entry as dirty. */
2226	# ifdef VBOX_WITH_STATISTICS
2227	if (!pPteSrc->n.u1Dirty)
2228	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtiedPage));
2229	else
2230	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageAlreadyDirty));
2231	# endif
2232
2233	pPteSrc->n.u1Dirty = 1;
2234
2235	if (pPdeDst->n.u1Present)
2236	{
2237	#ifndef IN_RING0
2238	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2239	* Our individual shadow handlers will provide more information and force a fatal exit.
2240	*/
2241	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2242	{
2243	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2244	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2245	return VINF_SUCCESS;
2246	}
2247	#endif
2248	/*
2249	* Map shadow page table.
2250	*/
2251	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2252	if (pShwPage)
2253	{
2254	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2255	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2256	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
2257	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
2258	{
2259	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2260	# ifdef VBOX_STRICT
2261	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2262	if (pPage)
2263	AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
2264	("Unexpected dirty bit tracking on monitored page %RGv (phys %RGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
2265	# endif
2266	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2267
2268	Assert(pPteSrc->n.u1Write);
2269
2270	pPteDst->n.u1Write = 1;
2271	pPteDst->n.u1Dirty = 1;
2272	pPteDst->n.u1Accessed = 1;
2273	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2274	PGM_INVL_PG(GCPtrPage);
2275
2276	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2277	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2278	}
2279	}
2280	else
2281	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2282	}
2283	}
2284	/** @todo Optimize accessed bit emulation? */
2285	# ifdef VBOX_STRICT
2286	/*
2287	* Sanity check.
2288	*/
2289	else if ( !pPteSrc->n.u1Dirty
2290	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2291	&& pPdeDst->n.u1Present)
2292	{
2293	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2294	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2295	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2296	if ( pPteDst->n.u1Present
2297	&& pPteDst->n.u1Write)
2298	LogFlow(("Writable present page %RGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2299	}
2300	# endif /* VBOX_STRICT */
2301	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2302	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2303	}
2304	AssertRC(rc);
2305	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2306	return rc;
2307
2308
2309	l_UpperLevelPageFault:
2310	/*
2311	* Pagefault detected while checking the PML4E, PDPE or PDE.
2312	* Single exit handler to get rid of duplicate code paths.
2313	*/
2314	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2315	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2316	Log(("CheckPageFault: real page fault at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2317
2318	if (
2319	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2320	pPml4eSrc->n.u1Present &&
2321	# endif
2322	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2323	pPdpeSrc->n.u1Present &&
2324	# endif
2325	pPdeSrc->n.u1Present)
2326	{
2327	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2328	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2329	{
2330	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2331	}
2332	else
2333	{
2334	/*
2335	* Map the guest page table.
2336	*/
2337	PGSTPT pPTSrc;
2338	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2339	if (RT_SUCCESS(rc))
2340	{
2341	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2342	const GSTPTE PteSrc = *pPteSrc;
2343	if (pPteSrc->n.u1Present)
2344	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2345	}
2346	AssertRC(rc);
2347	}
2348	}
2349	return VINF_EM_RAW_GUEST_TRAP;
2350	}
2351	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2352
2353
2354	/**
2355	* Sync a shadow page table.
2356	*
2357	* The shadow page table is not present. This includes the case where
2358	* there is a conflict with a mapping.
2359	*
2360	* @returns VBox status code.
2361	* @param pVCpu The VMCPU handle.
2362	* @param iPD Page directory index.
2363	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2364	* Assume this is a temporary mapping.
2365	* @param GCPtrPage GC Pointer of the page that caused the fault
2366	*/
2367	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2368	{
2369	PVM pVM = pVCpu->CTX_SUFF(pVM);
2370
2371	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2372	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2373	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2374
2375	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2376	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2377	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2378	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2379	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2380
2381	int rc = VINF_SUCCESS;
2382
2383	/*
2384	* Validate input a little bit.
2385	*/
2386	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2387	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2388	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2389	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2390
2391	/* Fetch the pgm pool shadow descriptor. */
2392	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2393	Assert(pShwPde);
2394
2395	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2396	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2397	PPGMPOOLPAGE pShwPde = NULL;
2398	PX86PDPAE pPDDst;
2399	PSHWPDE pPdeDst;
2400
2401	/* Fetch the pgm pool shadow descriptor. */
2402	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2403	AssertRCSuccessReturn(rc, rc);
2404	Assert(pShwPde);
2405
2406	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2407	pPdeDst = &pPDDst->a[iPDDst];
2408
2409	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2410	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2411	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2412	PX86PDPAE pPDDst;
2413	PX86PDPT pPdptDst;
2414	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2415	AssertRCSuccessReturn(rc, rc);
2416	Assert(pPDDst);
2417	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2418	# endif
2419	SHWPDE PdeDst = *pPdeDst;
2420
2421	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2422	/* Fetch the pgm pool shadow descriptor. */
2423	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2424	Assert(pShwPde);
2425	# endif
2426
2427	# ifndef PGM_WITHOUT_MAPPINGS
2428	/*
2429	* Check for conflicts.
2430	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2431	* HC: Simply resolve the conflict.
2432	*/
2433	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2434	{
2435	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2436	# ifndef IN_RING3
2437	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2438	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2439	return VERR_ADDRESS_CONFLICT;
2440	# else
2441	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2442	Assert(pMapping);
2443	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2444	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2445	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2446	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2447	# else
2448	AssertFailed(); /* can't happen for amd64 */
2449	# endif
2450	if (RT_FAILURE(rc))
2451	{
2452	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2453	return rc;
2454	}
2455	PdeDst = *pPdeDst;
2456	# endif
2457	}
2458	# else /* PGM_WITHOUT_MAPPINGS */
2459	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2460	# endif /* PGM_WITHOUT_MAPPINGS */
2461	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2462
2463	# if defined(IN_RC)
2464	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2465	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
2466	# endif
2467
2468	/*
2469	* Sync page directory entry.
2470	*/
2471	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2472	if (PdeSrc.n.u1Present)
2473	{
2474	/*
2475	* Allocate & map the page table.
2476	*/
2477	PSHWPT pPTDst;
2478	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2479	const bool fPageTable = !PdeSrc.b.u1Size;
2480	# else
2481	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2482	# endif
2483	PPGMPOOLPAGE pShwPage;
2484	RTGCPHYS GCPhys;
2485	if (fPageTable)
2486	{
2487	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2488	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2489	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2490	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2491	# endif
2492	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2493	}
2494	else
2495	{
2496	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
2497	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2498	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2499	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2500	# endif
2501	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, pShwPde->idx, iPDDst, &pShwPage);
2502	}
2503	if (rc == VINF_SUCCESS)
2504	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2505	else if (rc == VINF_PGM_CACHED_PAGE)
2506	{
2507	/*
2508	* The PT was cached, just hook it up.
2509	*/
2510	if (fPageTable)
2511	PdeDst.u = pShwPage->Core.Key
2512	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2513	else
2514	{
2515	PdeDst.u = pShwPage->Core.Key
2516	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2517	/* (see explanation and assumptions further down.) */
2518	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2519	{
2520	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2521	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2522	PdeDst.b.u1Write = 0;
2523	}
2524	}
2525	*pPdeDst = PdeDst;
2526	# if defined(IN_RC)
2527	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2528	# endif
2529	return VINF_SUCCESS;
2530	}
2531	else if (rc == VERR_PGM_POOL_FLUSHED)
2532	{
2533	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
2534	# if defined(IN_RC)
2535	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2536	# endif
2537	return VINF_PGM_SYNC_CR3;
2538	}
2539	else
2540	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2541	PdeDst.u &= X86_PDE_AVL_MASK;
2542	PdeDst.u \|= pShwPage->Core.Key;
2543
2544	/*
2545	* Page directory has been accessed (this is a fault situation, remember).
2546	*/
2547	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2548	if (fPageTable)
2549	{
2550	/*
2551	* Page table - 4KB.
2552	*
2553	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2554	*/
2555	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2556	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2557	PGSTPT pPTSrc;
2558	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2559	if (RT_SUCCESS(rc))
2560	{
2561	/*
2562	* Start by syncing the page directory entry so CSAM's TLB trick works.
2563	*/
2564	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2565	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2566	*pPdeDst = PdeDst;
2567	# if defined(IN_RC)
2568	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2569	# endif
2570
2571	/*
2572	* Directory/page user or supervisor privilege: (same goes for read/write)
2573	*
2574	* Directory Page Combined
2575	* U/S U/S U/S
2576	* 0 0 0
2577	* 0 1 0
2578	* 1 0 0
2579	* 1 1 1
2580	*
2581	* Simple AND operation. Table listed for completeness.
2582	*
2583	*/
2584	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4K));
2585	# ifdef PGM_SYNC_N_PAGES
2586	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2587	unsigned iPTDst = iPTBase;
2588	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2589	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2590	iPTDst = 0;
2591	else
2592	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2593	# else /* !PGM_SYNC_N_PAGES */
2594	unsigned iPTDst = 0;
2595	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2596	# endif /* !PGM_SYNC_N_PAGES */
2597	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2598	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2599	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2600	# else
2601	const unsigned offPTSrc = 0;
2602	# endif
2603	for (; iPTDst < iPTDstEnd; iPTDst++)
2604	{
2605	const unsigned iPTSrc = iPTDst + offPTSrc;
2606	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2607
2608	if (PteSrc.n.u1Present) /* we've already cleared it above */
2609	{
2610	# ifndef IN_RING0
2611	/*
2612	* Assuming kernel code will be marked as supervisor - and not as user level
2613	* and executed using a conforming code selector - And marked as readonly.
2614	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2615	*/
2616	PPGMPAGE pPage;
2617	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2618	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2619	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2620	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2621	)
2622	# endif
2623	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2624	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2625	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
2626	PteSrc.n.u1Present,
2627	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2628	PteSrc.n.u1User & PdeSrc.n.u1User,
2629	(uint64_t)PteSrc.u,
2630	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2631	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2632	}
2633	} /* for PTEs */
2634	}
2635	}
2636	else
2637	{
2638	/*
2639	* Big page - 2/4MB.
2640	*
2641	* We'll walk the ram range list in parallel and optimize lookups.
2642	* We will only sync on shadow page table at a time.
2643	*/
2644	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4M));
2645
2646	/**
2647	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2648	*/
2649
2650	/*
2651	* Start by syncing the page directory entry.
2652	*/
2653	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2654	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2655
2656	/*
2657	* If the page is not flagged as dirty and is writable, then make it read-only
2658	* at PD level, so we can set the dirty bit when the page is modified.
2659	*
2660	* ASSUMES that page access handlers are implemented on page table entry level.
2661	* Thus we will first catch the dirty access and set PDE.D and restart. If
2662	* there is an access handler, we'll trap again and let it work on the problem.
2663	*/
2664	/** @todo move the above stuff to a section in the PGM documentation. */
2665	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2666	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2667	{
2668	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2669	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2670	PdeDst.b.u1Write = 0;
2671	}
2672	*pPdeDst = PdeDst;
2673	# if defined(IN_RC)
2674	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2675	# endif
2676
2677	/*
2678	* Fill the shadow page table.
2679	*/
2680	/* Get address and flags from the source PDE. */
2681	SHWPTE PteDstBase;
2682	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2683
2684	/* Loop thru the entries in the shadow PT. */
2685	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2686	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2687	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2688	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2689	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2690	unsigned iPTDst = 0;
2691	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2692	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2693	{
2694	/* Advance ram range list. */
2695	while (pRam && GCPhys > pRam->GCPhysLast)
2696	pRam = pRam->CTX_SUFF(pNext);
2697	if (pRam && GCPhys >= pRam->GCPhys)
2698	{
2699	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2700	do
2701	{
2702	/* Make shadow PTE. */
2703	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2704	SHWPTE PteDst;
2705
2706	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2707	/* Try make the page writable if necessary. */
2708	if ( PteDstBase.n.u1Write
2709	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2710	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2711	{
2712	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
2713	AssertRCReturn(rc, rc);
2714	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2715	break;
2716	}
2717	# endif
2718
2719	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2720	{
2721	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2722	{
2723	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2724	PteDst.n.u1Write = 0;
2725	}
2726	else
2727	PteDst.u = 0;
2728	}
2729	# ifndef IN_RING0
2730	/*
2731	* Assuming kernel code will be marked as supervisor and not as user level and executed
2732	* using a conforming code selector. Don't check for readonly, as that implies the whole
2733	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2734	*/
2735	else if ( !PdeSrc.n.u1User
2736	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2737	PteDst.u = 0;
2738	# endif
2739	else
2740	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2741
2742	/* Only map writable pages writable. */
2743	if ( PteDst.n.u1Write
2744	&& PteDst.n.u1Present
2745	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2746	{
2747	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2748	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2749	}
2750
2751	# ifdef PGMPOOL_WITH_USER_TRACKING
2752	if (PteDst.n.u1Present)
2753	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2754	# endif
2755	/* commit it */
2756	pPTDst->a[iPTDst] = PteDst;
2757	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2758	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2759	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2760
2761	/* advance */
2762	GCPhys += PAGE_SIZE;
2763	iHCPage++;
2764	iPTDst++;
2765	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2766	&& GCPhys <= pRam->GCPhysLast);
2767	}
2768	else if (pRam)
2769	{
2770	Log(("Invalid pages at %RGp\n", GCPhys));
2771	do
2772	{
2773	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2774	GCPhys += PAGE_SIZE;
2775	iPTDst++;
2776	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2777	&& GCPhys < pRam->GCPhys);
2778	}
2779	else
2780	{
2781	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2782	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2783	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2784	}
2785	} /* while more PTEs */
2786	} /* 4KB / 4MB */
2787	}
2788	else
2789	AssertRelease(!PdeDst.n.u1Present);
2790
2791	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2792	if (RT_FAILURE(rc))
2793	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPTFailed));
2794	return rc;
2795
2796	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2797	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2798	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2799	&& !defined(IN_RC)
2800
2801	/*
2802	* Validate input a little bit.
2803	*/
2804	int rc = VINF_SUCCESS;
2805	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2806	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2807	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2808
2809	/* Fetch the pgm pool shadow descriptor. */
2810	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2811	Assert(pShwPde);
2812
2813	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2814	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2815	PPGMPOOLPAGE pShwPde;
2816	PX86PDPAE pPDDst;
2817	PSHWPDE pPdeDst;
2818
2819	/* Fetch the pgm pool shadow descriptor. */
2820	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2821	AssertRCSuccessReturn(rc, rc);
2822	Assert(pShwPde);
2823
2824	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2825	pPdeDst = &pPDDst->a[iPDDst];
2826
2827	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2828	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2829	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2830	PX86PDPAE pPDDst;
2831	PX86PDPT pPdptDst;
2832	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2833	AssertRCSuccessReturn(rc, rc);
2834	Assert(pPDDst);
2835	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2836
2837	/* Fetch the pgm pool shadow descriptor. */
2838	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2839	Assert(pShwPde);
2840
2841	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2842	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2843	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2844	PEPTPD pPDDst;
2845	PEPTPDPT pPdptDst;
2846
2847	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2848	if (rc != VINF_SUCCESS)
2849	{
2850	AssertRC(rc);
2851	return rc;
2852	}
2853	Assert(pPDDst);
2854	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2855
2856	/* Fetch the pgm pool shadow descriptor. */
2857	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2858	Assert(pShwPde);
2859	# endif
2860	SHWPDE PdeDst = *pPdeDst;
2861
2862	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2863	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2864
2865	GSTPDE PdeSrc;
2866	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2867	PdeSrc.n.u1Present = 1;
2868	PdeSrc.n.u1Write = 1;
2869	PdeSrc.n.u1Accessed = 1;
2870	PdeSrc.n.u1User = 1;
2871
2872	/*
2873	* Allocate & map the page table.
2874	*/
2875	PSHWPT pPTDst;
2876	PPGMPOOLPAGE pShwPage;
2877	RTGCPHYS GCPhys;
2878
2879	/* Virtual address = physical address */
2880	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
2881	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2882
2883	if ( rc == VINF_SUCCESS
2884	\|\| rc == VINF_PGM_CACHED_PAGE)
2885	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2886	else
2887	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2888
2889	PdeDst.u &= X86_PDE_AVL_MASK;
2890	PdeDst.u \|= pShwPage->Core.Key;
2891	PdeDst.n.u1Present = 1;
2892	PdeDst.n.u1Write = 1;
2893	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2894	PdeDst.n.u1Execute = 1;
2895	# else
2896	PdeDst.n.u1User = 1;
2897	PdeDst.n.u1Accessed = 1;
2898	# endif
2899	*pPdeDst = PdeDst;
2900
2901	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2902	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2903	return rc;
2904
2905	#else
2906	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2907	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2908	return VERR_INTERNAL_ERROR;
2909	#endif
2910	}
2911
2912
2913
2914	/**
2915	* Prefetch a page/set of pages.
2916	*
2917	* Typically used to sync commonly used pages before entering raw mode
2918	* after a CR3 reload.
2919	*
2920	* @returns VBox status code.
2921	* @param pVCpu The VMCPU handle.
2922	* @param GCPtrPage Page to invalidate.
2923	*/
2924	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
2925	{
2926	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2927	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
2928	/*
2929	* Check that all Guest levels thru the PDE are present, getting the
2930	* PD and PDE in the processes.
2931	*/
2932	int rc = VINF_SUCCESS;
2933	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2934	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2935	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
2936	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
2937	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2938	unsigned iPDSrc;
2939	X86PDPE PdpeSrc;
2940	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
2941	if (!pPDSrc)
2942	return VINF_SUCCESS; /* not present */
2943	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
2944	unsigned iPDSrc;
2945	PX86PML4E pPml4eSrc;
2946	X86PDPE PdpeSrc;
2947	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
2948	if (!pPDSrc)
2949	return VINF_SUCCESS; /* not present */
2950	# endif
2951	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2952	# else
2953	PGSTPD pPDSrc = NULL;
2954	const unsigned iPDSrc = 0;
2955	GSTPDE PdeSrc;
2956
2957	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2958	PdeSrc.n.u1Present = 1;
2959	PdeSrc.n.u1Write = 1;
2960	PdeSrc.n.u1Accessed = 1;
2961	PdeSrc.n.u1User = 1;
2962	# endif
2963
2964	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2965	{
2966	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2967	const X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
2968	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2969	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2970	PX86PDPAE pPDDst;
2971	X86PDEPAE PdeDst;
2972	# if PGM_GST_TYPE != PGM_TYPE_PAE
2973	X86PDPE PdpeSrc;
2974
2975	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
2976	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
2977	# endif
2978	int rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
2979	if (rc != VINF_SUCCESS)
2980	{
2981	AssertRC(rc);
2982	return rc;
2983	}
2984	Assert(pPDDst);
2985	PdeDst = pPDDst->a[iPDDst];
2986
2987	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2988	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2989	PX86PDPAE pPDDst;
2990	X86PDEPAE PdeDst;
2991
2992	# if PGM_GST_TYPE == PGM_TYPE_PROT
2993	/* AMD-V nested paging */
2994	X86PML4E Pml4eSrc;
2995	X86PDPE PdpeSrc;
2996	PX86PML4E pPml4eSrc = &Pml4eSrc;
2997
2998	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
2999	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3000	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3001	# endif
3002
3003	int rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3004	if (rc != VINF_SUCCESS)
3005	{
3006	AssertRC(rc);
3007	return rc;
3008	}
3009	Assert(pPDDst);
3010	PdeDst = pPDDst->a[iPDDst];
3011	# endif
3012	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3013	{
3014	if (!PdeDst.n.u1Present)
3015	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
3016	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3017	else
3018	{
3019	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3020	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3021	* makes no sense to prefetch more than one page.
3022	*/
3023	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3024	if (RT_SUCCESS(rc))
3025	rc = VINF_SUCCESS;
3026	}
3027	}
3028	}
3029	return rc;
3030
3031	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3032	return VINF_SUCCESS; /* ignore */
3033	#endif
3034	}
3035
3036
3037
3038
3039	/**
3040	* Syncs a page during a PGMVerifyAccess() call.
3041	*
3042	* @returns VBox status code (informational included).
3043	* @param pVCpu The VMCPU handle.
3044	* @param GCPtrPage The address of the page to sync.
3045	* @param fPage The effective guest page flags.
3046	* @param uErr The trap error code.
3047	*/
3048	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3049	{
3050	PVM pVM = pVCpu->CTX_SUFF(pVM);
3051
3052	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3053
3054	Assert(!HWACCMIsNestedPagingActive(pVM));
3055	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_TYPE_AMD64) \
3056	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3057
3058	# ifndef IN_RING0
3059	if (!(fPage & X86_PTE_US))
3060	{
3061	/*
3062	* Mark this page as safe.
3063	*/
3064	/** @todo not correct for pages that contain both code and data!! */
3065	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3066	CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
3067	}
3068	# endif
3069
3070	/*
3071	* Get guest PD and index.
3072	*/
3073	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3074	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3075	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3076	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3077	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3078	unsigned iPDSrc = 0;
3079	X86PDPE PdpeSrc;
3080	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3081
3082	if (pPDSrc)
3083	{
3084	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3085	return VINF_EM_RAW_GUEST_TRAP;
3086	}
3087	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3088	unsigned iPDSrc;
3089	PX86PML4E pPml4eSrc;
3090	X86PDPE PdpeSrc;
3091	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3092	if (!pPDSrc)
3093	{
3094	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3095	return VINF_EM_RAW_GUEST_TRAP;
3096	}
3097	# endif
3098	# else
3099	PGSTPD pPDSrc = NULL;
3100	const unsigned iPDSrc = 0;
3101	# endif
3102	int rc = VINF_SUCCESS;
3103
3104	/*
3105	* First check if the shadow pd is present.
3106	*/
3107	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3108	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
3109	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3110	PX86PDEPAE pPdeDst;
3111	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3112	PX86PDPAE pPDDst;
3113	# if PGM_GST_TYPE != PGM_TYPE_PAE
3114	X86PDPE PdpeSrc;
3115
3116	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3117	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3118	# endif
3119	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3120	if (rc != VINF_SUCCESS)
3121	{
3122	AssertRC(rc);
3123	return rc;
3124	}
3125	Assert(pPDDst);
3126	pPdeDst = &pPDDst->a[iPDDst];
3127
3128	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3129	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3130	PX86PDPAE pPDDst;
3131	PX86PDEPAE pPdeDst;
3132
3133	# if PGM_GST_TYPE == PGM_TYPE_PROT
3134	/* AMD-V nested paging */
3135	X86PML4E Pml4eSrc;
3136	X86PDPE PdpeSrc;
3137	PX86PML4E pPml4eSrc = &Pml4eSrc;
3138
3139	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3140	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3141	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3142	# endif
3143
3144	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3145	if (rc != VINF_SUCCESS)
3146	{
3147	AssertRC(rc);
3148	return rc;
3149	}
3150	Assert(pPDDst);
3151	pPdeDst = &pPDDst->a[iPDDst];
3152	# endif
3153
3154	if (!pPdeDst->n.u1Present)
3155	{
3156	# if defined(IN_RC)
3157	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3158	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
3159	# endif
3160	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3161	# if defined(IN_RC)
3162	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3163	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3164	# endif
3165	AssertRC(rc);
3166	if (rc != VINF_SUCCESS)
3167	return rc;
3168	}
3169
3170	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3171	/* Check for dirty bit fault */
3172	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3173	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3174	Log(("PGMVerifyAccess: success (dirty)\n"));
3175	else
3176	{
3177	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3178	#else
3179	{
3180	GSTPDE PdeSrc;
3181	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3182	PdeSrc.n.u1Present = 1;
3183	PdeSrc.n.u1Write = 1;
3184	PdeSrc.n.u1Accessed = 1;
3185	PdeSrc.n.u1User = 1;
3186
3187	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
3188	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3189	if (uErr & X86_TRAP_PF_US)
3190	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
3191	else /* supervisor */
3192	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3193
3194	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3195	if (RT_SUCCESS(rc))
3196	{
3197	/* Page was successfully synced */
3198	Log2(("PGMVerifyAccess: success (sync)\n"));
3199	rc = VINF_SUCCESS;
3200	}
3201	else
3202	{
3203	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", GCPtrPage, rc));
3204	return VINF_EM_RAW_GUEST_TRAP;
3205	}
3206	}
3207	return rc;
3208
3209	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3210
3211	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3212	return VERR_INTERNAL_ERROR;
3213	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3214	}
3215
3216
3217	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3218	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
3219	/**
3220	* Figures out which kind of shadow page this guest PDE warrants.
3221	*
3222	* @returns Shadow page kind.
3223	* @param pPdeSrc The guest PDE in question.
3224	* @param cr4 The current guest cr4 value.
3225	*/
3226	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
3227	{
3228	# if PMG_GST_TYPE == PGM_TYPE_AMD64
3229	if (!pPdeSrc->n.u1Size)
3230	# else
3231	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
3232	# endif
3233	return BTH_PGMPOOLKIND_PT_FOR_PT;
3234	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
3235	//{
3236	// case 0:
3237	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
3238	// case X86_PDE4M_RW:
3239	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
3240	// case X86_PDE4M_US:
3241	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
3242	// case X86_PDE4M_RW \| X86_PDE4M_US:
3243	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
3244	# if 0
3245	// case X86_PDE4M_PAE_NX:
3246	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
3247	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
3248	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
3249	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
3250	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
3251	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
3252	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
3253	# endif
3254	return BTH_PGMPOOLKIND_PT_FOR_BIG;
3255	//}
3256	}
3257	# endif
3258	#endif
3259
3260	#undef MY_STAM_COUNTER_INC
3261	#define MY_STAM_COUNTER_INC(a) do { } while (0)
3262
3263
3264	/**
3265	* Syncs the paging hierarchy starting at CR3.
3266	*
3267	* @returns VBox status code, no specials.
3268	* @param pVCpu The VMCPU handle.
3269	* @param cr0 Guest context CR0 register
3270	* @param cr3 Guest context CR3 register
3271	* @param cr4 Guest context CR4 register
3272	* @param fGlobal Including global page directories or not
3273	*/
3274	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3275	{
3276	PVM pVM = pVCpu->CTX_SUFF(pVM);
3277
3278	if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
3279	fGlobal = true; /* Change this CR3 reload to be a global one. */
3280
3281	LogFlow(("SyncCR3 %d\n", fGlobal));
3282
3283	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3284	/*
3285	* Update page access handlers.
3286	* The virtual are always flushed, while the physical are only on demand.
3287	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3288	* have to look into that later because it will have a bad influence on the performance.
3289	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3290	* bird: Yes, but that won't work for aliases.
3291	*/
3292	/** @todo this MUST go away. See #1557. */
3293	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3294	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3295	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3296	#endif
3297
3298	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3299	/*
3300	* Nested / EPT - almost no work.
3301	*/
3302	/** @todo check if this is really necessary; the call does it as well... */
3303	HWACCMFlushTLB(pVM);
3304	return VINF_SUCCESS;
3305
3306	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3307	/*
3308	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3309	* out the shadow parts when the guest modifies its tables.
3310	*/
3311	return VINF_SUCCESS;
3312
3313	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3314
3315	# ifdef PGM_WITHOUT_MAPPINGS
3316	Assert(pVM->pgm.s.fMappingsFixed);
3317	return VINF_SUCCESS;
3318	# else
3319	/* Nothing to do when mappings are fixed. */
3320	if (pVM->pgm.s.fMappingsFixed)
3321	return VINF_SUCCESS;
3322
3323	int rc = PGMMapResolveConflicts(pVM);
3324	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3325	if (rc == VINF_PGM_SYNC_CR3)
3326	{
3327	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3328	return VINF_PGM_SYNC_CR3;
3329	}
3330	# endif
3331	return VINF_SUCCESS;
3332	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3333	}
3334
3335
3336
3337
3338	#ifdef VBOX_STRICT
3339	#ifdef IN_RC
3340	# undef AssertMsgFailed
3341	# define AssertMsgFailed Log
3342	#endif
3343	#ifdef IN_RING3
3344	# include <VBox/dbgf.h>
3345
3346	/**
3347	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3348	*
3349	* @returns VBox status code (VINF_SUCCESS).
3350	* @param cr3 The root of the hierarchy.
3351	* @param crr The cr4, only PAE and PSE is currently used.
3352	* @param fLongMode Set if long mode, false if not long mode.
3353	* @param cMaxDepth Number of levels to dump.
3354	* @param pHlp Pointer to the output functions.
3355	*/
3356	__BEGIN_DECLS
3357	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3358	__END_DECLS
3359
3360	#endif
3361
3362	/**
3363	* Checks that the shadow page table is in sync with the guest one.
3364	*
3365	* @returns The number of errors.
3366	* @param pVM The virtual machine.
3367	* @param pVCpu The VMCPU handle.
3368	* @param cr3 Guest context CR3 register
3369	* @param cr4 Guest context CR4 register
3370	* @param GCPtr Where to start. Defaults to 0.
3371	* @param cb How much to check. Defaults to everything.
3372	*/
3373	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3374	{
3375	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3376	return 0;
3377	#else
3378	unsigned cErrors = 0;
3379	PVM pVM = pVCpu->CTX_SUFF(pVM);
3380
3381	#if PGM_GST_TYPE == PGM_TYPE_PAE
3382	/** @todo currently broken; crashes below somewhere */
3383	AssertFailed();
3384	#endif
3385
3386	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3387	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3388	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3389
3390	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3391	bool fBigPagesSupported = true;
3392	# else
3393	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
3394	# endif
3395	PPGMCPU pPGM = &pVCpu->pgm.s;
3396	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3397	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3398	# ifndef IN_RING0
3399	RTHCPHYS HCPhys; /* general usage. */
3400	# endif
3401	int rc;
3402
3403	/*
3404	* Check that the Guest CR3 and all its mappings are correct.
3405	*/
3406	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3407	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3408	false);
3409	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3410	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3411	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3412	# else
3413	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3414	# endif
3415	AssertRCReturn(rc, 1);
3416	HCPhys = NIL_RTHCPHYS;
3417	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3418	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3419	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3420	pgmGstGet32bitPDPtr(pPGM);
3421	RTGCPHYS GCPhys;
3422	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3423	AssertRCReturn(rc, 1);
3424	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3425	# endif
3426	# endif /* !IN_RING0 */
3427
3428	/*
3429	* Get and check the Shadow CR3.
3430	*/
3431	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3432	unsigned cPDEs = X86_PG_ENTRIES;
3433	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3434	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3435	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3436	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3437	# else
3438	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3439	# endif
3440	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3441	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3442	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3443	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3444	# endif
3445	if (cb != ~(RTGCPTR)0)
3446	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3447
3448	/** @todo call the other two PGMAssert() functions. /
3449
3450	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3451	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3452	# endif
3453
3454	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3455	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3456
3457	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3458	{
3459	PPGMPOOLPAGE pShwPdpt = NULL;
3460	PX86PML4E pPml4eSrc;
3461	PX86PML4E pPml4eDst;
3462	RTGCPHYS GCPhysPdptSrc;
3463
3464	pPml4eSrc = pgmGstGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3465	pPml4eDst = pgmShwGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3466
3467	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3468	if (!pPml4eDst->n.u1Present)
3469	{
3470	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3471	continue;
3472	}
3473
3474	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3475	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3476
3477	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3478	{
3479	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3480	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3481	cErrors++;
3482	continue;
3483	}
3484
3485	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3486	{
3487	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3488	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3489	cErrors++;
3490	continue;
3491	}
3492
3493	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3494	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3495	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3496	{
3497	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3498	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3499	cErrors++;
3500	continue;
3501	}
3502	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3503	{
3504	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3505
3506	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3507	/*
3508	* Check the PDPTEs too.
3509	*/
3510	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3511
3512	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3513	{
3514	unsigned iPDSrc;
3515	PPGMPOOLPAGE pShwPde = NULL;
3516	PX86PDPE pPdpeDst;
3517	RTGCPHYS GCPhysPdeSrc;
3518	# if PGM_GST_TYPE == PGM_TYPE_PAE
3519	X86PDPE PdpeSrc;
3520	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtr, &iPDSrc, &PdpeSrc);
3521	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
3522	# else
3523	PX86PML4E pPml4eSrc;
3524	X86PDPE PdpeSrc;
3525	PX86PDPT pPdptDst;
3526	PX86PDPAE pPDDst;
3527	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3528
3529	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3530	if (rc != VINF_SUCCESS)
3531	{
3532	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3533	GCPtr += 512 * _2M;
3534	continue; /* next PDPTE */
3535	}
3536	Assert(pPDDst);
3537	# endif
3538	Assert(iPDSrc == 0);
3539
3540	pPdpeDst = &pPdptDst->a[iPdpt];
3541
3542	if (!pPdpeDst->n.u1Present)
3543	{
3544	GCPtr += 512 * _2M;
3545	continue; /* next PDPTE */
3546	}
3547
3548	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3549	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3550
3551	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3552	{
3553	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3554	GCPtr += 512 * _2M;
3555	cErrors++;
3556	continue;
3557	}
3558
3559	if (GCPhysPdeSrc != pShwPde->GCPhys)
3560	{
3561	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3562	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3563	# else
3564	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3565	# endif
3566	GCPtr += 512 * _2M;
3567	cErrors++;
3568	continue;
3569	}
3570
3571	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3572	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3573	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3574	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3575	{
3576	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3577	GCPtr += 512 * _2M;
3578	cErrors++;
3579	continue;
3580	}
3581	# endif
3582
3583	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3584	{
3585	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3586	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3587	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3588	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3589	PCX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
3590	# endif
3591	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3592	/*
3593	* Iterate the shadow page directory.
3594	*/
3595	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3596	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3597
3598	for (;
3599	iPDDst < cPDEs;
3600	iPDDst++, GCPtr += cIncrement)
3601	{
3602	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3603	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pPGM, GCPtr);
3604	# else
3605	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3606	# endif
3607	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3608	{
3609	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3610	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3611	{
3612	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3613	cErrors++;
3614	continue;
3615	}
3616	}
3617	else if ( (PdeDst.u & X86_PDE_P)
3618	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3619	)
3620	{
3621	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3622	PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3623	if (!pPoolPage)
3624	{
3625	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3626	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3627	cErrors++;
3628	continue;
3629	}
3630	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3631
3632	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3633	{
3634	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3635	GCPtr, (uint64_t)PdeDst.u));
3636	cErrors++;
3637	}
3638
3639	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3640	{
3641	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3642	GCPtr, (uint64_t)PdeDst.u));
3643	cErrors++;
3644	}
3645
3646	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3647	if (!PdeSrc.n.u1Present)
3648	{
3649	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3650	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3651	cErrors++;
3652	continue;
3653	}
3654
3655	if ( !PdeSrc.b.u1Size
3656	\|\| !fBigPagesSupported)
3657	{
3658	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3659	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3660	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3661	# endif
3662	}
3663	else
3664	{
3665	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3666	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3667	{
3668	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3669	GCPtr, (uint64_t)PdeSrc.u));
3670	cErrors++;
3671	continue;
3672	}
3673	# endif
3674	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3675	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3676	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3677	# endif
3678	}
3679
3680	if ( pPoolPage->enmKind
3681	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3682	{
3683	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3684	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3685	cErrors++;
3686	}
3687
3688	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3689	if (!pPhysPage)
3690	{
3691	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3692	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3693	cErrors++;
3694	continue;
3695	}
3696
3697	if (GCPhysGst != pPoolPage->GCPhys)
3698	{
3699	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3700	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3701	cErrors++;
3702	continue;
3703	}
3704
3705	if ( !PdeSrc.b.u1Size
3706	\|\| !fBigPagesSupported)
3707	{
3708	/*
3709	* Page Table.
3710	*/
3711	const GSTPT *pPTSrc;
3712	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3713	if (RT_FAILURE(rc))
3714	{
3715	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3716	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3717	cErrors++;
3718	continue;
3719	}
3720	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3721	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3722	{
3723	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3724	// (This problem will go away when/if we shadow multiple CR3s.)
3725	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3726	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3727	cErrors++;
3728	continue;
3729	}
3730	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3731	{
3732	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3733	GCPtr, (uint64_t)PdeDst.u));
3734	cErrors++;
3735	continue;
3736	}
3737
3738	/* iterate the page table. */
3739	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3740	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3741	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3742	# else
3743	const unsigned offPTSrc = 0;
3744	# endif
3745	for (unsigned iPT = 0, off = 0;
3746	iPT < RT_ELEMENTS(pPTDst->a);
3747	iPT++, off += PAGE_SIZE)
3748	{
3749	const SHWPTE PteDst = pPTDst->a[iPT];
3750
3751	/* skip not-present entries. */
3752	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3753	continue;
3754	Assert(PteDst.n.u1Present);
3755
3756	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3757	if (!PteSrc.n.u1Present)
3758	{
3759	# ifdef IN_RING3
3760	PGMAssertHandlerAndFlagsInSync(pVM);
3761	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3762	# endif
3763	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3764	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3765	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3766	cErrors++;
3767	continue;
3768	}
3769
3770	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3771	# if 1 /** @todo sync accessed bit properly... */
3772	fIgnoreFlags \|= X86_PTE_A;
3773	# endif
3774
3775	/* match the physical addresses */
3776	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3777	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3778
3779	# ifdef IN_RING3
3780	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3781	if (RT_FAILURE(rc))
3782	{
3783	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3784	{
3785	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3786	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3787	cErrors++;
3788	continue;
3789	}
3790	}
3791	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3792	{
3793	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3794	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3795	cErrors++;
3796	continue;
3797	}
3798	# endif
3799
3800	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3801	if (!pPhysPage)
3802	{
3803	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3804	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3805	{
3806	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3807	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3808	cErrors++;
3809	continue;
3810	}
3811	# endif
3812	if (PteDst.n.u1Write)
3813	{
3814	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3815	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3816	cErrors++;
3817	}
3818	fIgnoreFlags \|= X86_PTE_RW;
3819	}
3820	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3821	{
3822	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3823	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3824	cErrors++;
3825	continue;
3826	}
3827
3828	/* flags */
3829	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3830	{
3831	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3832	{
3833	if (PteDst.n.u1Write)
3834	{
3835	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3836	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3837	cErrors++;
3838	continue;
3839	}
3840	fIgnoreFlags \|= X86_PTE_RW;
3841	}
3842	else
3843	{
3844	if (PteDst.n.u1Present)
3845	{
3846	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3847	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3848	cErrors++;
3849	continue;
3850	}
3851	fIgnoreFlags \|= X86_PTE_P;
3852	}
3853	}
3854	else
3855	{
3856	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3857	{
3858	if (PteDst.n.u1Write)
3859	{
3860	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3861	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3862	cErrors++;
3863	continue;
3864	}
3865	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3866	{
3867	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3868	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3869	cErrors++;
3870	continue;
3871	}
3872	if (PteDst.n.u1Dirty)
3873	{
3874	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3875	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3876	cErrors++;
3877	}
3878	# if 0 /** @todo sync access bit properly... */
3879	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3880	{
3881	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3882	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3883	cErrors++;
3884	}
3885	fIgnoreFlags \|= X86_PTE_RW;
3886	# else
3887	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3888	# endif
3889	}
3890	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3891	{
3892	/* access bit emulation (not implemented). */
3893	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3894	{
3895	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3896	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3897	cErrors++;
3898	continue;
3899	}
3900	if (!PteDst.n.u1Accessed)
3901	{
3902	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3903	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3904	cErrors++;
3905	}
3906	fIgnoreFlags \|= X86_PTE_P;
3907	}
3908	# ifdef DEBUG_sandervl
3909	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3910	# endif
3911	}
3912
3913	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3914	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3915	)
3916	{
3917	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3918	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3919	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3920	cErrors++;
3921	continue;
3922	}
3923	} /* foreach PTE */
3924	}
3925	else
3926	{
3927	/*
3928	* Big Page.
3929	*/
3930	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3931	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3932	{
3933	if (PdeDst.n.u1Write)
3934	{
3935	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3936	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3937	cErrors++;
3938	continue;
3939	}
3940	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3941	{
3942	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3943	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3944	cErrors++;
3945	continue;
3946	}
3947	# if 0 /** @todo sync access bit properly... */
3948	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3949	{
3950	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3951	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3952	cErrors++;
3953	}
3954	fIgnoreFlags \|= X86_PTE_RW;
3955	# else
3956	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3957	# endif
3958	}
3959	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3960	{
3961	/* access bit emulation (not implemented). */
3962	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3963	{
3964	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3965	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3966	cErrors++;
3967	continue;
3968	}
3969	if (!PdeDst.n.u1Accessed)
3970	{
3971	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3972	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3973	cErrors++;
3974	}
3975	fIgnoreFlags \|= X86_PTE_P;
3976	}
3977
3978	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3979	{
3980	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3981	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3982	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3983	cErrors++;
3984	}
3985
3986	/* iterate the page table. */
3987	for (unsigned iPT = 0, off = 0;
3988	iPT < RT_ELEMENTS(pPTDst->a);
3989	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3990	{
3991	const SHWPTE PteDst = pPTDst->a[iPT];
3992
3993	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3994	{
3995	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3996	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3997	cErrors++;
3998	}
3999
4000	/* skip not-present entries. */
4001	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4002	continue;
4003
4004	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4005
4006	/* match the physical addresses */
4007	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4008
4009	# ifdef IN_RING3
4010	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4011	if (RT_FAILURE(rc))
4012	{
4013	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4014	{
4015	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4016	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4017	cErrors++;
4018	}
4019	}
4020	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4021	{
4022	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4023	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4024	cErrors++;
4025	continue;
4026	}
4027	# endif
4028	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4029	if (!pPhysPage)
4030	{
4031	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4032	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4033	{
4034	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4035	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4036	cErrors++;
4037	continue;
4038	}
4039	# endif
4040	if (PteDst.n.u1Write)
4041	{
4042	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4043	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4044	cErrors++;
4045	}
4046	fIgnoreFlags \|= X86_PTE_RW;
4047	}
4048	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4049	{
4050	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4051	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4052	cErrors++;
4053	continue;
4054	}
4055
4056	/* flags */
4057	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4058	{
4059	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4060	{
4061	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4062	{
4063	if (PteDst.n.u1Write)
4064	{
4065	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4066	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4067	cErrors++;
4068	continue;
4069	}
4070	fIgnoreFlags \|= X86_PTE_RW;
4071	}
4072	}
4073	else
4074	{
4075	if (PteDst.n.u1Present)
4076	{
4077	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4078	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4079	cErrors++;
4080	continue;
4081	}
4082	fIgnoreFlags \|= X86_PTE_P;
4083	}
4084	}
4085
4086	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4087	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4088	)
4089	{
4090	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4091	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4092	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4093	cErrors++;
4094	continue;
4095	}
4096	} /* for each PTE */
4097	}
4098	}
4099	/* not present */
4100
4101	} /* for each PDE */
4102
4103	} /* for each PDPTE */
4104
4105	} /* for each PML4E */
4106
4107	# ifdef DEBUG
4108	if (cErrors)
4109	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4110	# endif
4111
4112	#endif /* GST == 32BIT, PAE or AMD64 */
4113	return cErrors;
4114
4115	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4116	}
4117	#endif /* VBOX_STRICT */
4118
4119
4120	/**
4121	* Sets up the CR3 for shadow paging
4122	*
4123	* @returns Strict VBox status code.
4124	* @retval VINF_SUCCESS.
4125	*
4126	* @param pVCpu The VMCPU handle.
4127	* @param GCPhysCR3 The physical address in the CR3 register.
4128	*/
4129	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4130	{
4131	PVM pVM = pVCpu->CTX_SUFF(pVM);
4132
4133	/* Update guest paging info. */
4134	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4135	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4136	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4137
4138	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4139
4140	/*
4141	* Map the page CR3 points at.
4142	*/
4143	RTHCPTR HCPtrGuestCR3;
4144	RTHCPHYS HCPhysGuestCR3;
4145	pgmLock(pVM);
4146	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4147	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4148	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPage);
4149	/** @todo this needs some reworking wrt. locking. */
4150	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4151	HCPtrGuestCR3 = NIL_RTHCPTR;
4152	int rc = VINF_SUCCESS;
4153	# else
4154	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3);
4155	# endif
4156	pgmUnlock(pVM);
4157	if (RT_SUCCESS(rc))
4158	{
4159	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4160	if (RT_SUCCESS(rc))
4161	{
4162	# ifdef IN_RC
4163	PGM_INVL_PG(pVM->pgm.s.GCPtrCR3Mapping);
4164	# endif
4165	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4166	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4167	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4168	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4169	# endif
4170	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))pVM->pgm.s.GCPtrCR3Mapping;
4171
4172	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4173	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4174	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4175	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4176	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4177	# endif
4178	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RCPTRTYPE(uint8_t *))pVM->pgm.s.GCPtrCR3Mapping + off);
4179	Log(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4180
4181	/*
4182	* Map the 4 PDs too.
4183	*/
4184	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(&pVCpu->pgm.s);
4185	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4186	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4187	{
4188	if (pGuestPDPT->a[i].n.u1Present)
4189	{
4190	RTHCPTR HCPtr;
4191	RTHCPHYS HCPhys;
4192	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4193	pgmLock(pVM);
4194	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4195	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4196	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4197	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4198	HCPtr = NIL_RTHCPTR;
4199	int rc2 = VINF_SUCCESS;
4200	# else
4201	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4202	# endif
4203	pgmUnlock(pVM);
4204	if (RT_SUCCESS(rc2))
4205	{
4206	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4207	AssertRCReturn(rc, rc);
4208
4209	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4210	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4211	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4212	# endif
4213	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))GCPtr;
4214	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4215	PGM_INVL_PG(GCPtr); /** @todo This ends up calling HWACCMInvalidatePage, is that correct? */
4216	continue;
4217	}
4218	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4219	}
4220
4221	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4222	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4223	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4224	# endif
4225	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4226	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4227	PGM_INVL_PG(GCPtr); /** @todo this shouldn't be necessary? */
4228	}
4229
4230	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4231	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4232	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4233	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4234	# endif
4235	# endif
4236	}
4237	else
4238	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4239	}
4240	else
4241	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4242
4243	#else /* prot/real stub */
4244	int rc = VINF_SUCCESS;
4245	#endif
4246
4247	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4248	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4249	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4250	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4251	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4252	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4253
4254	Assert(!HWACCMIsNestedPagingActive(pVM));
4255
4256	/*
4257	* Update the shadow root page as well since that's not fixed.
4258	*/
4259	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4260	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4261	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4262	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4263	PPGMPOOLPAGE pNewShwPageCR3;
4264
4265	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4266	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3);
4267	AssertFatalRC(rc);
4268	rc = VINF_SUCCESS;
4269
4270	/* Mark the page as locked; disallow flushing. */
4271	pgmPoolLockPage(pPool, pNewShwPageCR3);
4272
4273	# ifdef IN_RC
4274	/* NOTE: We can't deal with jumps to ring 3 here as we're now in an inconsistent state! */
4275	bool fLog = VMMGCLogDisable(pVM);
4276	pgmLock(pVM);
4277	# endif
4278
4279	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4280	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4281	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4282	# ifdef IN_RING0
4283	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4284	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4285	# elif defined(IN_RC)
4286	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4287	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4288	# else
4289	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4290	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4291	# endif
4292
4293	# ifndef PGM_WITHOUT_MAPPINGS
4294	/*
4295	* Apply all hypervisor mappings to the new CR3.
4296	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4297	* make sure we check for conflicts in the new CR3 root.
4298	*/
4299	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4300	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
4301	# endif
4302	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4303	AssertRCReturn(rc, rc);
4304	# endif
4305
4306	/* Set the current hypervisor CR3. */
4307	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4308	SELMShadowCR3Changed(pVM, pVCpu);
4309
4310	# ifdef IN_RC
4311	pgmUnlock(pVM);
4312	VMMGCLogRestore(pVM, fLog);
4313	# endif
4314
4315	/* Clean up the old CR3 root. */
4316	if (pOldShwPageCR3)
4317	{
4318	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4319	# ifndef PGM_WITHOUT_MAPPINGS
4320	/* Remove the hypervisor mappings from the shadow page table. */
4321	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4322	# endif
4323	/* Mark the page as unlocked; allow flushing again. */
4324	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4325
4326	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4327	}
4328
4329	# endif
4330
4331	return rc;
4332	}
4333
4334	/**
4335	* Unmaps the shadow CR3.
4336	*
4337	* @returns VBox status, no specials.
4338	* @param pVCpu The VMCPU handle.
4339	*/
4340	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4341	{
4342	LogFlow(("UnmapCR3\n"));
4343
4344	int rc = VINF_SUCCESS;
4345	PVM pVM = pVCpu->CTX_SUFF(pVM);
4346
4347	/*
4348	* Update guest paging info.
4349	*/
4350	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4351	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4352	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4353	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4354	# endif
4355	pVCpu->pgm.s.pGst32BitPdRC = 0;
4356
4357	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4358	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4359	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4360	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4361	# endif
4362	pVCpu->pgm.s.pGstPaePdptRC = 0;
4363	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4364	{
4365	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4366	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4367	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4368	# endif
4369	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4370	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4371	}
4372
4373	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4374	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4375	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4376	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4377	# endif
4378
4379	#else /* prot/real mode stub */
4380	/* nothing to do */
4381	#endif
4382
4383	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4384	/*
4385	* Update shadow paging info.
4386	*/
4387	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4388	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4389	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4390
4391	# if PGM_GST_TYPE != PGM_TYPE_REAL
4392	Assert(!HWACCMIsNestedPagingActive(pVM));
4393	# endif
4394
4395	# ifndef PGM_WITHOUT_MAPPINGS
4396	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4397	/* Remove the hypervisor mappings from the shadow page table. */
4398	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4399	# endif
4400
4401	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4402	{
4403	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4404
4405	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4406
4407	/* Mark the page as unlocked; allow flushing again. */
4408	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4409
4410	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4411	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4412	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4413	pVCpu->pgm.s.pShwPageCR3RC = 0;
4414	pVCpu->pgm.s.iShwUser = 0;
4415	pVCpu->pgm.s.iShwUserTable = 0;
4416	}
4417	# endif
4418	#endif /* !IN_RC*/
4419
4420	return rc;
4421	}
4422

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 19087

Download in other formats: