PGMAllBth.h@ 8536

Last change on this file since 8536 was 8536, checked in by vboxsync, 17 years ago
Split out the long mode only bits from the PDPE. Set accessed bits for the PDPE & PML4E
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 159.4 KB

Line
1	/* $Id: PGMAllBth.h 8536 2008-05-02 16:46:51Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	__END_DECLS
43
44
45	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
46	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE
47	# error "Invalid combination; PAE guest implies PAE shadow"
48	#endif
49
50	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
51	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE)
52	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
53	#endif
54
55	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
56	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE)
57	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
58	#endif
59
60	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64)
61	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64)
62	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
63	#endif
64
65	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
66	# define PGM_WITHOUT_MAPPINGS
67	#endif
68
69	/**
70	* #PF Handler for raw-mode guest execution.
71	*
72	* @returns VBox status code (appropriate for trap handling and GC return).
73	* @param pVM VM Handle.
74	* @param uErr The trap error code.
75	* @param pRegFrame Trap register frame.
76	* @param pvFault The fault address.
77	*/
78	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
79	{
80	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
81
82	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
83	/*
84	* Hide the instruction fetch trap indicator for now.
85	*/
86	/** @todo NXE will change this and we must fix NXE in the switcher too! */
87	if (uErr & X86_TRAP_PF_ID)
88	{
89	uErr &= ~X86_TRAP_PF_ID;
90	TRPMSetErrorCode(pVM, uErr);
91	}
92	# endif
93
94	/*
95	* Get PDs.
96	*/
97	int rc;
98	# if PGM_WITH_PAGING(PGM_GST_TYPE)
99	# if PGM_GST_TYPE == PGM_TYPE_32BIT
100	const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
101	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
102
103	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
104
105	# if PGM_GST_TYPE == PGM_TYPE_PAE
106	unsigned iPDSrc;
107	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc);
108
109	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
110	unsigned iPDSrc;
111	PX86PML4E pPml4e;
112	X86PDPE Pdpe;
113	PGSTPD pPDSrc;
114
115	pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, pvFault, &pPml4e, &Pdpe, &iPDSrc);
116	Assert(pPml4e);
117	# endif
118	/* Quick check for a valid guest trap. */
119	if (!pPDSrc)
120	{
121	LogFlow(("Trap0eHandler: guest PDPTR not present CR3=%VGp\n", (CPUMGetGuestCR3(pVM) & X86_CR3_PAGE_MASK)));
122	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eGuestTrap; });
123	TRPMSetErrorCode(pVM, uErr);
124	return VINF_EM_RAW_GUEST_TRAP;
125	}
126	# endif
127	# else
128	PGSTPD pPDSrc = NULL;
129	const unsigned iPDSrc = 0;
130	# endif
131
132	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
133	const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
134	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
135	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
136	const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
137	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
138
139	# if PGM_GST_TYPE == PGM_TYPE_PAE
140	/* Did we mark the PDPT as not present in SyncCR3? */
141	unsigned iPDPTE = ((RTGCUINTPTR)pvFault >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
142	if (!pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
143	pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 1;
144
145	# endif
146
147	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
148	const unsigned iPDDst = (((RTGCUINTPTR)pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
149	PX86PDPAE pPDDst;
150
151	rc = PGMShwGetLongModePDPtr(pVM, (RTGCUINTPTR)pvFault, &pPDDst);
152	if (rc != VINF_SUCCESS)
153	{
154	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
155	return rc;
156	}
157	Assert(pPDDst);
158	# endif
159
160	# if PGM_WITH_PAGING(PGM_GST_TYPE)
161	# ifdef PGM_SYNC_DIRTY_BIT
162	/*
163	* If we successfully correct the write protection fault due to dirty bit
164	* tracking, or this page fault is a genuine one, then return immediately.
165	*/
166	STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
167	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
168	STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
169	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
170	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
171	{
172	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
173	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
174	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
175	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
176	}
177	# endif
178
179	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
180	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
181
182	/*
183	* A common case is the not-present error caused by lazy page table syncing.
184	*
185	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
186	* so we can safely assume that the shadow PT is present when calling SyncPage later.
187	*
188	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
189	* of mapping conflict and defer to SyncCR3 in R3.
190	* (Again, we do NOT support access handlers for non-present guest pages.)
191	*
192	*/
193	# if PGM_WITH_PAGING(PGM_GST_TYPE)
194	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
195	# else
196	GSTPDE PdeSrc;
197	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
198	PdeSrc.n.u1Present = 1;
199	PdeSrc.n.u1Write = 1;
200	PdeSrc.n.u1Accessed = 1;
201	PdeSrc.n.u1User = 1;
202	# endif
203	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
204	&& !pPDDst->a[iPDDst].n.u1Present
205	&& PdeSrc.n.u1Present
206	)
207
208	{
209	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
210	STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
211	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
212	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
213	if (VBOX_SUCCESS(rc))
214	{
215	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
216	return rc;
217	}
218	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
219	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
220	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
221	return VINF_PGM_SYNC_CR3;
222	}
223
224	# if PGM_WITH_PAGING(PGM_GST_TYPE)
225	/*
226	* Check if this address is within any of our mappings.
227	*
228	* This is very fast and it's gonna save us a bit of effort below and prevent
229	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
230	* (BTW, it's impossible to have physical access handlers in a mapping.)
231	*/
232	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
233	{
234	STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
235	PPGMMAPPING pMapping = CTXALLSUFF(pVM->pgm.s.pMappings);
236	for ( ; pMapping; pMapping = CTXALLSUFF(pMapping->pNext))
237	{
238	if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
239	break;
240	if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
241	{
242	/*
243	* The first thing we check is if we've got an undetected conflict.
244	*/
245	if (!pVM->pgm.s.fMappingsFixed)
246	{
247	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
248	while (iPT-- > 0)
249	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
250	{
251	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
252	Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
253	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
254	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
255	return VINF_PGM_SYNC_CR3;
256	}
257	}
258
259	/*
260	* Check if the fault address is in a virtual page access handler range.
261	*/
262	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->HyperVirtHandlers, pvFault);
263	if ( pCur
264	&& (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
265	&& uErr & X86_TRAP_PF_RW)
266	{
267	# ifdef IN_GC
268	STAM_PROFILE_START(&pCur->Stat, h);
269	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
270	STAM_PROFILE_STOP(&pCur->Stat, h);
271	# else
272	AssertFailed();
273	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
274	# endif
275	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
276	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
277	return rc;
278	}
279
280	/*
281	* Pretend we're not here and let the guest handle the trap.
282	*/
283	TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
284	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
285	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
286	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
287	return VINF_EM_RAW_GUEST_TRAP;
288	}
289	}
290	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
291	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
292	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
293
294	/*
295	* Check if this fault address is flagged for special treatment,
296	* which means we'll have to figure out the physical address and
297	* check flags associated with it.
298	*
299	* ASSUME that we can limit any special access handling to pages
300	* in page tables which the guest believes to be present.
301	*/
302	if (PdeSrc.n.u1Present)
303	{
304	RTGCPHYS GCPhys = NIL_RTGCPHYS;
305
306	# if PGM_WITH_PAGING(PGM_GST_TYPE)
307	uint32_t cr4 = CPUMGetGuestCR4(pVM);
308	if ( PdeSrc.b.u1Size
309	&& (cr4 & X86_CR4_PSE))
310	GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK)
311	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
312	else
313	{
314	PGSTPT pPTSrc;
315	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
316	if (VBOX_SUCCESS(rc))
317	{
318	unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
319	if (pPTSrc->a[iPTESrc].n.u1Present)
320	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
321	}
322	}
323	# else
324	/* No paging so the fault address is the physical address */
325	GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
326	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
327
328	/*
329	* If we have a GC address we'll check if it has any flags set.
330	*/
331	if (GCPhys != NIL_RTGCPHYS)
332	{
333	STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
334
335	PPGMPAGE pPage;
336	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
337	if (VBOX_SUCCESS(rc))
338	{
339	if (PGM_PAGE_HAS_ANY_HANDLERS(pPage))
340	{
341	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
342	{
343	/*
344	* Physical page access handler.
345	*/
346	const RTGCPHYS GCPhysFault = GCPhys \| ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
347	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
348	if (pCur)
349	{
350	# ifdef PGM_SYNC_N_PAGES
351	/*
352	* If the region is write protected and we got a page not present fault, then sync
353	* the pages. If the fault was caused by a read, then restart the instruction.
354	* In case of write access continue to the GC write handler.
355	*
356	* ASSUMES that there is only one handler per page or that they have similar write properties.
357	*/
358	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
359	&& !(uErr & X86_TRAP_PF_P))
360	{
361	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
362	if ( VBOX_FAILURE(rc)
363	\|\| !(uErr & X86_TRAP_PF_RW)
364	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
365	{
366	AssertRC(rc);
367	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
368	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
369	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
370	return rc;
371	}
372	}
373	# endif
374
375	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
376	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
377	("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
378
379	#if defined(IN_GC) \|\| defined(IN_RING0)
380	if (CTXALLSUFF(pCur->pfnHandler))
381	{
382	STAM_PROFILE_START(&pCur->Stat, h);
383	rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
384	STAM_PROFILE_STOP(&pCur->Stat, h);
385	}
386	else
387	#endif
388	rc = VINF_EM_RAW_EMULATE_INSTR;
389	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
390	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
391	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
392	return rc;
393	}
394	}
395	# if PGM_WITH_PAGING(PGM_GST_TYPE)
396	else
397	{
398	# ifdef PGM_SYNC_N_PAGES
399	/*
400	* If the region is write protected and we got a page not present fault, then sync
401	* the pages. If the fault was caused by a read, then restart the instruction.
402	* In case of write access continue to the GC write handler.
403	*/
404	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
405	&& !(uErr & X86_TRAP_PF_P))
406	{
407	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
408	if ( VBOX_FAILURE(rc)
409	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
410	\|\| !(uErr & X86_TRAP_PF_RW))
411	{
412	AssertRC(rc);
413	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
414	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
415	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
416	return rc;
417	}
418	}
419	# endif
420	/*
421	* Ok, it's an virtual page access handler.
422	*
423	* Since it's faster to search by address, we'll do that first
424	* and then retry by GCPhys if that fails.
425	*/
426	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
427	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
428	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
429	*/
430	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
431	if (pCur)
432	{
433	AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
434	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
435	\|\| !(uErr & X86_TRAP_PF_P)
436	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
437	("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
438
439	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
440	&& ( uErr & X86_TRAP_PF_RW
441	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
442	{
443	# ifdef IN_GC
444	STAM_PROFILE_START(&pCur->Stat, h);
445	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
446	STAM_PROFILE_STOP(&pCur->Stat, h);
447	# else
448	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
449	# endif
450	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
451	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
452	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
453	return rc;
454	}
455	/* Unhandled part of a monitored page */
456	}
457	else
458	{
459	/* Check by physical address. */
460	PPGMVIRTHANDLER pCur;
461	unsigned iPage;
462	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
463	&pCur, &iPage);
464	Assert(VBOX_SUCCESS(rc) \|\| !pCur);
465	if ( pCur
466	&& ( uErr & X86_TRAP_PF_RW
467	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
468	{
469	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
470	# ifdef IN_GC
471	RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->GCPtr & PAGE_OFFSET_MASK);
472	Assert(off < pCur->cb);
473	STAM_PROFILE_START(&pCur->Stat, h);
474	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
475	STAM_PROFILE_STOP(&pCur->Stat, h);
476	# else
477	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
478	# endif
479	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
480	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
481	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
482	return rc;
483	}
484	}
485	}
486	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
487
488	/*
489	* There is a handled area of the page, but this fault doesn't belong to it.
490	* We must emulate the instruction.
491	*
492	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
493	* we first check if this was a page-not-present fault for a page with only
494	* write access handlers. Restart the instruction if it wasn't a write access.
495	*/
496	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
497
498	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
499	&& !(uErr & X86_TRAP_PF_P))
500	{
501	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
502	if ( VBOX_FAILURE(rc)
503	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
504	\|\| !(uErr & X86_TRAP_PF_RW))
505	{
506	AssertRC(rc);
507	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
508	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
509	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
510	return rc;
511	}
512	}
513
514	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
515	* It's writing to an unhandled part of the LDT page several million times.
516	*/
517	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
518	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%RHp%s%s\n",
519	rc, pPage->HCPhys,
520	PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ? " phys" : "",
521	PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ? " virt" : ""));
522	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
523	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
524	return rc;
525	} /* if any kind of handler */
526
527	# if PGM_WITH_PAGING(PGM_GST_TYPE)
528	if (uErr & X86_TRAP_PF_P)
529	{
530	/*
531	* The page isn't marked, but it might still be monitored by a virtual page access handler.
532	* (ASSUMES no temporary disabling of virtual handlers.)
533	*/
534	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
535	* we should correct both the shadow page table and physical memory flags, and not only check for
536	* accesses within the handler region but for access to pages with virtual handlers. */
537	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
538	if (pCur)
539	{
540	AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
541	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
542	\|\| !(uErr & X86_TRAP_PF_P)
543	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
544	("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
545
546	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
547	&& ( uErr & X86_TRAP_PF_RW
548	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
549	{
550	# ifdef IN_GC
551	STAM_PROFILE_START(&pCur->Stat, h);
552	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
553	STAM_PROFILE_STOP(&pCur->Stat, h);
554	# else
555	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
556	# endif
557	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
558	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
559	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
560	return rc;
561	}
562	}
563	}
564	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
565	}
566	else
567	{
568	/* When the guest accesses invalid physical memory (e.g. probing of RAM or accessing a remapped MMIO range), then we'll fall
569	* back to the recompiler to emulate the instruction.
570	*/
571	LogFlow(("pgmPhysGetPageEx %VGp failed with %Vrc\n", GCPhys, rc));
572	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersInvalid);
573	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
574	return VINF_EM_RAW_EMULATE_INSTR;
575	}
576
577	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
578
579	# ifdef PGM_OUT_OF_SYNC_IN_GC
580	/*
581	* We are here only if page is present in Guest page tables and trap is not handled
582	* by our handlers.
583	* Check it for page out-of-sync situation.
584	*/
585	STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
586
587	if (!(uErr & X86_TRAP_PF_P))
588	{
589	/*
590	* Page is not present in our page tables.
591	* Try to sync it!
592	* BTW, fPageShw is invalid in this branch!
593	*/
594	if (uErr & X86_TRAP_PF_US)
595	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
596	else /* supervisor */
597	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
598
599	# if defined(LOG_ENABLED) && !defined(IN_RING0)
600	RTGCPHYS GCPhys;
601	uint64_t fPageGst;
602	PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
603	Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
604	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
605	# endif /* LOG_ENABLED */
606
607	# if PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0)
608	if (CPUMGetGuestCPL(pVM, pRegFrame) == 0)
609	{
610	uint64_t fPageGst;
611	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
612	if ( VBOX_SUCCESS(rc)
613	&& !(fPageGst & X86_PTE_US))
614	{
615	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
616	if ( pvFault == (RTGCPTR)pRegFrame->eip
617	\|\| (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
618	# ifdef CSAM_DETECT_NEW_CODE_PAGES
619	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
620	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
621	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
622	)
623	{
624	LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
625	rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
626	if (rc != VINF_SUCCESS)
627	{
628	/*
629	* CSAM needs to perform a job in ring 3.
630	*
631	* Sync the page before going to the host context; otherwise we'll end up in a loop if
632	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
633	*/
634	LogFlow(("CSAM ring 3 job\n"));
635	int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
636	AssertRC(rc2);
637
638	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
639	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
640	return rc;
641	}
642	}
643	# ifdef CSAM_DETECT_NEW_CODE_PAGES
644	else
645	if ( uErr == X86_TRAP_PF_RW
646	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
647	&& pRegFrame->ecx < 0x10000
648	)
649	{
650	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
651	* to detect loading of new code pages.
652	*/
653
654	/*
655	* Decode the instruction.
656	*/
657	RTGCPTR PC;
658	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
659	if (rc == VINF_SUCCESS)
660	{
661	DISCPUSTATE Cpu;
662	uint32_t cbOp;
663	rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
664
665	/* For now we'll restrict this to rep movsw/d instructions */
666	if ( rc == VINF_SUCCESS
667	&& Cpu.pCurInstr->opcode == OP_MOVSWD
668	&& (Cpu.prefix & PREFIX_REP))
669	{
670	CSAMMarkPossibleCodePage(pVM, pvFault);
671	}
672	}
673	}
674	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
675
676	/*
677	* Mark this page as safe.
678	*/
679	/** @todo not correct for pages that contain both code and data!! */
680	Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
681	CSAMMarkPage(pVM, pvFault, true);
682	}
683	}
684	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0) */
685	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
686	if (VBOX_SUCCESS(rc))
687	{
688	/* The page was successfully synced, return to the guest. */
689	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
690	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
691	return VINF_SUCCESS;
692	}
693	}
694	else
695	{
696	/*
697	* A side effect of not flushing global PDEs are out of sync pages due
698	* to physical monitored regions, that are no longer valid.
699	* Assume for now it only applies to the read/write flag
700	*/
701	if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
702	{
703	if (uErr & X86_TRAP_PF_US)
704	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
705	else /* supervisor */
706	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
707
708
709	/*
710	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
711	*/
712	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
713	if (VBOX_SUCCESS(rc))
714	{
715	/*
716	* Page was successfully synced, return to guest.
717	*/
718	# ifdef VBOX_STRICT
719	RTGCPHYS GCPhys;
720	uint64_t fPageGst;
721	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
722	Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
723	LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
724
725	uint64_t fPageShw;
726	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
727	Assert(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW);
728	# endif /* VBOX_STRICT */
729	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
730	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
731	return VINF_SUCCESS;
732	}
733
734	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
735	if ( CPUMGetGuestCPL(pVM, pRegFrame) == 0
736	&& ((CPUMGetGuestCR0(pVM) & (X86_CR0_WP\|X86_CR0_PG)) == X86_CR0_PG)
737	&& (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P))
738	{
739	uint64_t fPageGst;
740	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
741	if ( VBOX_SUCCESS(rc)
742	&& !(fPageGst & X86_PTE_RW))
743	{
744	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
745	if (VBOX_SUCCESS(rc))
746	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulGC);
747	else
748	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulR3);
749	return rc;
750	}
751	else
752	AssertMsgFailed(("Unexpected r/w page %x flag=%x\n", pvFault, (uint32_t)fPageGst));
753	}
754
755	}
756
757	# if PGM_WITH_PAGING(PGM_GST_TYPE)
758	# ifdef VBOX_STRICT
759	/*
760	* Check for VMM page flags vs. Guest page flags consistency.
761	* Currently only for debug purposes.
762	*/
763	if (VBOX_SUCCESS(rc))
764	{
765	/* Get guest page flags. */
766	uint64_t fPageGst;
767	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
768	if (VBOX_SUCCESS(rc))
769	{
770	uint64_t fPageShw;
771	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
772
773	/*
774	* Compare page flags.
775	* Note: we have AVL, A, D bits desynched.
776	*/
777	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
778	("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
779	}
780	else
781	AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
782	}
783	else
784	AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
785	# endif /* VBOX_STRICT */
786	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
787	}
788	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
789	# endif /* PGM_OUT_OF_SYNC_IN_GC */
790	}
791	else
792	{
793	/*
794	* Page not present in Guest OS or invalid page table address.
795	* This is potential virtual page access handler food.
796	*
797	* For the present we'll say that our access handlers don't
798	* work for this case - we've already discarded the page table
799	* not present case which is identical to this.
800	*
801	* When we perchance find we need this, we will probably have AVL
802	* trees (offset based) to operate on and we can measure their speed
803	* agains mapping a page table and probably rearrange this handling
804	* a bit. (Like, searching virtual ranges before checking the
805	* physical address.)
806	*/
807	}
808	}
809
810
811	# if PGM_WITH_PAGING(PGM_GST_TYPE)
812	/*
813	* Conclusion, this is a guest trap.
814	*/
815	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
816	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
817	return VINF_EM_RAW_GUEST_TRAP;
818	# else
819	/* present, but not a monitored page; perhaps the guest is probing physical memory */
820	return VINF_EM_RAW_EMULATE_INSTR;
821	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
822
823
824	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
825
826	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
827	return VERR_INTERNAL_ERROR;
828	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
829	}
830
831
832	/**
833	* Emulation of the invlpg instruction.
834	*
835	*
836	* @returns VBox status code.
837	*
838	* @param pVM VM handle.
839	* @param GCPtrPage Page to invalidate.
840	*
841	* @remark ASSUMES that the guest is updating before invalidating. This order
842	* isn't required by the CPU, so this is speculative and could cause
843	* trouble.
844	*
845	* @todo Flush page or page directory only if necessary!
846	* @todo Add a #define for simply invalidating the page.
847	*/
848	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
849	{
850	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
851	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
852
853	LogFlow(("InvalidatePage %x\n", GCPtrPage));
854	/*
855	* Get the shadow PD entry and skip out if this PD isn't present.
856	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
857	*/
858	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
859	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
860	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
861	# else
862	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
863	# endif
864	const SHWPDE PdeDst = *pPdeDst;
865	if (!PdeDst.n.u1Present)
866	{
867	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
868	return VINF_SUCCESS;
869	}
870
871	/*
872	* Get the guest PD entry and calc big page.
873	*/
874	# if PGM_GST_TYPE == PGM_TYPE_32BIT
875	PX86PD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
876	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
877	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
878	# else /* PAE */
879	unsigned iPDSrc;
880	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
881	GSTPDE PdeSrc;
882
883	if (pPDSrc)
884	PdeSrc = pPDSrc->a[iPDSrc];
885	else
886	PdeSrc.u = 0;
887	# endif
888
889	const uint32_t cr4 = CPUMGetGuestCR4(pVM);
890	const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
891
892	# ifdef IN_RING3
893	/*
894	* If a CR3 Sync is pending we may ignore the invalidate page operation
895	* depending on the kind of sync and if it's a global page or not.
896	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
897	*/
898	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
899	if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
900	\|\| ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
901	&& fIsBigPage
902	&& PdeSrc.b.u1Global
903	&& (cr4 & X86_CR4_PGE)
904	)
905	)
906	# else
907	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
908	# endif
909	{
910	STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
911	return VINF_SUCCESS;
912	}
913	# endif /* IN_RING3 */
914
915
916	/*
917	* Deal with the Guest PDE.
918	*/
919	int rc = VINF_SUCCESS;
920	if (PdeSrc.n.u1Present)
921	{
922	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
923	{
924	/*
925	* Conflict - Let SyncPT deal with it to avoid duplicate code.
926	*/
927	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
928	Assert(PGMGetGuestMode(pVM) <= PGMMODE_32_BIT);
929	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
930	}
931	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
932	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
933	{
934	/*
935	* Mark not present so we can resync the PDE when it's used.
936	*/
937	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
938	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
939	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
940	pPdeDst->u = 0;
941	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
942	PGM_INVL_GUEST_TLBS();
943	}
944	# ifdef PGM_SYNC_ACCESSED_BIT
945	else if (!PdeSrc.n.u1Accessed)
946	{
947	/*
948	* Mark not present so we can set the accessed bit.
949	*/
950	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
951	pPdeDst->u = 0;
952	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
953	PGM_INVL_GUEST_TLBS();
954	}
955	# endif
956	else if (!fIsBigPage)
957	{
958	/*
959	* 4KB - page.
960	*/
961	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
962	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
963	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
964	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
965	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
966	# endif
967	if (pShwPage->GCPhys == GCPhys)
968	{
969	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
970	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
971	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
972	if (pPT->a[iPTEDst].n.u1Present)
973	{
974	# ifdef PGMPOOL_WITH_USER_TRACKING
975	/* This is very unlikely with caching/monitoring enabled. */
976	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
977	# endif
978	pPT->a[iPTEDst].u = 0;
979	}
980	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
981	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
982	if (VBOX_SUCCESS(rc))
983	rc = VINF_SUCCESS;
984	# endif
985	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
986	PGM_INVL_PG(GCPtrPage);
987	}
988	else
989	{
990	/*
991	* The page table address changed.
992	*/
993	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
994	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
995	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
996	pPdeDst->u = 0;
997	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
998	PGM_INVL_GUEST_TLBS();
999	}
1000	}
1001	else
1002	{
1003	/*
1004	* 2/4MB - page.
1005	*/
1006	/* Before freeing the page, check if anything really changed. */
1007	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1008	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1009	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1010	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1011	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1012	# endif
1013	if ( pShwPage->GCPhys == GCPhys
1014	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1015	{
1016	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1017	/** @todo PAT */
1018	# ifdef PGM_SYNC_DIRTY_BIT
1019	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1020	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1021	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1022	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1023	# else
1024	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1025	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD)))
1026	# endif
1027	{
1028	LogFlow(("Skipping flush for big page containing %VGv (PD=%X .u=%VX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1029	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
1030	return VINF_SUCCESS;
1031	}
1032	}
1033
1034	/*
1035	* Ok, the page table is present and it's been changed in the guest.
1036	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1037	* We could do this for some flushes in GC too, but we need an algorithm for
1038	* deciding which 4MB pages containing code likely to be executed very soon.
1039	*/
1040	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1041	pPdeDst->u = 0;
1042	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
1043	PGM_INVL_BIG_PG(GCPtrPage);
1044	}
1045	}
1046	else
1047	{
1048	/*
1049	* Page directory is not present, mark shadow PDE not present.
1050	*/
1051	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1052	{
1053	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1054	pPdeDst->u = 0;
1055	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1056	PGM_INVL_PG(GCPtrPage);
1057	}
1058	else
1059	{
1060	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1061	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
1062	}
1063	}
1064
1065	return rc;
1066
1067	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
1068	//# error not implemented
1069	return VERR_INTERNAL_ERROR;
1070
1071	#else /* guest real and protected mode */
1072	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1073	return VINF_SUCCESS;
1074	#endif
1075	}
1076
1077
1078	#ifdef PGMPOOL_WITH_USER_TRACKING
1079	/**
1080	* Update the tracking of shadowed pages.
1081	*
1082	* @param pVM The VM handle.
1083	* @param pShwPage The shadow page.
1084	* @param HCPhys The physical page we is being dereferenced.
1085	*/
1086	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1087	{
1088	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1089	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1090	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1091
1092	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1093	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1094	* 2. write protect all shadowed pages. I.e. implement caching.
1095	*/
1096	/*
1097	* Find the guest address.
1098	*/
1099	for (PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
1100	pRam;
1101	pRam = CTXALLSUFF(pRam->pNext))
1102	{
1103	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1104	while (iPage-- > 0)
1105	{
1106	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1107	{
1108	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1109	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1110	pShwPage->cPresent--;
1111	pPool->cPresent--;
1112	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1113	return;
1114	}
1115	}
1116	}
1117
1118	for (;;)
1119	AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1120	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1121	pShwPage->cPresent--;
1122	pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1123	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1124	}
1125
1126
1127	/**
1128	* Update the tracking of shadowed pages.
1129	*
1130	* @param pVM The VM handle.
1131	* @param pShwPage The shadow page.
1132	* @param u16 The top 16-bit of the pPage->HCPhys.
1133	* @param pPage Pointer to the guest page. this will be modified.
1134	* @param iPTDst The index into the shadow table.
1135	*/
1136	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1137	{
1138	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1139	/*
1140	* We're making certain assumptions about the placement of cRef and idx.
1141	*/
1142	Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1143	Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1144
1145	/*
1146	* Just deal with the simple first time here.
1147	*/
1148	if (!u16)
1149	{
1150	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1151	u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) \| pShwPage->idx;
1152	}
1153	else
1154	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1155
1156	/* write back, trying to be clever... */
1157	Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%VHp->%VHp iPTDst=%#x\n",
1158	u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) \| ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1159	((uint16_t )&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1160	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1161
1162	/* update statistics. */
1163	pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1164	pShwPage->cPresent++;
1165	if (pShwPage->iFirstPresent > iPTDst)
1166	pShwPage->iFirstPresent = iPTDst;
1167	}
1168	#endif /* PGMPOOL_WITH_USER_TRACKING */
1169
1170
1171	/**
1172	* Creates a 4K shadow page for a guest page.
1173	*
1174	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1175	* physical address. The PdeSrc argument only the flags are used. No page structured
1176	* will be mapped in this function.
1177	*
1178	* @param pVM VM handle.
1179	* @param pPteDst Destination page table entry.
1180	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1181	* Can safely assume that only the flags are being used.
1182	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1183	* @param pShwPage Pointer to the shadow page.
1184	* @param iPTDst The index into the shadow table.
1185	*
1186	* @remark Not used for 2/4MB pages!
1187	*/
1188	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1189	{
1190	if (PteSrc.n.u1Present)
1191	{
1192	/*
1193	* Find the ram range.
1194	*/
1195	PPGMPAGE pPage;
1196	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1197	if (VBOX_SUCCESS(rc))
1198	{
1199	/** @todo investiage PWT, PCD and PAT. */
1200	/*
1201	* Make page table entry.
1202	*/
1203	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1204	SHWPTE PteDst;
1205	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1206	{
1207	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1208	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1209	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1210	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1211	else
1212	{
1213	LogFlow(("SyncPageWorker: monitored page (%VGp) -> mark not present\n", HCPhys));
1214	PteDst.u = 0;
1215	}
1216	/** @todo count these two kinds. */
1217	}
1218	else
1219	{
1220	#ifdef PGM_SYNC_DIRTY_BIT
1221	# ifdef PGM_SYNC_ACCESSED_BIT
1222	/*
1223	* If the page or page directory entry is not marked accessed,
1224	* we mark the page not present.
1225	*/
1226	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1227	{
1228	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1229	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1230	PteDst.u = 0;
1231	}
1232	else
1233	# endif
1234	/*
1235	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1236	* when the page is modified.
1237	*/
1238	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1239	{
1240	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1241	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1242	\| (HCPhys & X86_PTE_PAE_PG_MASK)
1243	\| PGM_PTFLAGS_TRACK_DIRTY;
1244	}
1245	else
1246	{
1247	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1248	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1249	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1250	}
1251	#endif
1252	}
1253
1254	#ifdef PGMPOOL_WITH_USER_TRACKING
1255	/*
1256	* Keep user track up to date.
1257	*/
1258	if (PteDst.n.u1Present)
1259	{
1260	if (!pPteDst->n.u1Present)
1261	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1262	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1263	{
1264	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1265	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1266	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1267	}
1268	}
1269	else if (pPteDst->n.u1Present)
1270	{
1271	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1272	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1273	}
1274	#endif /* PGMPOOL_WITH_USER_TRACKING */
1275
1276	/*
1277	* Update statistics and commit the entry.
1278	*/
1279	if (!PteSrc.n.u1Global)
1280	pShwPage->fSeenNonGlobal = true;
1281	*pPteDst = PteDst;
1282	}
1283	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1284	/** @todo count these. */
1285	}
1286	else
1287	{
1288	/*
1289	* Page not-present.
1290	*/
1291	LogFlow(("SyncPageWorker: page not present in Pte\n"));
1292	#ifdef PGMPOOL_WITH_USER_TRACKING
1293	/* Keep user track up to date. */
1294	if (pPteDst->n.u1Present)
1295	{
1296	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1297	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1298	}
1299	#endif /* PGMPOOL_WITH_USER_TRACKING */
1300	pPteDst->u = 0;
1301	/** @todo count these. */
1302	}
1303	}
1304
1305
1306	/**
1307	* Syncs a guest OS page.
1308	*
1309	* There are no conflicts at this point, neither is there any need for
1310	* page table allocations.
1311	*
1312	* @returns VBox status code.
1313	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1314	* @param pVM VM handle.
1315	* @param PdeSrc Page directory entry of the guest.
1316	* @param GCPtrPage Guest context page address.
1317	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1318	* @param uErr Fault error (X86_TRAP_PF_*).
1319	*/
1320	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1321	{
1322	LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1323
1324	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
1325	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
1326
1327	# if PGM_WITH_NX(PGM_GST_TYPE)
1328	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1329	# endif
1330
1331	/*
1332	* Assert preconditions.
1333	*/
1334	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1335	Assert(PdeSrc.n.u1Present);
1336	Assert(cPages);
1337
1338	/*
1339	* Get the shadow PDE, find the shadow page table in the pool.
1340	*/
1341	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1342	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1343	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1344	# else /* PAE */
1345	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1346	# endif
1347	Assert(PdeDst.n.u1Present);
1348	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1349
1350	/*
1351	* Check that the page is present and that the shadow PDE isn't out of sync.
1352	*/
1353	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1354	RTGCPHYS GCPhys;
1355	if (!fBigPage)
1356	{
1357	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1358	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1359	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1360	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1361	# endif
1362	}
1363	else
1364	{
1365	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1366	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1367	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1368	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1369	# endif
1370	}
1371	if ( pShwPage->GCPhys == GCPhys
1372	&& PdeSrc.n.u1Present
1373	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1374	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1375	# if PGM_WITH_NX(PGM_GST_TYPE)
1376	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1377	# endif
1378	)
1379	{
1380	# ifdef PGM_SYNC_ACCESSED_BIT
1381	/*
1382	* Check that the PDE is marked accessed already.
1383	* Since we set the accessed bit before getting here on a #PF, this
1384	* check is only meant for dealing with non-#PF'ing paths.
1385	*/
1386	if (PdeSrc.n.u1Accessed)
1387	# endif
1388	{
1389	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1390	if (!fBigPage)
1391	{
1392	/*
1393	* 4KB Page - Map the guest page table.
1394	*/
1395	PGSTPT pPTSrc;
1396	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1397	if (VBOX_SUCCESS(rc))
1398	{
1399	# ifdef PGM_SYNC_N_PAGES
1400	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1401	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1402	{
1403	/*
1404	* This code path is currently only taken when the caller is PGMTrap0eHandler
1405	* for non-present pages!
1406	*
1407	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1408	* deal with locality.
1409	*/
1410	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1411	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1412	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1413	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1414	# else
1415	const unsigned offPTSrc = 0;
1416	# endif
1417	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1418	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1419	iPTDst = 0;
1420	else
1421	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1422	for (; iPTDst < iPTDstEnd; iPTDst++)
1423	{
1424	if (!pPTDst->a[iPTDst].n.u1Present)
1425	{
1426	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1427	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1428	NOREF(GCPtrCurPage);
1429	#ifndef IN_RING0
1430	/*
1431	* Assuming kernel code will be marked as supervisor - and not as user level
1432	* and executed using a conforming code selector - And marked as readonly.
1433	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1434	*/
1435	PPGMPAGE pPage;
1436	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1437	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1438	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1439	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1440	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1441	)
1442	#endif /* else: CSAM not active */
1443	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1444	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1445	GCPtrCurPage, PteSrc.n.u1Present,
1446	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1447	PteSrc.n.u1User & PdeSrc.n.u1User,
1448	(uint64_t)PteSrc.u,
1449	(uint64_t)pPTDst->a[iPTDst].u,
1450	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1451	}
1452	}
1453	}
1454	else
1455	# endif /* PGM_SYNC_N_PAGES */
1456	{
1457	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1458	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1459	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1460	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1461	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1462	GCPtrPage, PteSrc.n.u1Present,
1463	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1464	PteSrc.n.u1User & PdeSrc.n.u1User,
1465	(uint64_t)PteSrc.u,
1466	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1467	}
1468	}
1469	else /* MMIO or invalid page: emulated in #PF handler. */
1470	{
1471	LogFlow(("PGM_GCPHYS_2_PTR %VGp failed with %Vrc\n", GCPhys, rc));
1472	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1473	}
1474	}
1475	else
1476	{
1477	/*
1478	* 4/2MB page - lazy syncing shadow 4K pages.
1479	* (There are many causes of getting here, it's no longer only CSAM.)
1480	*/
1481	/* Calculate the GC physical address of this 4KB shadow page. */
1482	RTGCPHYS GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK) \| ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1483	/* Find ram range. */
1484	PPGMPAGE pPage;
1485	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1486	if (VBOX_SUCCESS(rc))
1487	{
1488	/*
1489	* Make shadow PTE entry.
1490	*/
1491	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1492	SHWPTE PteDst;
1493	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1494	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1495	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1496	{
1497	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1498	PteDst.n.u1Write = 0;
1499	else
1500	PteDst.u = 0;
1501	}
1502	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1503	# ifdef PGMPOOL_WITH_USER_TRACKING
1504	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1505	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1506	# endif
1507	pPTDst->a[iPTDst] = PteDst;
1508
1509
1510	# ifdef PGM_SYNC_DIRTY_BIT
1511	/*
1512	* If the page is not flagged as dirty and is writable, then make it read-only
1513	* at PD level, so we can set the dirty bit when the page is modified.
1514	*
1515	* ASSUMES that page access handlers are implemented on page table entry level.
1516	* Thus we will first catch the dirty access and set PDE.D and restart. If
1517	* there is an access handler, we'll trap again and let it work on the problem.
1518	*/
1519	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1520	* As for invlpg, it simply frees the whole shadow PT.
1521	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1522	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1523	{
1524	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1525	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1526	PdeDst.n.u1Write = 0;
1527	}
1528	else
1529	{
1530	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1531	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1532	}
1533	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1534	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1535	# else /* PAE */
1536	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1537	# endif
1538	# endif /* PGM_SYNC_DIRTY_BIT */
1539	Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1540	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1541	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1542	}
1543	else
1544	LogFlow(("PGM_GCPHYS_2_PTR %VGp (big) failed with %Vrc\n", GCPhys, rc));
1545	}
1546	return VINF_SUCCESS;
1547	}
1548	# ifdef PGM_SYNC_ACCESSED_BIT
1549	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1550	#endif
1551	}
1552	else
1553	{
1554	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1555	Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1556	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1557	}
1558
1559	/*
1560	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1561	* Yea, I'm lazy.
1562	*/
1563	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1564	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1565	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1566	# else /* PAE */
1567	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1568	# endif
1569	PGM_INVL_GUEST_TLBS();
1570	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1571
1572	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
1573
1574	# ifdef PGM_SYNC_N_PAGES
1575	/*
1576	* Get the shadow PDE, find the shadow page table in the pool.
1577	*/
1578	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1579	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1580	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1581	# else /* PAE */
1582	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1583	# endif
1584	Assert(PdeDst.n.u1Present);
1585	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1586	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1587
1588	# if PGM_SHW_TYPE == PGM_TYPE_PAE
1589	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1590	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1591	# else
1592	const unsigned offPTSrc = 0;
1593	# endif
1594
1595	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1596	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1597	{
1598	/*
1599	* This code path is currently only taken when the caller is PGMTrap0eHandler
1600	* for non-present pages!
1601	*
1602	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1603	* deal with locality.
1604	*/
1605	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1606	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1607	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1608	iPTDst = 0;
1609	else
1610	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1611	for (; iPTDst < iPTDstEnd; iPTDst++)
1612	{
1613	if (!pPTDst->a[iPTDst].n.u1Present)
1614	{
1615	GSTPTE PteSrc;
1616
1617	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1618
1619	/* Fake the page table entry */
1620	PteSrc.u = GCPtrCurPage;
1621	PteSrc.n.u1Present = 1;
1622	PteSrc.n.u1Dirty = 1;
1623	PteSrc.n.u1Accessed = 1;
1624	PteSrc.n.u1Write = 1;
1625	PteSrc.n.u1User = 1;
1626
1627	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1628
1629	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1630	GCPtrCurPage, PteSrc.n.u1Present,
1631	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1632	PteSrc.n.u1User & PdeSrc.n.u1User,
1633	(uint64_t)PteSrc.u,
1634	(uint64_t)pPTDst->a[iPTDst].u,
1635	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1636	}
1637	}
1638	}
1639	else
1640	# endif /* PGM_SYNC_N_PAGES */
1641	{
1642	GSTPTE PteSrc;
1643	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1644	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1645
1646	/* Fake the page table entry */
1647	PteSrc.u = GCPtrCurPage;
1648	PteSrc.n.u1Present = 1;
1649	PteSrc.n.u1Dirty = 1;
1650	PteSrc.n.u1Accessed = 1;
1651	PteSrc.n.u1Write = 1;
1652	PteSrc.n.u1User = 1;
1653	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1654
1655	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1656	GCPtrPage, PteSrc.n.u1Present,
1657	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1658	PteSrc.n.u1User & PdeSrc.n.u1User,
1659	(uint64_t)PteSrc.u,
1660	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1661	}
1662	return VINF_SUCCESS;
1663
1664	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1665	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1666	return VERR_INTERNAL_ERROR;
1667	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1668	}
1669
1670
1671
1672	#if PGM_WITH_PAGING(PGM_GST_TYPE)
1673
1674	/**
1675	* Investigate page fault and handle write protection page faults caused by
1676	* dirty bit tracking.
1677	*
1678	* @returns VBox status code.
1679	* @param pVM VM handle.
1680	* @param uErr Page fault error code.
1681	* @param pPdeDst Shadow page directory entry.
1682	* @param pPdeSrc Guest page directory entry.
1683	* @param GCPtrPage Guest context page address.
1684	*/
1685	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1686	{
1687	bool fWriteProtect = !!(CPUMGetGuestCR0(pVM) & X86_CR0_WP);
1688	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
1689	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
1690	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1691	# if PGM_WITH_NX(PGM_GST_TYPE)
1692	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1693	# endif
1694	unsigned uPageFaultLevel;
1695	int rc;
1696
1697	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1698	LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1699
1700	# if PGM_GST_TYPE == PGM_TYPE_PAE \
1701	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
1702
1703	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1704	PX86PML4E pPml4eSrc;
1705	PX86PDPE pPdpeSrc;
1706
1707	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc);
1708	Assert(pPml4eSrc);
1709
1710	/*
1711	* Real page fault? (PML4E level)
1712	*/
1713	if ( (uErr & X86_TRAP_PF_RSVD)
1714	\|\| !pPml4eSrc->n.u1Present
1715	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
1716	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1717	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
1718	)
1719	{
1720	uPageFaultLevel = 0;
1721	goto UpperLevelPageFault;
1722	}
1723	Assert(pPdpeSrc);
1724
1725	# else /* PAE */
1726	PX86PDPE pPdpeSrc = &pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtrPage >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
1727	# endif
1728
1729	/*
1730	* Real page fault? (PDPE level)
1731	*/
1732	if ( (uErr & X86_TRAP_PF_RSVD)
1733	\|\| !pPdpeSrc->n.u1Present
1734	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
1735	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
1736	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
1737	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
1738	# endif
1739	)
1740	{
1741	uPageFaultLevel = 1;
1742	goto UpperLevelPageFault;
1743	}
1744	# endif
1745
1746	/*
1747	* Real page fault? (PDE level)
1748	*/
1749	if ( (uErr & X86_TRAP_PF_RSVD)
1750	\|\| !pPdeSrc->n.u1Present
1751	# if PGM_WITH_NX(PGM_GST_TYPE)
1752	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
1753	# endif
1754	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1755	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
1756	{
1757	uPageFaultLevel = 2;
1758	goto UpperLevelPageFault;
1759	}
1760
1761	/*
1762	* First check the easy case where the page directory has been marked read-only to track
1763	* the dirty bit of an emulated BIG page
1764	*/
1765	if (pPdeSrc->b.u1Size && fBigPagesSupported)
1766	{
1767	/* Mark guest page directory as accessed */
1768	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1769	pPml4eSrc->n.u1Accessed = 1;
1770	pPdpeSrc->lm.u1Accessed = 1;
1771	# endif
1772	pPdeSrc->b.u1Accessed = 1;
1773
1774	/*
1775	* Only write protection page faults are relevant here.
1776	*/
1777	if (fWriteFault)
1778	{
1779	/* Mark guest page directory as dirty (BIG page only). */
1780	pPdeSrc->b.u1Dirty = 1;
1781
1782	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1783	{
1784	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1785
1786	Assert(pPdeSrc->b.u1Write);
1787
1788	pPdeDst->n.u1Write = 1;
1789	pPdeDst->n.u1Accessed = 1;
1790	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1791	PGM_INVL_BIG_PG(GCPtrPage);
1792	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1793	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1794	}
1795	}
1796	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1797	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1798	}
1799	/* else: 4KB page table */
1800
1801	/*
1802	* Map the guest page table.
1803	*/
1804	PGSTPT pPTSrc;
1805	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1806	if (VBOX_SUCCESS(rc))
1807	{
1808	/*
1809	* Real page fault?
1810	*/
1811	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
1812	const GSTPTE PteSrc = *pPteSrc;
1813	if ( !PteSrc.n.u1Present
1814	# if PGM_WITH_NX(PGM_GST_TYPE)
1815	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
1816	# endif
1817	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1818	\|\| (fUserLevelFault && !PteSrc.n.u1User)
1819	)
1820	{
1821	# ifdef IN_GC
1822	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1823	# endif
1824	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1825	LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1826
1827	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1828	* See the 2nd case above as well.
1829	*/
1830	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1831	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1832
1833	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1834	return VINF_EM_RAW_GUEST_TRAP;
1835	}
1836	LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1837
1838	/*
1839	* Set the accessed bits in the page directory and the page table.
1840	*/
1841	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1842	pPml4eSrc->n.u1Accessed = 1;
1843	pPdpeSrc->lm.u1Accessed = 1;
1844	# endif
1845	pPdeSrc->n.u1Accessed = 1;
1846	pPteSrc->n.u1Accessed = 1;
1847
1848	/*
1849	* Only write protection page faults are relevant here.
1850	*/
1851	if (fWriteFault)
1852	{
1853	/* Write access, so mark guest entry as dirty. */
1854	# if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1855	if (!pPteSrc->n.u1Dirty)
1856	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1857	else
1858	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1859	# endif
1860	pPteSrc->n.u1Dirty = 1;
1861
1862	if (pPdeDst->n.u1Present)
1863	{
1864	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1865	* Our individual shadow handlers will provide more information and force a fatal exit.
1866	*/
1867	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1868	{
1869	LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1870	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1871	return VINF_SUCCESS;
1872	}
1873
1874	/*
1875	* Map shadow page table.
1876	*/
1877	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1878	if (pShwPage)
1879	{
1880	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1881	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1882	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1883	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1884	{
1885	LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1886	# ifdef VBOX_STRICT
1887	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
1888	if (pPage)
1889	AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
1890	("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1891	# endif
1892	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1893
1894	Assert(pPteSrc->n.u1Write);
1895
1896	pPteDst->n.u1Write = 1;
1897	pPteDst->n.u1Dirty = 1;
1898	pPteDst->n.u1Accessed = 1;
1899	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1900	PGM_INVL_PG(GCPtrPage);
1901
1902	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1903	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1904	}
1905	}
1906	else
1907	AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
1908	}
1909	}
1910	/** @todo Optimize accessed bit emulation? */
1911	# ifdef VBOX_STRICT
1912	/*
1913	* Sanity check.
1914	*/
1915	else if ( !pPteSrc->n.u1Dirty
1916	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
1917	&& pPdeDst->n.u1Present)
1918	{
1919	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1920	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1921	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1922	if ( pPteDst->n.u1Present
1923	&& pPteDst->n.u1Write)
1924	LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
1925	}
1926	# endif /* VBOX_STRICT */
1927	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1928	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1929	}
1930	AssertRC(rc);
1931	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1932	return rc;
1933
1934
1935	UpperLevelPageFault:
1936	/* Pagefault detected while checking the PML4E, PDPE or PDE.
1937	* Single exit handler to get rid of duplicate code paths.
1938	*/
1939	# ifdef IN_GC
1940	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1941	# endif
1942	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1943	LogFlow(("CheckPageFault: real page fault at %VGv (%d)\n", GCPtrPage, uPageFaultLevel));
1944
1945	if (
1946	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1947	pPml4eSrc->n.u1Present &&
1948	# endif
1949	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
1950	pPdpeSrc->n.u1Present &&
1951	# endif
1952	pPdeSrc->n.u1Present)
1953	{
1954	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
1955	if (pPdeSrc->b.u1Size && fBigPagesSupported)
1956	{
1957	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1958	}
1959	else
1960	{
1961	/*
1962	* Map the guest page table.
1963	*/
1964	PGSTPT pPTSrc;
1965	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1966	if (VBOX_SUCCESS(rc))
1967	{
1968	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
1969	const GSTPTE PteSrc = *pPteSrc;
1970	if (pPteSrc->n.u1Present)
1971	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1972	}
1973	AssertRC(rc);
1974	}
1975	}
1976	return VINF_EM_RAW_GUEST_TRAP;
1977	}
1978
1979	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
1980
1981
1982	/**
1983	* Sync a shadow page table.
1984	*
1985	* The shadow page table is not present. This includes the case where
1986	* there is a conflict with a mapping.
1987	*
1988	* @returns VBox status code.
1989	* @param pVM VM handle.
1990	* @param iPD Page directory index.
1991	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
1992	* Assume this is a temporary mapping.
1993	* @param GCPtrPage GC Pointer of the page that caused the fault
1994	*/
1995	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
1996	{
1997	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1998	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
1999	LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
2000
2001	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
2002	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
2003
2004	/*
2005	* Validate input a little bit.
2006	*/
2007	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%VGv\n", iPDSrc, GCPtrPage));
2008	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2009	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2010	# else
2011	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2012	# endif
2013	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2014	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2015	SHWPDE PdeDst = *pPdeDst;
2016
2017	# ifndef PGM_WITHOUT_MAPPINGS
2018	/*
2019	* Check for conflicts.
2020	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2021	* HC: Simply resolve the conflict.
2022	*/
2023	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2024	{
2025	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2026	# ifndef IN_RING3
2027	Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
2028	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2029	return VERR_ADDRESS_CONFLICT;
2030	# else
2031	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2032	Assert(pMapping);
2033	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2034	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2035	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2036	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2037	# endif
2038	if (VBOX_FAILURE(rc))
2039	{
2040	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2041	return rc;
2042	}
2043	PdeDst = *pPdeDst;
2044	# endif
2045	}
2046	# else /* PGM_WITHOUT_MAPPINGS */
2047	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2048	# endif /* PGM_WITHOUT_MAPPINGS */
2049	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2050
2051	/*
2052	* Sync page directory entry.
2053	*/
2054	int rc = VINF_SUCCESS;
2055	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2056	if (PdeSrc.n.u1Present)
2057	{
2058	/*
2059	* Allocate & map the page table.
2060	*/
2061	PSHWPT pPTDst;
2062	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
2063	PPGMPOOLPAGE pShwPage;
2064	RTGCPHYS GCPhys;
2065	if (fPageTable)
2066	{
2067	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2068	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2069	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2070	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2071	# endif
2072	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2073	}
2074	else
2075	{
2076	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2077	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2078	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2079	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2080	# endif
2081	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2082	}
2083	if (rc == VINF_SUCCESS)
2084	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2085	else if (rc == VINF_PGM_CACHED_PAGE)
2086	{
2087	/*
2088	* The PT was cached, just hook it up.
2089	*/
2090	if (fPageTable)
2091	PdeDst.u = pShwPage->Core.Key
2092	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2093	else
2094	{
2095	PdeDst.u = pShwPage->Core.Key
2096	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2097	# ifdef PGM_SYNC_DIRTY_BIT /* (see explanation and assumptions further down.) */
2098	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2099	{
2100	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2101	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2102	PdeDst.b.u1Write = 0;
2103	}
2104	# endif
2105	}
2106	*pPdeDst = PdeDst;
2107	return VINF_SUCCESS;
2108	}
2109	else if (rc == VERR_PGM_POOL_FLUSHED)
2110	return VINF_PGM_SYNC_CR3;
2111	else
2112	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2113	PdeDst.u &= X86_PDE_AVL_MASK;
2114	PdeDst.u \|= pShwPage->Core.Key;
2115
2116	# ifdef PGM_SYNC_DIRTY_BIT
2117	/*
2118	* Page directory has been accessed (this is a fault situation, remember).
2119	*/
2120	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2121	# endif
2122	if (fPageTable)
2123	{
2124	/*
2125	* Page table - 4KB.
2126	*
2127	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2128	*/
2129	Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2130	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2131	PGSTPT pPTSrc;
2132	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2133	if (VBOX_SUCCESS(rc))
2134	{
2135	/*
2136	* Start by syncing the page directory entry so CSAM's TLB trick works.
2137	*/
2138	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2139	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2140	*pPdeDst = PdeDst;
2141
2142	/*
2143	* Directory/page user or supervisor privilege: (same goes for read/write)
2144	*
2145	* Directory Page Combined
2146	* U/S U/S U/S
2147	* 0 0 0
2148	* 0 1 0
2149	* 1 0 0
2150	* 1 1 1
2151	*
2152	* Simple AND operation. Table listed for completeness.
2153	*
2154	*/
2155	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
2156	# ifdef PGM_SYNC_N_PAGES
2157	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2158	unsigned iPTDst = iPTBase;
2159	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
2160	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2161	iPTDst = 0;
2162	else
2163	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2164	# else /* !PGM_SYNC_N_PAGES */
2165	unsigned iPTDst = 0;
2166	const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
2167	# endif /* !PGM_SYNC_N_PAGES */
2168	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2169	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2170	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2171	# else
2172	const unsigned offPTSrc = 0;
2173	# endif
2174	for (; iPTDst < iPTDstEnd; iPTDst++)
2175	{
2176	const unsigned iPTSrc = iPTDst + offPTSrc;
2177	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2178
2179	if (PteSrc.n.u1Present) /* we've already cleared it above */
2180	{
2181	# ifndef IN_RING0
2182	/*
2183	* Assuming kernel code will be marked as supervisor - and not as user level
2184	* and executed using a conforming code selector - And marked as readonly.
2185	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2186	*/
2187	PPGMPAGE pPage;
2188	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2189	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2190	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2191	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2192	)
2193	# endif
2194	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2195	Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
2196	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
2197	PteSrc.n.u1Present,
2198	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2199	PteSrc.n.u1User & PdeSrc.n.u1User,
2200	(uint64_t)PteSrc.u,
2201	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2202	(PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
2203	}
2204	} /* for PTEs */
2205	}
2206	}
2207	else
2208	{
2209	/*
2210	* Big page - 2/4MB.
2211	*
2212	* We'll walk the ram range list in parallel and optimize lookups.
2213	* We will only sync on shadow page table at a time.
2214	*/
2215	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
2216
2217	/**
2218	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2219	*/
2220
2221	/*
2222	* Start by syncing the page directory entry.
2223	*/
2224	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2225	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2226
2227	# ifdef PGM_SYNC_DIRTY_BIT
2228	/*
2229	* If the page is not flagged as dirty and is writable, then make it read-only
2230	* at PD level, so we can set the dirty bit when the page is modified.
2231	*
2232	* ASSUMES that page access handlers are implemented on page table entry level.
2233	* Thus we will first catch the dirty access and set PDE.D and restart. If
2234	* there is an access handler, we'll trap again and let it work on the problem.
2235	*/
2236	/** @todo move the above stuff to a section in the PGM documentation. */
2237	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2238	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2239	{
2240	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2241	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2242	PdeDst.b.u1Write = 0;
2243	}
2244	# endif /* PGM_SYNC_DIRTY_BIT */
2245	*pPdeDst = PdeDst;
2246
2247	/*
2248	* Fill the shadow page table.
2249	*/
2250	/* Get address and flags from the source PDE. */
2251	SHWPTE PteDstBase;
2252	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2253
2254	/* Loop thru the entries in the shadow PT. */
2255	const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2256	Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2257	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2258	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2259	PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
2260	unsigned iPTDst = 0;
2261	while (iPTDst < ELEMENTS(pPTDst->a))
2262	{
2263	/* Advance ram range list. */
2264	while (pRam && GCPhys > pRam->GCPhysLast)
2265	pRam = CTXALLSUFF(pRam->pNext);
2266	if (pRam && GCPhys >= pRam->GCPhys)
2267	{
2268	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2269	do
2270	{
2271	/* Make shadow PTE. */
2272	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2273	SHWPTE PteDst;
2274
2275	/* Make sure the RAM has already been allocated. */
2276	if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2277	{
2278	if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2279	{
2280	# ifdef IN_RING3
2281	int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2282	# else
2283	int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2284	# endif
2285	if (rc != VINF_SUCCESS)
2286	return rc;
2287	}
2288	}
2289
2290	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2291	{
2292	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2293	{
2294	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2295	PteDst.n.u1Write = 0;
2296	}
2297	else
2298	PteDst.u = 0;
2299	}
2300	# ifndef IN_RING0
2301	/*
2302	* Assuming kernel code will be marked as supervisor and not as user level and executed
2303	* using a conforming code selector. Don't check for readonly, as that implies the whole
2304	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2305	*/
2306	else if ( !PdeSrc.n.u1User
2307	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2308	PteDst.u = 0;
2309	# endif
2310	else
2311	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2312	# ifdef PGMPOOL_WITH_USER_TRACKING
2313	if (PteDst.n.u1Present)
2314	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2315	# endif
2316	/* commit it */
2317	pPTDst->a[iPTDst] = PteDst;
2318	Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2319	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2320	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2321
2322	/* advance */
2323	GCPhys += PAGE_SIZE;
2324	iHCPage++;
2325	iPTDst++;
2326	} while ( iPTDst < ELEMENTS(pPTDst->a)
2327	&& GCPhys <= pRam->GCPhysLast);
2328	}
2329	else if (pRam)
2330	{
2331	Log(("Invalid pages at %VGp\n", GCPhys));
2332	do
2333	{
2334	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2335	GCPhys += PAGE_SIZE;
2336	iPTDst++;
2337	} while ( iPTDst < ELEMENTS(pPTDst->a)
2338	&& GCPhys < pRam->GCPhys);
2339	}
2340	else
2341	{
2342	Log(("Invalid pages at %VGp (2)\n", GCPhys));
2343	for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2344	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2345	}
2346	} /* while more PTEs */
2347	} /* 4KB / 4MB */
2348	}
2349	else
2350	AssertRelease(!PdeDst.n.u1Present);
2351
2352	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2353	# ifdef IN_GC
2354	if (VBOX_FAILURE(rc))
2355	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2356	# endif
2357	return rc;
2358
2359	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
2360
2361	int rc = VINF_SUCCESS;
2362
2363	/*
2364	* Validate input a little bit.
2365	*/
2366	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2367	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2368	# else
2369	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2370	# endif
2371	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2372	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2373	SHWPDE PdeDst = *pPdeDst;
2374
2375	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2376	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2377
2378	GSTPDE PdeSrc;
2379	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2380	PdeSrc.n.u1Present = 1;
2381	PdeSrc.n.u1Write = 1;
2382	PdeSrc.n.u1Accessed = 1;
2383	PdeSrc.n.u1User = 1;
2384
2385	/*
2386	* Allocate & map the page table.
2387	*/
2388	PSHWPT pPTDst;
2389	PPGMPOOLPAGE pShwPage;
2390	RTGCPHYS GCPhys;
2391
2392	/* Virtual address = physical address */
2393	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK_32;
2394	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2395
2396	if ( rc == VINF_SUCCESS
2397	\|\| rc == VINF_PGM_CACHED_PAGE)
2398	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2399	else
2400	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2401
2402	PdeDst.u &= X86_PDE_AVL_MASK;
2403	PdeDst.u \|= pShwPage->Core.Key;
2404	PdeDst.n.u1Present = 1;
2405	*pPdeDst = PdeDst;
2406
2407	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2408	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2409	return rc;
2410
2411	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2412	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2413	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2414	return VERR_INTERNAL_ERROR;
2415	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2416	}
2417
2418
2419
2420	/**
2421	* Prefetch a page/set of pages.
2422	*
2423	* Typically used to sync commonly used pages before entering raw mode
2424	* after a CR3 reload.
2425	*
2426	* @returns VBox status code.
2427	* @param pVM VM handle.
2428	* @param GCPtrPage Page to invalidate.
2429	*/
2430	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2431	{
2432	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2433	/*
2434	* Check that all Guest levels thru the PDE are present, getting the
2435	* PD and PDE in the processes.
2436	*/
2437	int rc = VINF_SUCCESS;
2438	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2439	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2440	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2441	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2442	# else /* PAE */
2443	unsigned iPDSrc;
2444	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2445	if (!pPDSrc)
2446	return VINF_SUCCESS; /* not present */
2447	# endif
2448	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2449	# else
2450	PGSTPD pPDSrc = NULL;
2451	const unsigned iPDSrc = 0;
2452	GSTPDE PdeSrc;
2453
2454	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2455	PdeSrc.n.u1Present = 1;
2456	PdeSrc.n.u1Write = 1;
2457	PdeSrc.n.u1Accessed = 1;
2458	PdeSrc.n.u1User = 1;
2459	# endif
2460
2461	# ifdef PGM_SYNC_ACCESSED_BIT
2462	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2463	# else
2464	if (PdeSrc.n.u1Present)
2465	# endif
2466	{
2467	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2468	const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2469	# else
2470	const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2471	# endif
2472	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2473	{
2474	if (!PdeDst.n.u1Present)
2475	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2476	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2477	else
2478	{
2479	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2480	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2481	* makes no sense to prefetch more than one page.
2482	*/
2483	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2484	if (VBOX_SUCCESS(rc))
2485	rc = VINF_SUCCESS;
2486	}
2487	}
2488	}
2489	return rc;
2490
2491	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2492
2493	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2494	return VERR_INTERNAL_ERROR;
2495	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2496	}
2497
2498
2499
2500
2501	/**
2502	* Syncs a page during a PGMVerifyAccess() call.
2503	*
2504	* @returns VBox status code (informational included).
2505	* @param GCPtrPage The address of the page to sync.
2506	* @param fPage The effective guest page flags.
2507	* @param uErr The trap error code.
2508	*/
2509	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2510	{
2511	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2512
2513	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2514
2515	# ifndef IN_RING0
2516	if (!(fPage & X86_PTE_US))
2517	{
2518	/*
2519	* Mark this page as safe.
2520	*/
2521	/** @todo not correct for pages that contain both code and data!! */
2522	Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2523	CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2524	}
2525	# endif
2526	/*
2527	* Get guest PD and index.
2528	*/
2529
2530	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2531	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2532	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2533	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2534	# else /* PAE */
2535	unsigned iPDSrc;
2536	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2537
2538	if (pPDSrc)
2539	{
2540	Log(("PGMVerifyAccess: access violation for %VGv due to non-present PDPTR\n", GCPtrPage));
2541	return VINF_EM_RAW_GUEST_TRAP;
2542	}
2543	# endif
2544	# else
2545	PGSTPD pPDSrc = NULL;
2546	const unsigned iPDSrc = 0;
2547	# endif
2548	int rc = VINF_SUCCESS;
2549
2550	/*
2551	* First check if the shadow pd is present.
2552	*/
2553	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2554	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2555	# else
2556	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2557	# endif
2558	if (!pPdeDst->n.u1Present)
2559	{
2560	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2561	AssertRC(rc);
2562	if (rc != VINF_SUCCESS)
2563	return rc;
2564	}
2565
2566	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2567	/* Check for dirty bit fault */
2568	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2569	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2570	Log(("PGMVerifyAccess: success (dirty)\n"));
2571	else
2572	{
2573	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2574	#else
2575	{
2576	GSTPDE PdeSrc;
2577	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2578	PdeSrc.n.u1Present = 1;
2579	PdeSrc.n.u1Write = 1;
2580	PdeSrc.n.u1Accessed = 1;
2581	PdeSrc.n.u1User = 1;
2582
2583	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2584	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2585	if (uErr & X86_TRAP_PF_US)
2586	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2587	else /* supervisor */
2588	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2589
2590	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2591	if (VBOX_SUCCESS(rc))
2592	{
2593	/* Page was successfully synced */
2594	Log2(("PGMVerifyAccess: success (sync)\n"));
2595	rc = VINF_SUCCESS;
2596	}
2597	else
2598	{
2599	Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2600	return VINF_EM_RAW_GUEST_TRAP;
2601	}
2602	}
2603	return rc;
2604
2605	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2606
2607	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2608	return VERR_INTERNAL_ERROR;
2609	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2610	}
2611
2612
2613	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2614	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2615	/**
2616	* Figures out which kind of shadow page this guest PDE warrants.
2617	*
2618	* @returns Shadow page kind.
2619	* @param pPdeSrc The guest PDE in question.
2620	* @param cr4 The current guest cr4 value.
2621	*/
2622	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
2623	{
2624	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
2625	return BTH_PGMPOOLKIND_PT_FOR_PT;
2626	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
2627	//{
2628	// case 0:
2629	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2630	// case X86_PDE4M_RW:
2631	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2632	// case X86_PDE4M_US:
2633	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2634	// case X86_PDE4M_RW \| X86_PDE4M_US:
2635	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2636	# if 0
2637	// case X86_PDE4M_PAE_NX:
2638	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2639	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
2640	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2641	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2642	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2643	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2644	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2645	# endif
2646	return BTH_PGMPOOLKIND_PT_FOR_BIG;
2647	//}
2648	}
2649	# endif
2650	#endif
2651
2652	#undef MY_STAM_COUNTER_INC
2653	#define MY_STAM_COUNTER_INC(a) do { } while (0)
2654
2655
2656	/**
2657	* Syncs the paging hierarchy starting at CR3.
2658	*
2659	* @returns VBox status code, no specials.
2660	* @param pVM The virtual machine.
2661	* @param cr0 Guest context CR0 register
2662	* @param cr3 Guest context CR3 register
2663	* @param cr4 Guest context CR4 register
2664	* @param fGlobal Including global page directories or not
2665	*/
2666	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2667	{
2668	if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2669	fGlobal = true; /* Change this CR3 reload to be a global one. */
2670
2671	/*
2672	* Update page access handlers.
2673	* The virtual are always flushed, while the physical are only on demand.
2674	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2675	* have to look into that later because it will have a bad influence on the performance.
2676	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
2677	* bird: Yes, but that won't work for aliases.
2678	*/
2679	/** @todo this MUST go away. See #1557. */
2680	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2681	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2682	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2683
2684	#ifdef PGMPOOL_WITH_MONITORING
2685	/*
2686	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2687	* Occationally we will have to clear all the shadow page tables because we wanted
2688	* to monitor a page which was mapped by too many shadowed page tables. This operation
2689	* sometimes refered to as a 'lightweight flush'.
2690	*/
2691	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2692	pgmPoolMonitorModifiedClearAll(pVM);
2693	else
2694	{
2695	# ifdef IN_RING3
2696	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2697	pgmPoolClearAll(pVM);
2698	# else
2699	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2700	return VINF_PGM_SYNC_CR3;
2701	# endif
2702	}
2703	#endif
2704
2705	Assert(fGlobal \|\| (cr4 & X86_CR4_PGE));
2706	MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2707
2708	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2709	/*
2710	* Get page directory addresses.
2711	*/
2712	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2713	PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2714	# else /* PGM_SHW_TYPE == PGM_TYPE_PAE */
2715	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2716	PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2717	# endif
2718	# endif
2719
2720	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2721	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2722	Assert(pPDSrc);
2723	# ifndef IN_GC
2724	Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & GST_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2725	# endif
2726	# endif
2727
2728	/*
2729	* Iterate the page directory.
2730	*/
2731	PPGMMAPPING pMapping;
2732	unsigned iPdNoMapping;
2733	const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2734	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2735
2736	/* Only check mappings if they are supposed to be put into the shadow page table. */
2737	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2738	{
2739	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2740	iPdNoMapping = (pMapping) ? (pMapping->GCPtr >> GST_PD_SHIFT) : ~0U;
2741	}
2742	else
2743	{
2744	pMapping = 0;
2745	iPdNoMapping = ~0U;
2746	}
2747	# if PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2748	for (unsigned iPDPTE = 0; iPDPTE < GST_PDPE_ENTRIES; iPDPTE++)
2749	{
2750	unsigned iPDSrc;
2751	# if PGM_SHW_TYPE == PGM_TYPE_PAE
2752	PX86PDPAE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2753	# else
2754	AssertFailed(); /* @todo */
2755	PX86PDPE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[iPDPTE * X86_PG_AMD64_ENTRIES];
2756	# endif
2757	PX86PDEPAE pPDEDst = &pPDPAE->a[iPDPTE * X86_PG_PAE_ENTRIES];
2758	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, iPDPTE << X86_PDPT_SHIFT, &iPDSrc);
2759
2760	if (pPDSrc == NULL)
2761	{
2762	/* PDPT not present */
2763	if (pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
2764	{
2765	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2766	{
2767	if ( pPDEDst[iPD].n.u1Present
2768	&& !(pPDEDst[iPD].u & PGM_PDFLAGS_MAPPING))
2769	{
2770	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst[iPD].u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPDPTE * X86_PG_PAE_ENTRIES + iPD);
2771	pPDEDst[iPD].u = 0;
2772	}
2773	}
2774	}
2775	if (!(pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].u & PGM_PLXFLAGS_MAPPING))
2776	pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 0;
2777	continue;
2778	}
2779	# else /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2780	{
2781	# endif /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2782	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2783	{
2784	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2785	Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2786	# elif PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2787	AssertMsg(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst, ("%p vs %p\n", &pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512], pPDEDst));
2788	# endif
2789	register GSTPDE PdeSrc = pPDSrc->a[iPD];
2790	if ( PdeSrc.n.u1Present
2791	&& (PdeSrc.n.u1User \|\| fRawR0Enabled))
2792	{
2793	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2794	\|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
2795	&& !defined(PGM_WITHOUT_MAPPINGS)
2796
2797	/*
2798	* Check for conflicts with GC mappings.
2799	*/
2800	# if PGM_GST_TYPE == PGM_TYPE_PAE
2801	if (iPD + iPDPTE * X86_PG_PAE_ENTRIES == iPdNoMapping)
2802	# else
2803	if (iPD == iPdNoMapping)
2804	# endif
2805	{
2806	if (pVM->pgm.s.fMappingsFixed)
2807	{
2808	/* It's fixed, just skip the mapping. */
2809	const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
2810	iPD += cPTs - 1;
2811	pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
2812	pMapping = pMapping->CTXALLSUFF(pNext);
2813	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2814	continue;
2815	}
2816	# ifdef IN_RING3
2817	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2818	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
2819	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2820	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
2821	# endif
2822	if (VBOX_FAILURE(rc))
2823	return rc;
2824
2825	/*
2826	* Update iPdNoMapping and pMapping.
2827	*/
2828	pMapping = pVM->pgm.s.pMappingsR3;
2829	while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
2830	pMapping = pMapping->pNextR3;
2831	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2832	# else
2833	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2834	return VINF_PGM_SYNC_CR3;
2835	# endif
2836	}
2837	# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
2838	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2839	# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
2840	/*
2841	* Sync page directory entry.
2842	*
2843	* The current approach is to allocated the page table but to set
2844	* the entry to not-present and postpone the page table synching till
2845	* it's actually used.
2846	*/
2847	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2848	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2849	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2850	const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
2851	# else
2852	const unsigned iPdShw = iPD; NOREF(iPdShw);
2853	# endif
2854	{
2855	SHWPDE PdeDst = *pPDEDst;
2856	if (PdeDst.n.u1Present)
2857	{
2858	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2859	RTGCPHYS GCPhys;
2860	if ( !PdeSrc.b.u1Size
2861	\|\| !(cr4 & X86_CR4_PSE))
2862	{
2863	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2864	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2865	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2866	GCPhys \|= i * (PAGE_SIZE / 2);
2867	# endif
2868	}
2869	else
2870	{
2871	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2872	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2873	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2874	GCPhys \|= i * X86_PAGE_2M_SIZE;
2875	# endif
2876	}
2877
2878	if ( pShwPage->GCPhys == GCPhys
2879	&& pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2880	&& ( pShwPage->fCached
2881	\|\| ( !fGlobal
2882	&& ( false
2883	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2884	\|\| ( (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2885	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE)) /* global 2/4MB page. */
2886	\|\| ( !pShwPage->fSeenNonGlobal
2887	&& (cr4 & X86_CR4_PGE))
2888	# endif
2889	)
2890	)
2891	)
2892	&& ( (PdeSrc.u & (X86_PDE_US \| X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US \| X86_PDE_RW))
2893	\|\| ( (cr4 & X86_CR4_PSE)
2894	&& ((PdeSrc.u & (X86_PDE_US \| X86_PDE4M_PS \| X86_PDE4M_D)) \| PGM_PDFLAGS_TRACK_DIRTY)
2895	== ((PdeDst.u & (X86_PDE_US \| X86_PDE_RW \| PGM_PDFLAGS_TRACK_DIRTY)) \| X86_PDE4M_PS))
2896	)
2897	)
2898	{
2899	# ifdef VBOX_WITH_STATISTICS
2900	if ( !fGlobal
2901	&& (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2902	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE))
2903	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2904	else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2905	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2906	else
2907	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2908	# endif /* VBOX_WITH_STATISTICS */
2909	/** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2910	* The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2911	//# ifdef PGMPOOL_WITH_CACHE
2912	// pgmPoolCacheUsed(pPool, pShwPage);
2913	//# endif
2914	}
2915	else
2916	{
2917	pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
2918	pPDEDst->u = 0;
2919	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
2920	}
2921	}
2922	else
2923	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
2924	pPDEDst++;
2925	}
2926	}
2927	# if PGM_GST_TYPE == PGM_TYPE_PAE
2928	else if (iPD + iPDPTE * X86_PG_PAE_ENTRIES != iPdNoMapping)
2929	# else
2930	else if (iPD != iPdNoMapping)
2931	# endif
2932	{
2933	/*
2934	* Check if there is any page directory to mark not present here.
2935	*/
2936	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2937	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2938	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2939	const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
2940	# else
2941	const unsigned iPdShw = iPD; NOREF(iPdShw);
2942	# endif
2943	{
2944	if (pPDEDst->n.u1Present)
2945	{
2946	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
2947	pPDEDst->u = 0;
2948	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
2949	}
2950	pPDEDst++;
2951	}
2952	}
2953	else
2954	{
2955	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2956	\|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
2957	&& !defined(PGM_WITHOUT_MAPPINGS)
2958
2959	const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
2960
2961	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2962	if (pVM->pgm.s.fMappingsFixed)
2963	{
2964	/* It's fixed, just skip the mapping. */
2965	pMapping = pMapping->CTXALLSUFF(pNext);
2966	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2967	}
2968	else
2969	{
2970	/*
2971	* Check for conflicts for subsequent pagetables
2972	* and advance to the next mapping.
2973	*/
2974	iPdNoMapping = ~0U;
2975	unsigned iPT = cPTs;
2976	while (iPT-- > 1)
2977	{
2978	if ( pPDSrc->a[iPD + iPT].n.u1Present
2979	&& (pPDSrc->a[iPD + iPT].n.u1User \|\| fRawR0Enabled))
2980	{
2981	# ifdef IN_RING3
2982	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2983	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
2984	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2985	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
2986	# endif
2987	if (VBOX_FAILURE(rc))
2988	return rc;
2989
2990	/*
2991	* Update iPdNoMapping and pMapping.
2992	*/
2993	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2994	while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
2995	pMapping = pMapping->CTXALLSUFF(pNext);
2996	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2997	break;
2998	# else
2999	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3000	return VINF_PGM_SYNC_CR3;
3001	# endif
3002	}
3003	}
3004	if (iPdNoMapping == ~0U && pMapping)
3005	{
3006	pMapping = pMapping->CTXALLSUFF(pNext);
3007	if (pMapping)
3008	iPdNoMapping = pMapping->GCPtr >> GST_PD_SHIFT;
3009	}
3010	}
3011
3012	/* advance. */
3013	iPD += cPTs - 1;
3014	pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3015	# if PGM_GST_TYPE != PGM_SHW_TYPE
3016	AssertCompile(PGM_GST_TYPE == PGM_TYPE_32BIT && PGM_SHW_TYPE == PGM_TYPE_PAE);
3017	# endif
3018	# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
3019	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3020	# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
3021	}
3022
3023	} /* for iPD */
3024	} /* for each PDPTE (PAE) */
3025
3026	return VINF_SUCCESS;
3027
3028	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3029	//# error not implemented
3030	return VERR_INTERNAL_ERROR;
3031	#else /* guest real and protected mode */
3032	return VINF_SUCCESS;
3033	#endif
3034	}
3035
3036
3037
3038
3039	#ifdef VBOX_STRICT
3040	#ifdef IN_GC
3041	# undef AssertMsgFailed
3042	# define AssertMsgFailed Log
3043	#endif
3044	#ifdef IN_RING3
3045	# include <VBox/dbgf.h>
3046
3047	/**
3048	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3049	*
3050	* @returns VBox status code (VINF_SUCCESS).
3051	* @param pVM The VM handle.
3052	* @param cr3 The root of the hierarchy.
3053	* @param crr The cr4, only PAE and PSE is currently used.
3054	* @param fLongMode Set if long mode, false if not long mode.
3055	* @param cMaxDepth Number of levels to dump.
3056	* @param pHlp Pointer to the output functions.
3057	*/
3058	__BEGIN_DECLS
3059	PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3060	__END_DECLS
3061
3062	#endif
3063
3064	/**
3065	* Checks that the shadow page table is in sync with the guest one.
3066	*
3067	* @returns The number of errors.
3068	* @param pVM The virtual machine.
3069	* @param cr3 Guest context CR3 register
3070	* @param cr4 Guest context CR4 register
3071	* @param GCPtr Where to start. Defaults to 0.
3072	* @param cb How much to check. Defaults to everything.
3073	*/
3074	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
3075	{
3076	unsigned cErrors = 0;
3077
3078	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3079	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
3080
3081	PPGM pPGM = &pVM->pgm.s;
3082	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3083	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3084	# ifndef IN_RING0
3085	RTHCPHYS HCPhys; /* general usage. */
3086	# endif
3087	int rc;
3088
3089	/*
3090	* Check that the Guest CR3 and all its mappings are correct.
3091	*/
3092	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3093	("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3094	false);
3095	# ifndef IN_RING0
3096	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3097	rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
3098	# else
3099	rc = PGMShwGetPage(pVM, pPGM->pGstPaePDPTGC, NULL, &HCPhysShw);
3100	# endif
3101	AssertRCReturn(rc, 1);
3102	HCPhys = NIL_RTHCPHYS;
3103	rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3104	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
3105	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3106	RTGCPHYS GCPhys;
3107	rc = PGMR3DbgHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
3108	AssertRCReturn(rc, 1);
3109	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
3110	# endif
3111	#endif /* !IN_RING0 */
3112
3113	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3114	const GSTPD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
3115	# endif
3116
3117	/*
3118	* Get and check the Shadow CR3.
3119	*/
3120	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3121	const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
3122	unsigned cPDEs = ELEMENTS(pPDDst->a);
3123	# else
3124	const X86PDPAE pPDDst = pPGM->CTXMID(ap,PaePDs[0]); / use it as a 2048 entry PD */
3125	unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
3126	# endif
3127	if (cb != ~(RTGCUINTPTR)0)
3128	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3129
3130	/** @todo call the other two PGMAssert() functions. /
3131
3132	# if PGM_GST_TYPE == PGM_TYPE_PAE
3133	/*
3134	* Check the 4 PDPTs too.
3135	*/
3136	for (unsigned i = 0; i < 4; i++)
3137	{
3138	RTHCPTR HCPtr;
3139	RTHCPHYS HCPhys;
3140	RTGCPHYS GCPhys = pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[i].u & X86_PDPE_PG_MASK;
3141	int rc2 = pgmRamGCPhys2HCPtrAndHCPhysWithFlags(&pVM->pgm.s, GCPhys, &HCPtr, &HCPhys);
3142	if (VBOX_SUCCESS(rc2))
3143	{
3144	AssertMsg( pVM->pgm.s.apGstPaePDsHC[i] == (R3R0PTRTYPE(PX86PDPAE))HCPtr
3145	&& pVM->pgm.s.aGCPhysGstPaePDs[i] == GCPhys,
3146	("idx %d apGstPaePDsHC %VHv vs %VHv aGCPhysGstPaePDs %VGp vs %VGp\n",
3147	i, pVM->pgm.s.apGstPaePDsHC[i], HCPtr, pVM->pgm.s.aGCPhysGstPaePDs[i], GCPhys));
3148	}
3149	}
3150	# endif
3151
3152	/*
3153	* Iterate the shadow page directory.
3154	*/
3155	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3156	unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
3157	cPDEs += iPDDst;
3158	for (;
3159	iPDDst < cPDEs;
3160	iPDDst++, GCPtr += _4G / cPDEs)
3161	{
3162	# if PGM_GST_TYPE == PGM_TYPE_PAE
3163	uint32_t iPDSrc;
3164	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pPGM, (RTGCUINTPTR)GCPtr, &iPDSrc);
3165	if (!pPDSrc)
3166	{
3167	AssertMsg(!pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].n.u1Present, ("Guest PDTPR not present, shadow PDPTR %VX64\n", pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].u));
3168	continue;
3169	}
3170	#endif
3171
3172	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3173	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3174	{
3175	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3176	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3177	{
3178	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3179	cErrors++;
3180	continue;
3181	}
3182	}
3183	else if ( (PdeDst.u & X86_PDE_P)
3184	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3185	)
3186	{
3187	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3188	PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3189	if (!pPoolPage)
3190	{
3191	AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
3192	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3193	cErrors++;
3194	continue;
3195	}
3196	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3197
3198	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3199	{
3200	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
3201	GCPtr, (uint64_t)PdeDst.u));
3202	cErrors++;
3203	}
3204
3205	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3206	{
3207	AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
3208	GCPtr, (uint64_t)PdeDst.u));
3209	cErrors++;
3210	}
3211
3212	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3213	if (!PdeSrc.n.u1Present)
3214	{
3215	AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3216	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3217	cErrors++;
3218	continue;
3219	}
3220
3221	if ( !PdeSrc.b.u1Size
3222	\|\| !(cr4 & X86_CR4_PSE))
3223	{
3224	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3225	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3226	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3227	# endif
3228	}
3229	else
3230	{
3231	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3232	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3233	{
3234	AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3235	GCPtr, (uint64_t)PdeSrc.u));
3236	cErrors++;
3237	continue;
3238	}
3239	# endif
3240	GCPhysGst = PdeSrc.u & GST_PDE_BIG_PG_MASK;
3241	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3242	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3243	# endif
3244	}
3245
3246	if ( pPoolPage->enmKind
3247	!= (!PdeSrc.b.u1Size \|\| !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3248	{
3249	AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
3250	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3251	cErrors++;
3252	}
3253
3254	PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3255	if (!pPhysPage)
3256	{
3257	AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3258	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3259	cErrors++;
3260	continue;
3261	}
3262
3263	if (GCPhysGst != pPoolPage->GCPhys)
3264	{
3265	AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
3266	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3267	cErrors++;
3268	continue;
3269	}
3270
3271	if ( !PdeSrc.b.u1Size
3272	\|\| !(cr4 & X86_CR4_PSE))
3273	{
3274	/*
3275	* Page Table.
3276	*/
3277	const GSTPT *pPTSrc;
3278	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3279	if (VBOX_FAILURE(rc))
3280	{
3281	AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3282	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3283	cErrors++;
3284	continue;
3285	}
3286	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3287	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3288	{
3289	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3290	// (This problem will go away when/if we shadow multiple CR3s.)
3291	AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3292	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3293	cErrors++;
3294	continue;
3295	}
3296	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3297	{
3298	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
3299	GCPtr, (uint64_t)PdeDst.u));
3300	cErrors++;
3301	continue;
3302	}
3303
3304	/* iterate the page table. */
3305	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3306	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3307	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3308	# else
3309	const unsigned offPTSrc = 0;
3310	# endif
3311	for (unsigned iPT = 0, off = 0;
3312	iPT < ELEMENTS(pPTDst->a);
3313	iPT++, off += PAGE_SIZE)
3314	{
3315	const SHWPTE PteDst = pPTDst->a[iPT];
3316
3317	/* skip not-present entries. */
3318	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3319	continue;
3320	Assert(PteDst.n.u1Present);
3321
3322	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3323	if (!PteSrc.n.u1Present)
3324	{
3325	#ifdef IN_RING3
3326	PGMAssertHandlerAndFlagsInSync(pVM);
3327	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3328	#endif
3329	AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
3330	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3331	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3332	cErrors++;
3333	continue;
3334	}
3335
3336	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3337	# if 1 /** @todo sync accessed bit properly... */
3338	fIgnoreFlags \|= X86_PTE_A;
3339	# endif
3340
3341	/* match the physical addresses */
3342	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3343	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3344
3345	# ifdef IN_RING3
3346	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3347	if (VBOX_FAILURE(rc))
3348	{
3349	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3350	{
3351	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3352	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3353	cErrors++;
3354	continue;
3355	}
3356	}
3357	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3358	{
3359	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3360	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3361	cErrors++;
3362	continue;
3363	}
3364	# endif
3365
3366	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3367	if (!pPhysPage)
3368	{
3369	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3370	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3371	{
3372	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3373	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3374	cErrors++;
3375	continue;
3376	}
3377	# endif
3378	if (PteDst.n.u1Write)
3379	{
3380	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3381	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3382	cErrors++;
3383	}
3384	fIgnoreFlags \|= X86_PTE_RW;
3385	}
3386	else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
3387	{
3388	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3389	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3390	cErrors++;
3391	continue;
3392	}
3393
3394	/* flags */
3395	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3396	{
3397	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3398	{
3399	if (PteDst.n.u1Write)
3400	{
3401	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
3402	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3403	cErrors++;
3404	continue;
3405	}
3406	fIgnoreFlags \|= X86_PTE_RW;
3407	}
3408	else
3409	{
3410	if (PteDst.n.u1Present)
3411	{
3412	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
3413	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3414	cErrors++;
3415	continue;
3416	}
3417	fIgnoreFlags \|= X86_PTE_P;
3418	}
3419	}
3420	else
3421	{
3422	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3423	{
3424	if (PteDst.n.u1Write)
3425	{
3426	AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3427	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3428	cErrors++;
3429	continue;
3430	}
3431	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3432	{
3433	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3434	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3435	cErrors++;
3436	continue;
3437	}
3438	if (PteDst.n.u1Dirty)
3439	{
3440	AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3441	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3442	cErrors++;
3443	}
3444	# if 0 /** @todo sync access bit properly... */
3445	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3446	{
3447	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3448	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3449	cErrors++;
3450	}
3451	fIgnoreFlags \|= X86_PTE_RW;
3452	# else
3453	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3454	# endif
3455	}
3456	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3457	{
3458	/* access bit emulation (not implemented). */
3459	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3460	{
3461	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3462	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3463	cErrors++;
3464	continue;
3465	}
3466	if (!PteDst.n.u1Accessed)
3467	{
3468	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3469	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3470	cErrors++;
3471	}
3472	fIgnoreFlags \|= X86_PTE_P;
3473	}
3474	# ifdef DEBUG_sandervl
3475	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3476	# endif
3477	}
3478
3479	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3480	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3481	)
3482	{
3483	AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3484	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3485	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3486	cErrors++;
3487	continue;
3488	}
3489	} /* foreach PTE */
3490	}
3491	else
3492	{
3493	/*
3494	* Big Page.
3495	*/
3496	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3497	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3498	{
3499	if (PdeDst.n.u1Write)
3500	{
3501	AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3502	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3503	cErrors++;
3504	continue;
3505	}
3506	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3507	{
3508	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3509	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3510	cErrors++;
3511	continue;
3512	}
3513	# if 0 /** @todo sync access bit properly... */
3514	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3515	{
3516	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3517	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3518	cErrors++;
3519	}
3520	fIgnoreFlags \|= X86_PTE_RW;
3521	# else
3522	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3523	# endif
3524	}
3525	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3526	{
3527	/* access bit emulation (not implemented). */
3528	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3529	{
3530	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3531	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3532	cErrors++;
3533	continue;
3534	}
3535	if (!PdeDst.n.u1Accessed)
3536	{
3537	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3538	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3539	cErrors++;
3540	}
3541	fIgnoreFlags \|= X86_PTE_P;
3542	}
3543
3544	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3545	{
3546	AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3547	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3548	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3549	cErrors++;
3550	}
3551
3552	/* iterate the page table. */
3553	for (unsigned iPT = 0, off = 0;
3554	iPT < ELEMENTS(pPTDst->a);
3555	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3556	{
3557	const SHWPTE PteDst = pPTDst->a[iPT];
3558
3559	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3560	{
3561	AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3562	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3563	cErrors++;
3564	}
3565
3566	/* skip not-present entries. */
3567	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3568	continue;
3569
3570	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3571
3572	/* match the physical addresses */
3573	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3574
3575	# ifdef IN_RING3
3576	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3577	if (VBOX_FAILURE(rc))
3578	{
3579	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3580	{
3581	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3582	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3583	cErrors++;
3584	}
3585	}
3586	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3587	{
3588	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3589	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3590	cErrors++;
3591	continue;
3592	}
3593	# endif
3594
3595	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3596	if (!pPhysPage)
3597	{
3598	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3599	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3600	{
3601	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3602	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3603	cErrors++;
3604	continue;
3605	}
3606	# endif
3607	if (PteDst.n.u1Write)
3608	{
3609	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3610	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3611	cErrors++;
3612	}
3613	fIgnoreFlags \|= X86_PTE_RW;
3614	}
3615	else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
3616	{
3617	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3618	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3619	cErrors++;
3620	continue;
3621	}
3622
3623	/* flags */
3624	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3625	{
3626	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3627	{
3628	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
3629	{
3630	if (PteDst.n.u1Write)
3631	{
3632	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3633	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3634	cErrors++;
3635	continue;
3636	}
3637	fIgnoreFlags \|= X86_PTE_RW;
3638	}
3639	}
3640	else
3641	{
3642	if (PteDst.n.u1Present)
3643	{
3644	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3645	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3646	cErrors++;
3647	continue;
3648	}
3649	fIgnoreFlags \|= X86_PTE_P;
3650	}
3651	}
3652
3653	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3654	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3655	)
3656	{
3657	AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3658	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3659	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3660	cErrors++;
3661	continue;
3662	}
3663	} /* foreach PTE */
3664	}
3665	}
3666	/* not present */
3667
3668	} /* forearch PDE */
3669
3670	# ifdef DEBUG
3671	if (cErrors)
3672	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3673	# endif
3674
3675	#elif PGM_GST_TYPE == PGM_TYPE_PAE
3676	//# error not implemented
3677
3678
3679	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3680	//# error not implemented
3681
3682	/#else: guest real and protected mode /
3683	#endif
3684	return cErrors;
3685	}
3686	#endif /* VBOX_STRICT */
3687

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 8536

Download in other formats: