PGMAllBth.h@ 6927

Last change on this file since 6927 was 6927, checked in by vboxsync, 17 years ago

Converted MM_RAM_FLAGS_VIRTUAL_HANDLER, MM_RAM_FLAGS_VIRTUAL_WRITE
and MM_RAM_FLAGS_VIRTUAL_ALL into a two bit state variable in PGMPAGE.
I've checked this trice because, like last time, bugs may have odd
sideeffects and hide for a while before showing up. Hope I got this
right (unlike for phys).

Fixed a regression from the MM_RAM_FLAGS_PHYSICAL in the physical read/write code.

Property svn:eol-style set to native
Property svn:keywords set to Id

File size: 146.4 KB

Line
1	/* $Id: PGMAllBth.h 6927 2008-02-12 20:44:35Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 innotek GmbH
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*/
19
20	/*******************************************************************************
21	* Internal Functions *
22	*******************************************************************************/
23	__BEGIN_DECLS
24	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
25	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
26	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
27	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
28	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
29	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
30	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
31	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal);
32	#ifdef VBOX_STRICT
33	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
34	#endif
35	#ifdef PGMPOOL_WITH_USER_TRACKING
36	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
37	#endif
38	__END_DECLS
39
40
41	/**
42	* #PF Handler for raw-mode guest execution.
43	*
44	* @returns VBox status code (appropriate for trap handling and GC return).
45	* @param pVM VM Handle.
46	* @param uErr The trap error code.
47	* @param pRegFrame Trap register frame.
48	* @param pvFault The fault address.
49	*/
50	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
51	{
52	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
53
54	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
55	# error "32-bit guest mode is only implemented for 32-bit and PAE shadow modes."
56	# endif
57
58	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
59	/*
60	* Hide the instruction fetch trap indicator for now.
61	*/
62	/** @todo NXE will change this and we must fix NXE in the switcher too! */
63	if (uErr & X86_TRAP_PF_ID)
64	{
65	uErr &= ~X86_TRAP_PF_ID;
66	TRPMSetErrorCode(pVM, uErr);
67	}
68	# endif
69
70	/*
71	* Get PDs.
72	*/
73	int rc;
74	# if PGM_WITH_PAGING(PGM_GST_TYPE)
75	# if PGM_GST_TYPE == PGM_TYPE_32BIT
76	const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
77	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
78	# else /* PAE */
79	unsigned iPDSrc;
80	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc);
81	# endif
82	# else
83	PGSTPD pPDSrc = NULL;
84	const unsigned iPDSrc = 0;
85	# endif
86
87	const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
88	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
89	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
90	# else /* PAE */
91	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries. */
92	# endif
93
94	# if PGM_WITH_PAGING(PGM_GST_TYPE)
95	# ifdef PGM_SYNC_DIRTY_BIT
96	/*
97	* If we successfully correct the write protection fault due to dirty bit
98	* tracking, or this page fault is a genuine one, then return immediately.
99	*/
100	STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
101	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
102	STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
103	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
104	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
105	{
106	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
107	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
108	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
109	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
110	}
111	# endif
112
113	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
114	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
115
116	/*
117	* A common case is the not-present error caused by lazy page table syncing.
118	*
119	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
120	* so we can safely assume that the shadow PT is present when calling SyncPage later.
121	*
122	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
123	* of mapping conflict and defer to SyncCR3 in R3.
124	* (Again, we do NOT support access handlers for non-present guest pages.)
125	*
126	*/
127	# if PGM_WITH_PAGING(PGM_GST_TYPE)
128	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
129	# else
130	GSTPDE PdeSrc;
131	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
132	PdeSrc.n.u1Present = 1;
133	PdeSrc.n.u1Write = 1;
134	PdeSrc.n.u1Accessed = 1;
135	PdeSrc.n.u1User = 1;
136	# endif
137	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
138	&& !pPDDst->a[iPDDst].n.u1Present
139	&& PdeSrc.n.u1Present
140	)
141
142	{
143	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
144	STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
145	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
146	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
147	if (VBOX_SUCCESS(rc))
148	{
149	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
150	return rc;
151	}
152	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
153	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
154	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
155	return VINF_PGM_SYNC_CR3;
156	}
157
158	# if PGM_WITH_PAGING(PGM_GST_TYPE)
159	/*
160	* Check if this address is within any of our mappings.
161	*
162	* This is very fast and it's gonna save us a bit of effort below and prevent
163	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
164	* (BTW, it's impossible to have physical access handlers in a mapping.)
165	*/
166	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
167	{
168	STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
169	PPGMMAPPING pMapping = CTXALLSUFF(pVM->pgm.s.pMappings);
170	for ( ; pMapping; pMapping = CTXALLSUFF(pMapping->pNext))
171	{
172	if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
173	break;
174	if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
175	{
176	/*
177	* The first thing we check is if we've got an undetected conflict.
178	*/
179	if (!pVM->pgm.s.fMappingsFixed)
180	{
181	unsigned iPT = pMapping->cPTs;
182	while (iPT-- > 0)
183	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
184	{
185	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
186	Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
187	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
188	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
189	return VINF_PGM_SYNC_CR3;
190	}
191	}
192
193	/*
194	* Check if the fault address is in a virtual page access handler range.
195	*/
196	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->HyperVirtHandlers, pvFault);
197	if ( pCur
198	&& (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
199	&& uErr & X86_TRAP_PF_RW)
200	{
201	# ifdef IN_GC
202	STAM_PROFILE_START(&pCur->Stat, h);
203	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
204	STAM_PROFILE_STOP(&pCur->Stat, h);
205	# else
206	AssertFailed();
207	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
208	# endif
209	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
210	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
211	return rc;
212	}
213
214	/*
215	* Pretend we're not here and let the guest handle the trap.
216	*/
217	TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
218	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
219	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
220	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
221	return VINF_EM_RAW_GUEST_TRAP;
222	}
223	}
224	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
225	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
226	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
227
228	/*
229	* Check if this fault address is flagged for special treatment,
230	* which means we'll have to figure out the physical address and
231	* check flags associated with it.
232	*
233	* ASSUME that we can limit any special access handling to pages
234	* in page tables which the guest believes to be present.
235	*/
236	if (PdeSrc.n.u1Present)
237	{
238	RTGCPHYS GCPhys = ~0U;
239
240	# if PGM_WITH_PAGING(PGM_GST_TYPE)
241	uint32_t cr4 = CPUMGetGuestCR4(pVM);
242	if ( PdeSrc.b.u1Size
243	&& (cr4 & X86_CR4_PSE))
244	GCPhys = (PdeSrc.u & GST_PDE4M_PG_MASK)
245	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
246	else
247	{
248	PVBOXPT pPTSrc;
249	# ifdef IN_GC
250	rc = PGMGCDynMapGCPage(pVM, PdeSrc.u & GST_PDE_PG_MASK, (void **)&pPTSrc);
251	# else
252	pPTSrc = (PVBOXPT)MMPhysGCPhys2HCVirt(pVM, PdeSrc.u & GST_PDE_PG_MASK, sizeof(*pPTSrc));
253	if (pPTSrc == 0)
254	rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
255	# endif
256	if (VBOX_SUCCESS(rc))
257	{
258	unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> PAGE_SHIFT) & GST_PT_MASK;
259	if (pPTSrc->a[iPTESrc].n.u1Present)
260	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
261	}
262	}
263	# else
264	/* No paging so the fault address is the physical address */
265	GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
266	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
267
268	/*
269	* If we have a GC address we'll check if it has any flags set.
270	*/
271	if (GCPhys != ~0U)
272	{
273	STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
274
275	PPGMPAGE pPage;
276	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
277	if (VBOX_SUCCESS(rc))
278	{
279	if (PGM_PAGE_HAVE_ANY_HANDLERS(pPage))
280	{
281	if (PGM_PAGE_HAVE_ANY_PHYSICAL_HANDLERS(pPage))
282	{
283	/*
284	* Physical page access handler.
285	*/
286	const RTGCPHYS GCPhysFault = GCPhys \| ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
287	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
288	if (pCur)
289	{
290	# ifdef PGM_SYNC_N_PAGES
291	/*
292	* If the region is write protected and we got a page not present fault, then sync
293	* the pages. If the fault was caused by a read, then restart the instruction.
294	* In case of write access continue to the GC write handler.
295	*
296	* ASSUMES that there is only one handler per page or that they have similar write properties.
297	*/
298	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
299	&& !(uErr & X86_TRAP_PF_P))
300	{
301	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
302	if ( VBOX_FAILURE(rc)
303	\|\| !(uErr & X86_TRAP_PF_RW)
304	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
305	{
306	AssertRC(rc);
307	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
308	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
309	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
310	return rc;
311	}
312	}
313	# endif
314
315	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
316	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
317	("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
318
319	#if defined(IN_GC) \|\| defined(IN_RING0)
320	if (CTXALLSUFF(pCur->pfnHandler))
321	{
322	STAM_PROFILE_START(&pCur->Stat, h);
323	rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
324	STAM_PROFILE_STOP(&pCur->Stat, h);
325	}
326	else
327	#endif
328	rc = VINF_EM_RAW_EMULATE_INSTR;
329	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
330	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
331	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
332	return rc;
333	}
334	}
335	# if PGM_WITH_PAGING(PGM_GST_TYPE)
336	else
337	{
338	# ifdef PGM_SYNC_N_PAGES
339	/*
340	* If the region is write protected and we got a page not present fault, then sync
341	* the pages. If the fault was caused by a read, then restart the instruction.
342	* In case of write access continue to the GC write handler.
343	*/
344	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
345	&& !(uErr & X86_TRAP_PF_P))
346	{
347	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
348	if ( VBOX_FAILURE(rc)
349	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
350	\|\| !(uErr & X86_TRAP_PF_RW))
351	{
352	AssertRC(rc);
353	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
354	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
355	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
356	return rc;
357	}
358	}
359	# endif
360	/*
361	* Ok, it's an virtual page access handler.
362	*
363	* Since it's faster to search by address, we'll do that first
364	* and then retry by GCPhys if that fails.
365	*/
366	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
367	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
368	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
369	*/
370	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
371	if (pCur)
372	{
373	AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
374	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
375	\|\| !(uErr & X86_TRAP_PF_P)
376	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
377	("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
378
379	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
380	&& ( uErr & X86_TRAP_PF_RW
381	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
382	{
383	# ifdef IN_GC
384	STAM_PROFILE_START(&pCur->Stat, h);
385	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
386	STAM_PROFILE_STOP(&pCur->Stat, h);
387	# else
388	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
389	# endif
390	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
391	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
392	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
393	return rc;
394	}
395	/* Unhandled part of a monitored page */
396	}
397	else
398	{
399	/* Check by physical address. */
400	PPGMVIRTHANDLER pCur;
401	unsigned iPage;
402	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
403	&pCur, &iPage);
404	Assert(VBOX_SUCCESS(rc) \|\| !pCur);
405	if ( pCur
406	&& ( uErr & X86_TRAP_PF_RW
407	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
408	{
409	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
410	# ifdef IN_GC
411	RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->GCPtr & PAGE_OFFSET_MASK);
412	Assert(off < pCur->cb);
413	STAM_PROFILE_START(&pCur->Stat, h);
414	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
415	STAM_PROFILE_STOP(&pCur->Stat, h);
416	# else
417	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
418	# endif
419	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
420	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
421	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
422	return rc;
423	}
424	}
425	}
426	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
427
428	/*
429	* There is a handled area of the page, but this fault doesn't belong to it.
430	* We must emulate the instruction.
431	*
432	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
433	* we first check if this was a page-not-present fault for a page with only
434	* write access handlers. Restart the instruction if it wasn't a write access.
435	*/
436	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
437
438	if ( !PGM_PAGE_HAVE_ACTIVE_ALL_HANDLERS(pPage)
439	&& !(uErr & X86_TRAP_PF_P))
440	{
441	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
442	if ( VBOX_FAILURE(rc)
443	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
444	\|\| !(uErr & X86_TRAP_PF_RW))
445	{
446	AssertRC(rc);
447	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
448	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
449	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
450	return rc;
451	}
452	}
453
454	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
455	* It's writing to an unhandled part of the LDT page several million times.
456	*/
457	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
458	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%RHp%s%s\n",
459	rc, pPage->HCPhys,
460	PGM_PAGE_HAVE_ANY_PHYSICAL_HANDLERS(pPage) ? " phys" : "",
461	PGM_PAGE_HAVE_ANY_VIRTUAL_HANDLERS(pPage) ? " virt" : ""));
462	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
463	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
464	return rc;
465	} /* if any kind of handler */
466
467	# if PGM_WITH_PAGING(PGM_GST_TYPE)
468	if (uErr & X86_TRAP_PF_P)
469	{
470	/*
471	* The page isn't marked, but it might still be monitored by a virtual page access handler.
472	* (ASSUMES no temporary disabling of virtual handlers.)
473	*/
474	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
475	* we should correct both the shadow page table and physical memory flags, and not only check for
476	* accesses within the handler region but for access to pages with virtual handlers. */
477	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
478	if (pCur)
479	{
480	AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
481	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
482	\|\| !(uErr & X86_TRAP_PF_P)
483	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
484	("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
485
486	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
487	&& ( uErr & X86_TRAP_PF_RW
488	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
489	{
490	# ifdef IN_GC
491	STAM_PROFILE_START(&pCur->Stat, h);
492	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
493	STAM_PROFILE_STOP(&pCur->Stat, h);
494	# else
495	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
496	# endif
497	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
498	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
499	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
500	return rc;
501	}
502	}
503	}
504	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
505	}
506	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
507
508	# ifdef PGM_OUT_OF_SYNC_IN_GC
509	/*
510	* We are here only if page is present in Guest page tables and trap is not handled
511	* by our handlers.
512	* Check it for page out-of-sync situation.
513	*/
514	STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
515
516	if (!(uErr & X86_TRAP_PF_P))
517	{
518	/*
519	* Page is not present in our page tables.
520	* Try to sync it!
521	* BTW, fPageShw is invalid in this branch!
522	*/
523	if (uErr & X86_TRAP_PF_US)
524	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
525	else /* supervisor */
526	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
527
528	# if defined(LOG_ENABLED) && !defined(IN_RING0)
529	RTGCPHYS GCPhys;
530	uint64_t fPageGst;
531	PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
532	Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
533	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
534	# endif /* LOG_ENABLED */
535
536	# if PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0)
537	if (CPUMGetGuestCPL(pVM, pRegFrame) == 0)
538	{
539	uint64_t fPageGst;
540	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
541	if ( VBOX_SUCCESS(rc)
542	&& !(fPageGst & X86_PTE_US))
543	{
544	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
545	if ( pvFault == (RTGCPTR)pRegFrame->eip
546	\|\| (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
547	# ifdef CSAM_DETECT_NEW_CODE_PAGES
548	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
549	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
550	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
551	)
552	{
553	LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
554	rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
555	if (rc != VINF_SUCCESS)
556	{
557	/*
558	* CSAM needs to perform a job in ring 3.
559	*
560	* Sync the page before going to the host context; otherwise we'll end up in a loop if
561	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
562	*/
563	LogFlow(("CSAM ring 3 job\n"));
564	int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
565	AssertRC(rc2);
566
567	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
568	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
569	return rc;
570	}
571	}
572	# ifdef CSAM_DETECT_NEW_CODE_PAGES
573	else
574	if ( uErr == X86_TRAP_PF_RW
575	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
576	&& pRegFrame->ecx < 0x10000
577	)
578	{
579	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
580	* to detect loading of new code pages.
581	*/
582
583	/*
584	* Decode the instruction.
585	*/
586	RTGCPTR PC;
587	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
588	if (rc == VINF_SUCCESS)
589	{
590	DISCPUSTATE Cpu;
591	uint32_t cbOp;
592	rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
593
594	/* For now we'll restrict this to rep movsw/d instructions */
595	if ( rc == VINF_SUCCESS
596	&& Cpu.pCurInstr->opcode == OP_MOVSWD
597	&& (Cpu.prefix & PREFIX_REP))
598	{
599	CSAMMarkPossibleCodePage(pVM, pvFault);
600	}
601	}
602	}
603	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
604
605	/*
606	* Mark this page as safe.
607	*/
608	/** @todo not correct for pages that contain both code and data!! */
609	Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
610	CSAMMarkPage(pVM, pvFault, true);
611	}
612	}
613	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
614	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
615	if (VBOX_SUCCESS(rc))
616	{
617	/* The page was successfully synced, return to the guest. */
618	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
619	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
620	return VINF_SUCCESS;
621	}
622	}
623	else
624	{
625	/*
626	* A side effect of not flushing global PDEs are out of sync pages due
627	* to physical monitored regions, that are no longer valid.
628	* Assume for now it only applies to the read/write flag
629	*/
630	if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
631	{
632	if (uErr & X86_TRAP_PF_US)
633	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
634	else /* supervisor */
635	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
636
637
638	/*
639	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
640	*/
641	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
642	if (VBOX_SUCCESS(rc))
643	{
644	/*
645	* Page was successfully synced, return to guest.
646	*/
647	# ifdef VBOX_STRICT
648	RTGCPHYS GCPhys;
649	uint64_t fPageGst;
650	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
651	Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
652	LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
653
654	uint64_t fPageShw;
655	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
656	Assert(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW);
657	# endif /* VBOX_STRICT */
658	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
659	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
660	return VINF_SUCCESS;
661	}
662	}
663
664	# if PGM_WITH_PAGING(PGM_GST_TYPE)
665	# ifdef VBOX_STRICT
666	/*
667	* Check for VMM page flags vs. Guest page flags consistency.
668	* Currently only for debug purposes.
669	*/
670	if (VBOX_SUCCESS(rc))
671	{
672	/* Get guest page flags. */
673	uint64_t fPageGst;
674	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
675	if (VBOX_SUCCESS(rc))
676	{
677	uint64_t fPageShw;
678	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
679
680	/*
681	* Compare page flags.
682	* Note: we have AVL, A, D bits desynched.
683	*/
684	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
685	("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
686	}
687	else
688	AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
689	}
690	else
691	AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
692	# endif /* VBOX_STRICT */
693	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
694	}
695	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
696	# endif /* PGM_OUT_OF_SYNC_IN_GC */
697	}
698	else
699	{
700	/*
701	* Page not present in Guest OS or invalid page table address.
702	* This is potential virtual page access handler food.
703	*
704	* For the present we'll say that our access handlers don't
705	* work for this case - we've already discarded the page table
706	* not present case which is identical to this.
707	*
708	* When we perchance find we need this, we will probably have AVL
709	* trees (offset based) to operate on and we can measure their speed
710	* agains mapping a page table and probably rearrange this handling
711	* a bit. (Like, searching virtual ranges before checking the
712	* physical address.)
713	*/
714	}
715	}
716
717
718	# if PGM_WITH_PAGING(PGM_GST_TYPE)
719	/*
720	* Conclusion, this is a guest trap.
721	*/
722	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
723	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
724	return VINF_EM_RAW_GUEST_TRAP;
725	# else
726	/* present, but not a monitored page; perhaps the guest is probing physical memory */
727	return VINF_EM_RAW_EMULATE_INSTR;
728	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
729
730
731	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
732
733	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
734	return VERR_INTERNAL_ERROR;
735	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
736	}
737
738
739	/**
740	* Emulation of the invlpg instruction.
741	*
742	*
743	* @returns VBox status code.
744	*
745	* @param pVM VM handle.
746	* @param GCPtrPage Page to invalidate.
747	*
748	* @remark ASSUMES that the guest is updating before invalidating. This order
749	* isn't required by the CPU, so this is speculative and could cause
750	* trouble.
751	*
752	* @todo Flush page or page directory only if necessary!
753	* @todo Add a #define for simply invalidating the page.
754	*/
755	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
756	{
757	#if PGM_GST_TYPE == PGM_TYPE_32BIT
758
759	LogFlow(("InvalidatePage %x\n", GCPtrPage));
760	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
761	/*
762	* Get the shadow PD entry and skip out if this PD isn't present.
763	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
764	*/
765	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
766	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
767	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
768	# else
769	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
770	# endif
771	const SHWPDE PdeDst = *pPdeDst;
772	if (!PdeDst.n.u1Present)
773	{
774	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
775	return VINF_SUCCESS;
776	}
777
778	/*
779	* Get the guest PD entry and calc big page.
780	*/
781	# if PGM_GST_TYPE == PGM_TYPE_32BIT
782	PX86PD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
783	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
784	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
785	# else /* PAE */
786	unsigned iPDSrc;
787	PX86PD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
788
789	GSTPDE PdeSrc;;
790	PdeSrc.u = pgmGstGetPaePDE(&pVM->pgm.s, GCPtrPage);
791	# endif
792
793	const uint32_t cr4 = CPUMGetGuestCR4(pVM);
794	const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
795
796	# ifdef IN_RING3
797	/*
798	* If a CR3 Sync is pending we may ignore the invalidate page operation
799	* depending on the kind of sync and if it's a global page or not.
800	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
801	*/
802	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
803	if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
804	\|\| ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
805	&& fIsBigPage
806	&& PdeSrc.b.u1Global
807	&& (cr4 & X86_CR4_PGE)
808	)
809	)
810	# else
811	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
812	# endif
813	{
814	STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
815	return VINF_SUCCESS;
816	}
817	# endif /* IN_RING3 */
818
819
820	/*
821	* Deal with the Guest PDE.
822	*/
823	int rc = VINF_SUCCESS;
824	if (PdeSrc.n.u1Present)
825	{
826	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
827	{
828	/*
829	* Conflict - Let SyncPT deal with it to avoid duplicate code.
830	*/
831	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
832	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
833	}
834	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
835	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
836	{
837	/*
838	* Mark not present so we can resync the PDE when it's used.
839	*/
840	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
841	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
842	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
843	pPdeDst->u = 0;
844	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
845	PGM_INVL_GUEST_TLBS();
846	}
847	# ifdef PGM_SYNC_ACCESSED_BIT
848	else if (!PdeSrc.n.u1Accessed)
849	{
850	/*
851	* Mark not present so we can set the accessed bit.
852	*/
853	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
854	pPdeDst->u = 0;
855	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
856	PGM_INVL_GUEST_TLBS();
857	}
858	# endif
859	else if (!fIsBigPage)
860	{
861	/*
862	* 4KB - page.
863	*/
864	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
865	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
866	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
867	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
868	# endif
869	if (pShwPage->GCPhys == GCPhys)
870	{
871	#if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
872	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
873	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
874	if (pPT->a[iPTEDst].n.u1Present)
875	{
876	# ifdef PGMPOOL_WITH_USER_TRACKING
877	/* This is very unlikely with caching/monitoring enabled. */
878	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
879	# endif
880	pPT->a[iPTEDst].u = 0;
881	}
882	#else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
883	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
884	if (VBOX_SUCCESS(rc))
885	rc = VINF_SUCCESS;
886	#endif
887	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
888	PGM_INVL_PG(GCPtrPage);
889	}
890	else
891	{
892	/*
893	* The page table address changed.
894	*/
895	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
896	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
897	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
898	pPdeDst->u = 0;
899	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
900	PGM_INVL_GUEST_TLBS();
901	}
902	}
903	else
904	{
905	/*
906	* 4MB - page.
907	*/
908	/* Before freeing the page, check if anything really changed. */
909	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
910	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
911	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
912	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
913	# endif
914	if ( pShwPage->GCPhys == GCPhys
915	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
916	{
917	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
918	/** @todo PAT */
919	# ifdef PGM_SYNC_DIRTY_BIT
920	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
921	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
922	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
923	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
924	# else
925	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
926	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD)))
927	# endif
928	{
929	LogFlow(("Skipping flush for big page containing %VGv (PD=%X)-> nothing has changed!\n", GCPtrPage, iPDSrc));
930	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
931	return VINF_SUCCESS;
932	}
933	}
934
935	/*
936	* Ok, the page table is present and it's been changed in the guest.
937	* If we're in host context, we'll just mark it as not present taking the lazy approach.
938	* We could do this for some flushes in GC too, but we need an algorithm for
939	* deciding which 4MB pages containing code likely to be executed very soon.
940	*/
941	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
942	pPdeDst->u = 0;
943	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
944	DUMP_PDE_BIG("PGMInvalidatePage", iPDSrc, PdeSrc);
945	PGM_INVL_BIG_PG(GCPtrPage);
946	}
947	}
948	else
949	{
950	/*
951	* Page directory is not present, mark shadow PDE not present.
952	*/
953	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
954	{
955	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
956	pPdeDst->u = 0;
957	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
958	PGM_INVL_PG(GCPtrPage);
959	}
960	else
961	{
962	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
963	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
964	}
965	}
966
967	return rc;
968
969	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
970	# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
971	# endif
972	return VINF_SUCCESS;
973
974	#elif PGM_GST_TYPE == PGM_TYPE_PAE
975	# if PGM_SHW_TYPE == PGM_TYPE_PAE
976	//# error not implemented
977	return VERR_INTERNAL_ERROR;
978
979	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
980	# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
981	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
982
983	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
984	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
985	//# error not implemented
986	return VERR_INTERNAL_ERROR;
987
988	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
989	# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
990	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
991
992	#else /* guest real and protected mode */
993	/* There's no such thing when paging is disabled. */
994	return VINF_SUCCESS;
995	#endif
996	}
997
998
999	#ifdef PGMPOOL_WITH_USER_TRACKING
1000	/**
1001	* Update the tracking of shadowed pages.
1002	*
1003	* @param pVM The VM handle.
1004	* @param pShwPage The shadow page.
1005	* @param HCPhys The physical page we is being dereferenced.
1006	*/
1007	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1008	{
1009	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1010	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1011	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1012
1013	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1014	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1015	* 2. write protect all shadowed pages. I.e. implement caching.
1016	*/
1017	/*
1018	* Find the guest address.
1019	*/
1020	for (PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
1021	pRam;
1022	pRam = CTXALLSUFF(pRam->pNext))
1023	{
1024	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1025	while (iPage-- > 0)
1026	{
1027	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1028	{
1029	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1030	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1031	pShwPage->cPresent--;
1032	pPool->cPresent--;
1033	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1034	return;
1035	}
1036	}
1037	}
1038
1039	for (;;)
1040	AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1041	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1042	pShwPage->cPresent--;
1043	pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1044	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1045	}
1046
1047
1048	/**
1049	* Update the tracking of shadowed pages.
1050	*
1051	* @param pVM The VM handle.
1052	* @param pShwPage The shadow page.
1053	* @param u16 The top 16-bit of the pPage->HCPhys.
1054	* @param pPage Pointer to the guest page. this will be modified.
1055	* @param iPTDst The index into the shadow table.
1056	*/
1057	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1058	{
1059	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1060	/*
1061	* We're making certain assumptions about the placement of cRef and idx.
1062	*/
1063	Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1064	Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1065
1066	/*
1067	* Just deal with the simple first time here.
1068	*/
1069	if (!u16)
1070	{
1071	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1072	u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) \| pShwPage->idx;
1073	}
1074	else
1075	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1076
1077	/* write back, trying to be clever... */
1078	Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%VHp->%VHp iPTDst=%#x\n",
1079	u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) \| ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1080	((uint16_t )&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1081	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1082
1083	/* update statistics. */
1084	pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1085	pShwPage->cPresent++;
1086	if (pShwPage->iFirstPresent > iPTDst)
1087	pShwPage->iFirstPresent = iPTDst;
1088	}
1089	#endif /* PGMPOOL_WITH_USER_TRACKING */
1090
1091
1092	/**
1093	* Creates a 4K shadow page for a guest page.
1094	*
1095	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1096	* physical address. The PdeSrc argument only the flags are used. No page structured
1097	* will be mapped in this function.
1098	*
1099	* @param pVM VM handle.
1100	* @param pPteDst Destination page table entry.
1101	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1102	* Can safely assume that only the flags are being used.
1103	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1104	* @param pShwPage Pointer to the shadow page.
1105	* @param iPTDst The index into the shadow table.
1106	*
1107	* @remark Not used for 2/4MB pages!
1108	*/
1109	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, VBOXPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1110	{
1111	if (PteSrc.n.u1Present)
1112	{
1113	/*
1114	* Find the ram range.
1115	*/
1116	PPGMPAGE pPage;
1117	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1118	if (VBOX_SUCCESS(rc))
1119	{
1120	/** @todo investiage PWT, PCD and PAT. */
1121	/*
1122	* Make page table entry.
1123	*/
1124	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1125	SHWPTE PteDst;
1126	if (PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPage))
1127	{
1128	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1129	if (!PGM_PAGE_HAVE_ACTIVE_ALL_HANDLERS(pPage))
1130	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1131	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1132	else
1133	{
1134	LogFlow(("SyncPageWorker: monitored page (%VGp) -> mark not present\n", HCPhys));
1135	PteDst.u = 0;
1136	}
1137	/** @todo count these two kinds. */
1138	}
1139	else
1140	{
1141	#ifdef PGM_SYNC_DIRTY_BIT
1142	# ifdef PGM_SYNC_ACCESSED_BIT
1143	/*
1144	* If the page or page directory entry is not marked accessed,
1145	* we mark the page not present.
1146	*/
1147	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1148	{
1149	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1150	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1151	PteDst.u = 0;
1152	}
1153	else
1154	# endif
1155	/*
1156	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1157	* when the page is modified.
1158	*/
1159	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1160	{
1161	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1162	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1163	\| (HCPhys & X86_PTE_PAE_PG_MASK)
1164	\| PGM_PTFLAGS_TRACK_DIRTY;
1165	}
1166	else
1167	{
1168	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1169	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1170	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1171	}
1172	#endif
1173	}
1174
1175	#ifdef PGMPOOL_WITH_USER_TRACKING
1176	/*
1177	* Keep user track up to date.
1178	*/
1179	if (PteDst.n.u1Present)
1180	{
1181	if (!pPteDst->n.u1Present)
1182	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1183	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1184	{
1185	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1186	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1187	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1188	}
1189	}
1190	else if (pPteDst->n.u1Present)
1191	{
1192	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1193	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1194	}
1195	#endif /* PGMPOOL_WITH_USER_TRACKING */
1196
1197	/*
1198	* Update statistics and commit the entry.
1199	*/
1200	if (!PteSrc.n.u1Global)
1201	pShwPage->fSeenNonGlobal = true;
1202	*pPteDst = PteDst;
1203	}
1204	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1205	/** @todo count these. */
1206	}
1207	else
1208	{
1209	/*
1210	* Page not-present.
1211	*/
1212	LogFlow(("SyncPageWorker: page not present in Pte\n"));
1213	#ifdef PGMPOOL_WITH_USER_TRACKING
1214	/* Keep user track up to date. */
1215	if (pPteDst->n.u1Present)
1216	{
1217	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1218	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1219	}
1220	#endif /* PGMPOOL_WITH_USER_TRACKING */
1221	pPteDst->u = 0;
1222	/** @todo count these. */
1223	}
1224	}
1225
1226
1227	/**
1228	* Syncs a guest OS page.
1229	*
1230	* There are no conflicts at this point, neither is there any need for
1231	* page table allocations.
1232	*
1233	* @returns VBox status code.
1234	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1235	* @param pVM VM handle.
1236	* @param PdeSrc Page directory entry of the guest.
1237	* @param GCPtrPage Guest context page address.
1238	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1239	* @param uErr Fault error (X86_TRAP_PF_*).
1240	*/
1241	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1242	{
1243	LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1244
1245	#if PGM_GST_TYPE == PGM_TYPE_32BIT
1246
1247	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1248	# error "Invalid shadow mode for 32-bit guest mode!"
1249	# endif
1250
1251	/*
1252	* Assert preconditions.
1253	*/
1254	# if GC_ARCH_BITS != 32
1255	Assert(GCPtrPage < _4G); //???
1256	# endif
1257	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> X86_PD_SHIFT) & GST_PD_MASK]);
1258	Assert(PdeSrc.n.u1Present);
1259	Assert(cPages);
1260
1261	/*
1262	* Get the shadow PDE, find the shadow page table in the pool.
1263	*/
1264	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1265	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1266	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1267	# else /* PAE */
1268	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1269	# endif
1270	Assert(PdeDst.n.u1Present);
1271	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1272
1273	/*
1274	* Check that the page is present and that the shadow PDE isn't out of sync.
1275	*/
1276	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1277	RTGCPHYS GCPhys;
1278	if (!fBigPage)
1279	{
1280	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1281	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1282	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1283	# endif
1284	}
1285	else
1286	{
1287	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1288	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1289	GCPhys \|= GCPtrPage & X86_PAGE_2M_SIZE;
1290	# endif
1291	}
1292	if ( pShwPage->GCPhys == GCPhys
1293	&& PdeSrc.n.u1Present
1294	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1295	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1296	)
1297	{
1298	# ifdef PGM_SYNC_ACCESSED_BIT
1299	/*
1300	* Check that the PDE is marked accessed already.
1301	* Since we set the accessed bit before getting here on a #PF, this
1302	* check is only meant for dealing with non-#PF'ing paths.
1303	*/
1304	if (PdeSrc.n.u1Accessed)
1305	# endif
1306	{
1307	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1308	if (!fBigPage)
1309	{
1310	/*
1311	* 4KB Page - Map the guest page table.
1312	*/
1313	PVBOXPT pPTSrc;
1314	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1315	if (VBOX_SUCCESS(rc))
1316	{
1317	# ifdef PGM_SYNC_N_PAGES
1318	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1319	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1320	{
1321	/*
1322	* This code path is currently only taken when the caller is PGMTrap0eHandler
1323	* for non-present pages!
1324	*
1325	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1326	* deal with locality.
1327	*/
1328	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1329	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1330	const unsigned offPTSrc = 0;
1331	# else
1332	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1333	# endif
1334	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1335	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1336	iPTDst = 0;
1337	else
1338	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1339	for (; iPTDst < iPTDstEnd; iPTDst++)
1340	{
1341	if (!pPTDst->a[iPTDst].n.u1Present)
1342	{
1343	VBOXPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1344	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1345	NOREF(GCPtrCurPage);
1346	#ifndef IN_RING0
1347	/*
1348	* Assuming kernel code will be marked as supervisor - and not as user level
1349	* and executed using a conforming code selector - And marked as readonly.
1350	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1351	*/
1352	PPGMPAGE pPage;
1353	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1354	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1355	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1356	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1357	&& PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPage))
1358	)
1359	#endif /* else: CSAM not active */
1360	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1361	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1362	GCPtrCurPage, PteSrc.n.u1Present,
1363	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1364	PteSrc.n.u1User & PdeSrc.n.u1User,
1365	(uint64_t)PteSrc.u,
1366	(uint64_t)pPTDst->a[iPTDst].u,
1367	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1368	}
1369	}
1370	}
1371	else
1372	# endif /* PGM_SYNC_N_PAGES */
1373	{
1374	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1375	VBOXPTE PteSrc = pPTSrc->a[iPTSrc];
1376	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1377	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1378	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1379	GCPtrPage, PteSrc.n.u1Present,
1380	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1381	PteSrc.n.u1User & PdeSrc.n.u1User,
1382	(uint64_t)PteSrc.u,
1383	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1384	}
1385	}
1386	else /* MMIO or invalid page: emulated in #PF handler. */
1387	{
1388	LogFlow(("PGM_GCPHYS_2_PTR %VGp failed with %Vrc\n", GCPhys, rc));
1389	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1390	}
1391	}
1392	else
1393	{
1394	/*
1395	* 4/2MB page - lazy syncing shadow 4K pages.
1396	* (There are many causes of getting here, it's no longer only CSAM.)
1397	*/
1398	/* Calculate the GC physical address of this 4KB shadow page. */
1399	RTGCPHYS GCPhys = (PdeSrc.u & X86_PDE4M_PAE_PG_MASK) \| ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1400	/* Find ram range. */
1401	PPGMPAGE pPage;
1402	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1403	if (VBOX_SUCCESS(rc))
1404	{
1405	/*
1406	* Make shadow PTE entry.
1407	*/
1408	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1409	SHWPTE PteDst;
1410	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1411	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1412	if (PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPage))
1413	{
1414	if (!PGM_PAGE_HAVE_ACTIVE_ALL_HANDLERS(pPage))
1415	PteDst.n.u1Write = 0;
1416	else
1417	PteDst.u = 0;
1418	}
1419	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1420	# ifdef PGMPOOL_WITH_USER_TRACKING
1421	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1422	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1423	# endif
1424	pPTDst->a[iPTDst] = PteDst;
1425
1426
1427	# ifdef PGM_SYNC_DIRTY_BIT
1428	/*
1429	* If the page is not flagged as dirty and is writable, then make it read-only
1430	* at PD level, so we can set the dirty bit when the page is modified.
1431	*
1432	* ASSUMES that page access handlers are implemented on page table entry level.
1433	* Thus we will first catch the dirty access and set PDE.D and restart. If
1434	* there is an access handler, we'll trap again and let it work on the problem.
1435	*/
1436	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1437	* As for invlpg, it simply frees the whole shadow PT.
1438	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1439	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1440	{
1441	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1442	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1443	PdeDst.n.u1Write = 0;
1444	}
1445	else
1446	{
1447	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1448	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1449	}
1450	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1451	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1452	# else /* PAE */
1453	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1454	# endif
1455	# endif /* PGM_SYNC_DIRTY_BIT */
1456	Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1457	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1458	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1459	}
1460	else
1461	LogFlow(("PGM_GCPHYS_2_PTR %VGp (big) failed with %Vrc\n", GCPhys, rc));
1462	}
1463	return VINF_SUCCESS;
1464	}
1465	# ifdef PGM_SYNC_ACCESSED_BIT
1466	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1467	#endif
1468	}
1469	else
1470	{
1471	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1472	Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1473	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1474	}
1475
1476	/*
1477	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1478	* Yea, I'm lazy.
1479	*/
1480	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1481	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1482	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1483	# else /* PAE */
1484	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1485	# endif
1486	PGM_INVL_GUEST_TLBS();
1487	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1488
1489	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
1490
1491	# ifdef PGM_SYNC_N_PAGES
1492	/*
1493	* Get the shadow PDE, find the shadow page table in the pool.
1494	*/
1495	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1496	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1497	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1498	# else /* PAE */
1499	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1500	# endif
1501	Assert(PdeDst.n.u1Present);
1502	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1503	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1504
1505	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1506	const unsigned offPTSrc = 0;
1507	# else
1508	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1509	# endif
1510
1511	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1512	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1513	{
1514	/*
1515	* This code path is currently only taken when the caller is PGMTrap0eHandler
1516	* for non-present pages!
1517	*
1518	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1519	* deal with locality.
1520	*/
1521	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1522	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1523	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1524	iPTDst = 0;
1525	else
1526	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1527	for (; iPTDst < iPTDstEnd; iPTDst++)
1528	{
1529	if (!pPTDst->a[iPTDst].n.u1Present)
1530	{
1531	VBOXPTE PteSrc;
1532
1533	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1534
1535	/* Fake the page table entry */
1536	PteSrc.u = GCPtrCurPage;
1537	PteSrc.n.u1Present = 1;
1538	PteSrc.n.u1Dirty = 1;
1539	PteSrc.n.u1Accessed = 1;
1540	PteSrc.n.u1Write = 1;
1541	PteSrc.n.u1User = 1;
1542
1543	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1544
1545	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1546	GCPtrCurPage, PteSrc.n.u1Present,
1547	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1548	PteSrc.n.u1User & PdeSrc.n.u1User,
1549	(uint64_t)PteSrc.u,
1550	(uint64_t)pPTDst->a[iPTDst].u,
1551	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1552	}
1553	}
1554	}
1555	else
1556	# endif /* PGM_SYNC_N_PAGES */
1557	{
1558	VBOXPTE PteSrc;
1559	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1560	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1561
1562	/* Fake the page table entry */
1563	PteSrc.u = GCPtrCurPage;
1564	PteSrc.n.u1Present = 1;
1565	PteSrc.n.u1Dirty = 1;
1566	PteSrc.n.u1Accessed = 1;
1567	PteSrc.n.u1Write = 1;
1568	PteSrc.n.u1User = 1;
1569	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1570
1571	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1572	GCPtrPage, PteSrc.n.u1Present,
1573	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1574	PteSrc.n.u1User & PdeSrc.n.u1User,
1575	(uint64_t)PteSrc.u,
1576	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1577	}
1578	return VINF_SUCCESS;
1579
1580	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1581	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1582	return VERR_INTERNAL_ERROR;
1583	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1584	}
1585
1586
1587
1588	#if PGM_WITH_PAGING(PGM_GST_TYPE)
1589
1590	# ifdef PGM_SYNC_DIRTY_BIT
1591
1592	/**
1593	* Investigate page fault and handle write protection page faults caused by
1594	* dirty bit tracking.
1595	*
1596	* @returns VBox status code.
1597	* @param pVM VM handle.
1598	* @param uErr Page fault error code.
1599	* @param pPdeDst Shadow page directory entry.
1600	* @param pPdeSrc Guest page directory entry.
1601	* @param GCPtrPage Guest context page address.
1602	*/
1603	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1604	{
1605	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1606	LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1607
1608	/*
1609	* Real page fault?
1610	*/
1611	if ( (uErr & X86_TRAP_PF_RSVD)
1612	\|\| !pPdeSrc->n.u1Present
1613	#if PGM_GST_TYPE == PGM_TYPE_PAE
1614	\|\| ((uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
1615	#endif
1616	\|\| ((uErr & X86_TRAP_PF_RW) && !pPdeSrc->n.u1Write)
1617	\|\| ((uErr & X86_TRAP_PF_US) && !pPdeSrc->n.u1User) )
1618	{
1619	# ifdef IN_GC
1620	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1621	# endif
1622	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1623	LogFlow(("CheckPageFault: real page fault at %VGv (1)\n", GCPtrPage));
1624
1625	if (pPdeSrc->n.u1Present)
1626	{
1627	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1628	* See the 2nd case below as well.
1629	*/
1630	if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1631	{
1632	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1633	}
1634	else
1635	{
1636	/*
1637	* Map the guest page table.
1638	*/
1639	PGSTPT pPTSrc;
1640	int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1641	if (VBOX_SUCCESS(rc))
1642	{
1643	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & GST_PT_MASK];
1644	const GSTPTE PteSrc = *pPteSrc;
1645	if (pPteSrc->n.u1Present)
1646	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1647	}
1648	AssertRC(rc);
1649	}
1650	}
1651	return VINF_EM_RAW_GUEST_TRAP;
1652	}
1653
1654	/*
1655	* First check the easy case where the page directory has been marked read-only to track
1656	* the dirty bit of an emulated BIG page
1657	*/
1658	if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1659	{
1660	/* Mark guest page directory as accessed */
1661	pPdeSrc->b.u1Accessed = 1;
1662
1663	/*
1664	* Only write protection page faults are relevant here.
1665	*/
1666	if (uErr & X86_TRAP_PF_RW)
1667	{
1668	/* Mark guest page directory as dirty (BIG page only). */
1669	pPdeSrc->b.u1Dirty = 1;
1670
1671	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1672	{
1673	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1674
1675	Assert(pPdeSrc->b.u1Write);
1676
1677	pPdeDst->n.u1Write = 1;
1678	pPdeDst->n.u1Accessed = 1;
1679	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1680	PGM_INVL_BIG_PG(GCPtrPage);
1681	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1682	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1683	}
1684	}
1685	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1686	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1687	}
1688	/* else: 4KB page table */
1689
1690	/*
1691	* Map the guest page table.
1692	*/
1693	PGSTPT pPTSrc;
1694	int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1695	if (VBOX_SUCCESS(rc))
1696	{
1697	/*
1698	* Real page fault?
1699	*/
1700	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & GST_PT_MASK];
1701	const GSTPTE PteSrc = *pPteSrc;
1702	if ( !PteSrc.n.u1Present
1703	#if PGM_GST_TYPE == PGM_TYPE_PAE
1704	\|\| ((uErr & X86_TRAP_PF_ID) && !PteSrc.n.u1NoExecute)
1705	#endif
1706	\|\| ((uErr & X86_TRAP_PF_RW) && !PteSrc.n.u1Write)
1707	\|\| ((uErr & X86_TRAP_PF_US) && !PteSrc.n.u1User)
1708	)
1709	{
1710	# ifdef IN_GC
1711	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1712	# endif
1713	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1714	LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1715
1716	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1717	* See the 2nd case above as well.
1718	*/
1719	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1720	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1721
1722	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1723	return VINF_EM_RAW_GUEST_TRAP;
1724	}
1725	LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1726
1727	/*
1728	* Set the accessed bits in the page directory and the page table.
1729	*/
1730	pPdeSrc->n.u1Accessed = 1;
1731	pPteSrc->n.u1Accessed = 1;
1732
1733	/*
1734	* Only write protection page faults are relevant here.
1735	*/
1736	if (uErr & X86_TRAP_PF_RW)
1737	{
1738	/* Write access, so mark guest entry as dirty. */
1739	# if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1740	if (!pPteSrc->n.u1Dirty)
1741	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1742	else
1743	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1744	# endif
1745	pPteSrc->n.u1Dirty = 1;
1746
1747	if (pPdeDst->n.u1Present)
1748	{
1749	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1750	* Our individual shadow handlers will provide more information and force a fatal exit.
1751	*/
1752	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1753	{
1754	LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1755	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1756	return VINF_SUCCESS;
1757	}
1758
1759	/*
1760	* Map shadow page table.
1761	*/
1762	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1763	if (pShwPage)
1764	{
1765	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1766	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1767	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1768	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1769	{
1770	LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1771	# ifdef VBOX_STRICT
1772	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
1773	if (pPage)
1774	AssertMsg(!PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPage),
1775	("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1776	# endif
1777	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1778
1779	Assert(pPteSrc->n.u1Write);
1780
1781	pPteDst->n.u1Write = 1;
1782	pPteDst->n.u1Dirty = 1;
1783	pPteDst->n.u1Accessed = 1;
1784	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1785	PGM_INVL_PG(GCPtrPage);
1786
1787	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1788	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1789	}
1790	}
1791	else
1792	AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
1793	}
1794	}
1795	/** @todo Optimize accessed bit emulation? */
1796	# ifdef VBOX_STRICT
1797	/*
1798	* Sanity check.
1799	*/
1800	else if ( !pPteSrc->n.u1Dirty
1801	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
1802	&& pPdeDst->n.u1Present)
1803	{
1804	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1805	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1806	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1807	if ( pPteDst->n.u1Present
1808	&& pPteDst->n.u1Write)
1809	LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
1810	}
1811	# endif /* VBOX_STRICT */
1812	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1813	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1814	}
1815	AssertRC(rc);
1816	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1817	return rc;
1818	}
1819
1820	# endif
1821
1822	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
1823
1824
1825	/**
1826	* Sync a shadow page table.
1827	*
1828	* The shadow page table is not present. This includes the case where
1829	* there is a conflict with a mapping.
1830	*
1831	* @returns VBox status code.
1832	* @param pVM VM handle.
1833	* @param iPD Page directory index.
1834	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
1835	* Assume this is a temporary mapping.
1836	* @param GCPtrPage GC Pointer of the page that caused the fault
1837	*/
1838	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
1839	{
1840	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1841	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
1842	LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
1843
1844	#if PGM_GST_TYPE == PGM_TYPE_32BIT
1845
1846	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1847	# error "Invalid shadow mode for 32-bit guest mode!"
1848	# endif
1849
1850	/*
1851	* Validate input a little bit.
1852	*/
1853	Assert(iPDSrc == (GCPtrPage >> GST_PD_SHIFT));
1854	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1855	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
1856	# else
1857	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
1858	# endif
1859	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1860	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
1861	SHWPDE PdeDst = *pPdeDst;
1862
1863	/*
1864	* Check for conflicts.
1865	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
1866	* HC: Simply resolve the conflict.
1867	*/
1868	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1869	{
1870	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1871	# ifndef IN_RING3
1872	Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
1873	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1874	return VERR_ADDRESS_CONFLICT;
1875	# else
1876	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
1877	Assert(pMapping);
1878	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPDSrc);
1879	if (VBOX_FAILURE(rc))
1880	{
1881	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1882	return rc;
1883	}
1884	PdeDst = *pPdeDst;
1885	# endif
1886	}
1887	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
1888
1889	/*
1890	* Sync page directory entry.
1891	*/
1892	int rc = VINF_SUCCESS;
1893	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1894	if (PdeSrc.n.u1Present)
1895	{
1896	/*
1897	* Allocate & map the page table.
1898	*/
1899	PSHWPT pPTDst;
1900	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1901	PPGMPOOLPAGE pShwPage;
1902	RTGCPHYS GCPhys;
1903	if (fPageTable)
1904	{
1905	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1906	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1907	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1908	# endif
1909	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1910	}
1911	else
1912	{
1913	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1914	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1915	GCPhys \|= GCPtrPage & RT_BIT(X86_PAGE_2M_SHIFT);
1916	# endif
1917	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1918	}
1919	if (rc == VINF_SUCCESS)
1920	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1921	else if (rc == VINF_PGM_CACHED_PAGE)
1922	{
1923	/*
1924	* The PT was cached, just hook it up.
1925	*/
1926	if (fPageTable)
1927	PdeDst.u = pShwPage->Core.Key
1928	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1929	else
1930	{
1931	PdeDst.u = pShwPage->Core.Key
1932	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1933	# ifdef PGM_SYNC_DIRTY_BIT /* (see explanation and assumtions further down.) */
1934	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1935	{
1936	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1937	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1938	PdeDst.b.u1Write = 0;
1939	}
1940	# endif
1941	}
1942	*pPdeDst = PdeDst;
1943	return VINF_SUCCESS;
1944	}
1945	else if (rc == VERR_PGM_POOL_FLUSHED)
1946	return VINF_PGM_SYNC_CR3;
1947	else
1948	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
1949	PdeDst.u &= X86_PDE_AVL_MASK;
1950	PdeDst.u \|= pShwPage->Core.Key;
1951
1952	# ifdef PGM_SYNC_DIRTY_BIT
1953	/*
1954	* Page directory has been accessed (this is a fault situation, remember).
1955	*/
1956	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
1957	# endif
1958	if (fPageTable)
1959	{
1960	/*
1961	* Page table - 4KB.
1962	*
1963	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
1964	*/
1965	Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
1966	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
1967	PGSTPT pPTSrc;
1968	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1969	if (VBOX_SUCCESS(rc))
1970	{
1971	/*
1972	* Start by syncing the page directory entry so CSAM's TLB trick works.
1973	*/
1974	PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK))
1975	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1976	*pPdeDst = PdeDst;
1977
1978	/*
1979	* Directory/page user or supervisor privilege: (same goes for read/write)
1980	*
1981	* Directory Page Combined
1982	* U/S U/S U/S
1983	* 0 0 0
1984	* 0 1 0
1985	* 1 0 0
1986	* 1 1 1
1987	*
1988	* Simple AND operation. Table listed for completeness.
1989	*
1990	*/
1991	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
1992	# ifdef PGM_SYNC_N_PAGES
1993	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1994	unsigned iPTDst = iPTBase;
1995	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1996	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
1997	iPTDst = 0;
1998	else
1999	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2000	# else /* !PGM_SYNC_N_PAGES */
2001	unsigned iPTDst = 0;
2002	const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
2003	# endif /* !PGM_SYNC_N_PAGES */
2004	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2005	const unsigned offPTSrc = 0;
2006	# else
2007	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2008	# endif
2009	for (; iPTDst < iPTDstEnd; iPTDst++)
2010	{
2011	const unsigned iPTSrc = iPTDst + offPTSrc;
2012	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2013
2014	if (PteSrc.n.u1Present) /* we've already cleared it above */
2015	{
2016	#ifndef IN_RING0
2017	/*
2018	* Assuming kernel code will be marked as supervisor - and not as user level
2019	* and executed using a conforming code selector - And marked as readonly.
2020	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2021	*/
2022	PPGMPAGE pPage;
2023	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2024	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2025	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2026	&& PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPage))
2027	)
2028	#endif
2029	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2030	Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
2031	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
2032	PteSrc.n.u1Present,
2033	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2034	PteSrc.n.u1User & PdeSrc.n.u1User,
2035	(uint64_t)PteSrc.u,
2036	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2037	(PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
2038	}
2039	} /* for PTEs */
2040	}
2041	}
2042	else
2043	{
2044	/*
2045	* Big page - 2/4MB.
2046	*
2047	* We'll walk the ram range list in parallel and optimize lookups.
2048	* We will only sync on shadow page table at a time.
2049	*/
2050	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
2051
2052	/**
2053	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2054	*/
2055
2056	/*
2057	* Start by syncing the page directory entry.
2058	*/
2059	PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2060	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2061
2062	# ifdef PGM_SYNC_DIRTY_BIT
2063	/*
2064	* If the page is not flagged as dirty and is writable, then make it read-only
2065	* at PD level, so we can set the dirty bit when the page is modified.
2066	*
2067	* ASSUMES that page access handlers are implemented on page table entry level.
2068	* Thus we will first catch the dirty access and set PDE.D and restart. If
2069	* there is an access handler, we'll trap again and let it work on the problem.
2070	*/
2071	/** @todo move the above stuff to a section in the PGM documentation. */
2072	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2073	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2074	{
2075	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2076	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2077	PdeDst.b.u1Write = 0;
2078	}
2079	# endif /* PGM_SYNC_DIRTY_BIT */
2080	*pPdeDst = PdeDst;
2081
2082	/*
2083	* Fill the shadow page table.
2084	*/
2085	/* Get address and flags from the source PDE. */
2086	SHWPTE PteDstBase;
2087	PteDstBase.u = PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2088
2089	/* Loop thru the entries in the shadow PT. */
2090	const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2091	Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2092	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2093	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2094	PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
2095	unsigned iPTDst = 0;
2096	while (iPTDst < ELEMENTS(pPTDst->a))
2097	{
2098	/* Advance ram range list. */
2099	while (pRam && GCPhys > pRam->GCPhysLast)
2100	pRam = CTXALLSUFF(pRam->pNext);
2101	if (pRam && GCPhys >= pRam->GCPhys)
2102	{
2103	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2104	do
2105	{
2106	/* Make shadow PTE. */
2107	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2108	SHWPTE PteDst;
2109
2110	/* Make sure the RAM has already been allocated. */
2111	if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2112	{
2113	if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2114	{
2115	# ifdef IN_RING3
2116	int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2117	# else
2118	int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2119	# endif
2120	if (rc != VINF_SUCCESS)
2121	return rc;
2122	}
2123	}
2124
2125	if (PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPage))
2126	{
2127	if (!PGM_PAGE_HAVE_ACTIVE_ALL_HANDLERS(pPage))
2128	{
2129	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2130	PteDst.n.u1Write = 0;
2131	}
2132	else
2133	PteDst.u = 0;
2134	}
2135	# ifndef IN_RING0
2136	/*
2137	* Assuming kernel code will be marked as supervisor and not as user level and executed
2138	* using a conforming code selector. Don't check for readonly, as that implies the whole
2139	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2140	*/
2141	else if ( !PdeSrc.n.u1User
2142	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2143	PteDst.u = 0;
2144	# endif
2145	else
2146	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2147	# ifdef PGMPOOL_WITH_USER_TRACKING
2148	if (PteDst.n.u1Present)
2149	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2150	# endif
2151	/* commit it */
2152	pPTDst->a[iPTDst] = PteDst;
2153	Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2154	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2155	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2156
2157	/* advance */
2158	GCPhys += PAGE_SIZE;
2159	iHCPage++;
2160	iPTDst++;
2161	} while ( iPTDst < ELEMENTS(pPTDst->a)
2162	&& GCPhys <= pRam->GCPhysLast);
2163	}
2164	else if (pRam)
2165	{
2166	Log(("Invalid pages at %VGp\n", GCPhys));
2167	do
2168	{
2169	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2170	GCPhys += PAGE_SIZE;
2171	iPTDst++;
2172	} while ( iPTDst < ELEMENTS(pPTDst->a)
2173	&& GCPhys < pRam->GCPhys);
2174	}
2175	else
2176	{
2177	Log(("Invalid pages at %VGp (2)\n", GCPhys));
2178	for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2179	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2180	}
2181	} /* while more PTEs */
2182	} /* 4KB / 4MB */
2183	}
2184	else
2185	AssertRelease(!PdeDst.n.u1Present);
2186
2187	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2188	# ifdef IN_GC
2189	if (VBOX_FAILURE(rc))
2190	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2191	# endif
2192	return rc;
2193
2194	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
2195
2196	int rc = VINF_SUCCESS;
2197
2198	/*
2199	* Validate input a little bit.
2200	*/
2201	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2202	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2203	# else
2204	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2205	# endif
2206	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2207	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2208	SHWPDE PdeDst = *pPdeDst;
2209
2210	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2211	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2212
2213	GSTPDE PdeSrc;
2214	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2215	PdeSrc.n.u1Present = 1;
2216	PdeSrc.n.u1Write = 1;
2217	PdeSrc.n.u1Accessed = 1;
2218	PdeSrc.n.u1User = 1;
2219
2220	/*
2221	* Allocate & map the page table.
2222	*/
2223	PSHWPT pPTDst;
2224	PPGMPOOLPAGE pShwPage;
2225	RTGCPHYS GCPhys;
2226
2227	/* Virtual address = physical address */
2228	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK_32;
2229	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2230
2231	if ( rc == VINF_SUCCESS
2232	\|\| rc == VINF_PGM_CACHED_PAGE)
2233	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2234	else
2235	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2236
2237	PdeDst.u &= X86_PDE_AVL_MASK;
2238	PdeDst.u \|= pShwPage->Core.Key;
2239	PdeDst.n.u1Present = 1;
2240	*pPdeDst = PdeDst;
2241
2242	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2243	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2244	return rc;
2245
2246	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2247
2248	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2249	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2250	return VERR_INTERNAL_ERROR;
2251	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2252	}
2253
2254
2255
2256	/**
2257	* Prefetch a page/set of pages.
2258	*
2259	* Typically used to sync commonly used pages before entering raw mode
2260	* after a CR3 reload.
2261	*
2262	* @returns VBox status code.
2263	* @param pVM VM handle.
2264	* @param GCPtrPage Page to invalidate.
2265	*/
2266	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2267	{
2268	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2269
2270	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2271	# error "Invalid shadow mode for 32-bit guest mode!"
2272	# endif
2273
2274	/*
2275	* Check that all Guest levels thru the PDE are present, getting the
2276	* PD and PDE in the processes.
2277	*/
2278	int rc = VINF_SUCCESS;
2279	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2280	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2281	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2282	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2283	# else /* PAE */
2284	unsigned iPDSrc
2285	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2286	# endif
2287	# else
2288	PGSTPD pPDSrc = NULL;
2289	const unsigned iPDSrc = 0;
2290	# endif
2291
2292	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2293	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2294	# else
2295	GSTPDE PdeSrc;
2296	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2297	PdeSrc.n.u1Present = 1;
2298	PdeSrc.n.u1Write = 1;
2299	PdeSrc.n.u1Accessed = 1;
2300	PdeSrc.n.u1User = 1;
2301	# endif
2302
2303	# ifdef PGM_SYNC_ACCESSED_BIT
2304	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2305	# else
2306	if (PdeSrc.n.u1Present)
2307	# endif
2308	{
2309	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2310	const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2311	# else
2312	const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2313	# endif
2314	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2315	{
2316	if (!PdeDst.n.u1Present)
2317	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2318	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2319	else
2320	{
2321	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2322	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2323	* makes no sense to prefetch more than one page.
2324	*/
2325	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2326	if (VBOX_SUCCESS(rc))
2327	rc = VINF_SUCCESS;
2328	}
2329	}
2330	}
2331	return rc;
2332
2333	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2334
2335	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2336	return VERR_INTERNAL_ERROR;
2337	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2338	}
2339
2340
2341
2342
2343	/**
2344	* Syncs a page during a PGMVerifyAccess() call.
2345	*
2346	* @returns VBox status code (informational included).
2347	* @param GCPtrPage The address of the page to sync.
2348	* @param fPage The effective guest page flags.
2349	* @param uErr The trap error code.
2350	*/
2351	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2352	{
2353	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2354
2355	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2356
2357	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2358	# error "Invalid shadow mode for 32-bit guest mode!"
2359	# endif
2360
2361	#ifndef IN_RING0
2362	if (!(fPage & X86_PTE_US))
2363	{
2364	/*
2365	* Mark this page as safe.
2366	*/
2367	/** @todo not correct for pages that contain both code and data!! */
2368	Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2369	CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2370	}
2371	#endif
2372	/*
2373	* Get guest PD and index.
2374	*/
2375
2376	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2377	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2378	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2379	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2380	# else /* PAE */
2381	unsigned iPDSrc;
2382	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2383	# endif
2384	# else
2385	PGSTPD pPDSrc = NULL;
2386	const unsigned iPDSrc = 0;
2387	# endif
2388	int rc = VINF_SUCCESS;
2389
2390	/*
2391	* First check if the shadow pd is present.
2392	*/
2393	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2394	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2395	# else
2396	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2397	# endif
2398	if (!pPdeDst->n.u1Present)
2399	{
2400	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2401	AssertRC(rc);
2402	if (rc != VINF_SUCCESS)
2403	return rc;
2404	}
2405
2406	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2407	/* Check for dirty bit fault */
2408	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2409	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2410	Log(("PGMVerifyAccess: success (dirty)\n"));
2411	else
2412	{
2413	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2414	#else
2415	{
2416	GSTPDE PdeSrc;
2417	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2418	PdeSrc.n.u1Present = 1;
2419	PdeSrc.n.u1Write = 1;
2420	PdeSrc.n.u1Accessed = 1;
2421	PdeSrc.n.u1User = 1;
2422
2423	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2424	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2425	if (uErr & X86_TRAP_PF_US)
2426	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2427	else /* supervisor */
2428	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2429
2430	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2431	if (VBOX_SUCCESS(rc))
2432	{
2433	/* Page was successfully synced */
2434	Log2(("PGMVerifyAccess: success (sync)\n"));
2435	rc = VINF_SUCCESS;
2436	}
2437	else
2438	{
2439	Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2440	return VINF_EM_RAW_GUEST_TRAP;
2441	}
2442	}
2443	return rc;
2444
2445	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2446
2447	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2448	return VERR_INTERNAL_ERROR;
2449	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2450	}
2451
2452
2453	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2454	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2455	/**
2456	* Figures out which kind of shadow page this guest PDE warrants.
2457	*
2458	* @returns Shadow page kind.
2459	* @param pPdeSrc The guest PDE in question.
2460	* @param cr4 The current guest cr4 value.
2461	*/
2462	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
2463	{
2464	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
2465	return BTH_PGMPOOLKIND_PT_FOR_PT;
2466	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
2467	//{
2468	// case 0:
2469	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2470	// case X86_PDE4M_RW:
2471	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2472	// case X86_PDE4M_US:
2473	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2474	// case X86_PDE4M_RW \| X86_PDE4M_US:
2475	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2476	# if 0
2477	// case X86_PDE4M_PAE_NX:
2478	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2479	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
2480	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2481	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2482	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2483	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2484	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2485	# endif
2486	return BTH_PGMPOOLKIND_PT_FOR_BIG;
2487	//}
2488	}
2489	# endif
2490	#endif
2491
2492	#undef MY_STAM_COUNTER_INC
2493	#define MY_STAM_COUNTER_INC(a) do { } while (0)
2494
2495
2496	/**
2497	* Syncs the paging hierarchy starting at CR3.
2498	*
2499	* @returns VBox status code, no specials.
2500	* @param pVM The virtual machine.
2501	* @param cr0 Guest context CR0 register
2502	* @param cr3 Guest context CR3 register
2503	* @param cr4 Guest context CR4 register
2504	* @param fGlobal Including global page directories or not
2505	*/
2506	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal)
2507	{
2508	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2509	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2510	if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2511	fGlobal = true; /* Change this CR3 reload to be a global one. */
2512	# endif
2513	#endif
2514
2515	/*
2516	* Update page access handlers.
2517	* The virtual are always flushed, while the physical are only on demand.
2518	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2519	* have to look into that later because it will have a bad influence on the performance.
2520	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
2521	* bird: Yes, but that won't work for aliases.
2522	*/
2523	/** @todo this MUST go away. See #1557. */
2524	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2525	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2526	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2527
2528	#ifdef PGMPOOL_WITH_MONITORING
2529	/*
2530	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2531	* Occationally we will have to clear all the shadow page tables because we wanted
2532	* to monitor a page which was mapped by too many shadowed page tables. This operation
2533	* sometimes refered to as a 'lightweight flush'.
2534	*/
2535	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2536	pgmPoolMonitorModifiedClearAll(pVM);
2537	else
2538	{
2539	# ifdef IN_RING3
2540	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2541	pgmPoolClearAll(pVM);
2542	# else
2543	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2544	return VINF_PGM_SYNC_CR3;
2545	# endif
2546	}
2547	#endif
2548
2549	Assert(fGlobal \|\| (cr4 & X86_CR4_PGE));
2550	MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2551
2552	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2553	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2554	/*
2555	* Get page directory addresses.
2556	*/
2557	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2558	PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2559	# else
2560	PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2561	# endif
2562
2563	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2564	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2565	# else /* PAE */
2566	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, 0);
2567	# endif
2568
2569	Assert(pPDSrc);
2570	#ifndef IN_GC
2571	Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2572	#endif
2573
2574	/*
2575	* Iterate the page directory.
2576	*/
2577	PPGMMAPPING pMapping;
2578	unsigned iPdNoMapping;
2579	const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2580	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2581
2582	/* Only check mappings if they are supposed to be put into the shadow page table. */
2583	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2584	{
2585	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2586	iPdNoMapping = (pMapping) ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U; /** PAE todo */
2587	}
2588	else
2589	{
2590	pMapping = 0;
2591	iPdNoMapping = ~0U;
2592	}
2593
2594	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2595	{
2596	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2597	Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2598	# else
2599	Assert(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst);
2600	# endif
2601	register GSTPDE PdeSrc = pPDSrc->a[iPD];
2602	if ( PdeSrc.n.u1Present
2603	&& (PdeSrc.n.u1User \|\| fRawR0Enabled))
2604	{
2605	/*
2606	* Check for conflicts with GC mappings.
2607	*/
2608	if (iPD == iPdNoMapping)
2609	{
2610	if (pVM->pgm.s.fMappingsFixed)
2611	{
2612	/* It's fixed, just skip the mapping. */
2613	const unsigned cPTs = pMapping->cPTs;
2614	iPD += cPTs - 1;
2615	pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2616	pMapping = pMapping->CTXALLSUFF(pNext);
2617	iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2618	continue;
2619	}
2620
2621	#ifdef IN_RING3
2622	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2623	if (VBOX_FAILURE(rc))
2624	return rc;
2625
2626	/*
2627	* Update iPdNoMapping and pMapping.
2628	*/
2629	pMapping = pVM->pgm.s.pMappingsR3;
2630	while (pMapping && pMapping->GCPtr < (iPD << X86_PD_SHIFT))
2631	pMapping = pMapping->pNextR3;
2632	iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2633	#else
2634	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2635	return VINF_PGM_SYNC_CR3;
2636	#endif
2637	}
2638
2639	/*
2640	* Sync page directory entry.
2641	*
2642	* The current approach is to allocated the page table but to set
2643	* the entry to not-present and postpone the page table synching till
2644	* it's actually used.
2645	*/
2646	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2647	const unsigned iPdShw = iPD; NOREF(iPdShw);
2648	# else
2649	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2650	# endif
2651	{
2652	SHWPDE PdeDst = *pPDEDst;
2653	if (PdeDst.n.u1Present)
2654	{
2655	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2656	RTGCPHYS GCPhys;
2657	if ( !PdeSrc.b.u1Size
2658	\|\| !(cr4 & X86_CR4_PSE))
2659	{
2660	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2661	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2662	GCPhys \|= i * (PAGE_SIZE / 2);
2663	# endif
2664	}
2665	else
2666	{
2667	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
2668	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2669	GCPhys \|= i * X86_PAGE_2M_SIZE;
2670	# endif
2671	}
2672
2673	if ( pShwPage->GCPhys == GCPhys
2674	&& pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2675	&& ( pShwPage->fCached
2676	\|\| ( !fGlobal
2677	&& ( false
2678	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2679	\|\| ( (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2680	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE)) /* global 2/4MB page. */
2681	\|\| ( !pShwPage->fSeenNonGlobal
2682	&& (cr4 & X86_CR4_PGE))
2683	# endif
2684	)
2685	)
2686	)
2687	&& ( (PdeSrc.u & (X86_PDE_US \| X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US \| X86_PDE_RW))
2688	\|\| ( (cr4 & X86_CR4_PSE)
2689	&& ((PdeSrc.u & (X86_PDE_US \| X86_PDE4M_PS \| X86_PDE4M_D)) \| PGM_PDFLAGS_TRACK_DIRTY)
2690	== ((PdeDst.u & (X86_PDE_US \| X86_PDE_RW \| PGM_PDFLAGS_TRACK_DIRTY)) \| X86_PDE4M_PS))
2691	)
2692	)
2693	{
2694	# ifdef VBOX_WITH_STATISTICS
2695	if ( !fGlobal
2696	&& (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2697	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE))
2698	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2699	else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2700	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2701	else
2702	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2703	# endif /* VBOX_WITH_STATISTICS */
2704	/** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2705	* The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2706	//# ifdef PGMPOOL_WITH_CACHE
2707	// pgmPoolCacheUsed(pPool, pShwPage);
2708	//# endif
2709	}
2710	else
2711	{
2712	pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
2713	pPDEDst->u = 0;
2714	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
2715	}
2716	}
2717	else
2718	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
2719	pPDEDst++;
2720	}
2721	}
2722	else if (iPD != iPdNoMapping)
2723	{
2724	/*
2725	* Check if there is any page directory to mark not present here.
2726	*/
2727	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2728	const unsigned iPdShw = iPD; NOREF(iPdShw);
2729	# else
2730	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2731	# endif
2732	{
2733	if (pPDEDst->n.u1Present)
2734	{
2735	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
2736	pPDEDst->u = 0;
2737	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
2738	}
2739	pPDEDst++;
2740	}
2741	}
2742	else
2743	{
2744	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2745	const unsigned cPTs = pMapping->cPTs;
2746	if (pVM->pgm.s.fMappingsFixed)
2747	{
2748	/* It's fixed, just skip the mapping. */
2749	pMapping = pMapping->CTXALLSUFF(pNext);
2750	iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2751	}
2752	else
2753	{
2754	/*
2755	* Check for conflicts for subsequent pagetables
2756	* and advance to the next mapping.
2757	*/
2758	iPdNoMapping = ~0U;
2759	unsigned iPT = cPTs;
2760	while (iPT-- > 1)
2761	{
2762	if ( pPDSrc->a[iPD + iPT].n.u1Present
2763	&& (pPDSrc->a[iPD + iPT].n.u1User \|\| fRawR0Enabled))
2764	{
2765	# ifdef IN_RING3
2766	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2767	if (VBOX_FAILURE(rc))
2768	return rc;
2769
2770	/*
2771	* Update iPdNoMapping and pMapping.
2772	*/
2773	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2774	while (pMapping && pMapping->GCPtr < (iPD << X86_PD_SHIFT))
2775	pMapping = pMapping->CTXALLSUFF(pNext);
2776	iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2777	break;
2778	# else
2779	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2780	return VINF_PGM_SYNC_CR3;
2781	# endif
2782	}
2783	}
2784	if (iPdNoMapping == ~0U && pMapping)
2785	{
2786	pMapping = pMapping->CTXALLSUFF(pNext);
2787	if (pMapping)
2788	iPdNoMapping = pMapping->GCPtr >> X86_PD_SHIFT;
2789	}
2790	}
2791	/* advance. */
2792	iPD += cPTs - 1;
2793	pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2794	}
2795
2796	} /* for iPD */
2797	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2798	# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
2799	# endif
2800
2801	return VINF_SUCCESS;
2802
2803	#elif PGM_GST_TYPE == PGM_TYPE_PAE
2804	# if PGM_SHW_TYPE == PGM_TYPE_PAE
2805	//# error not implemented
2806	return VERR_INTERNAL_ERROR;
2807
2808	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2809	# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
2810	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2811
2812	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
2813	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
2814	//# error not implemented
2815	return VERR_INTERNAL_ERROR;
2816
2817	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2818	# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
2819	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2820
2821	#else /* guest real and protected mode */
2822
2823	return VINF_SUCCESS;
2824	#endif
2825	}
2826
2827
2828
2829
2830	#ifdef VBOX_STRICT
2831	#ifdef IN_GC
2832	# undef AssertMsgFailed
2833	# define AssertMsgFailed Log
2834	#endif
2835	#ifdef IN_RING3
2836	# include <VBox/dbgf.h>
2837
2838	/**
2839	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
2840	*
2841	* @returns VBox status code (VINF_SUCCESS).
2842	* @param pVM The VM handle.
2843	* @param cr3 The root of the hierarchy.
2844	* @param crr The cr4, only PAE and PSE is currently used.
2845	* @param fLongMode Set if long mode, false if not long mode.
2846	* @param cMaxDepth Number of levels to dump.
2847	* @param pHlp Pointer to the output functions.
2848	*/
2849	__BEGIN_DECLS
2850	PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
2851	__END_DECLS
2852
2853	#endif
2854
2855	/**
2856	* Checks that the shadow page table is in sync with the guest one.
2857	*
2858	* @returns The number of errors.
2859	* @param pVM The virtual machine.
2860	* @param cr3 Guest context CR3 register
2861	* @param cr4 Guest context CR4 register
2862	* @param GCPtr Where to start. Defaults to 0.
2863	* @param cb How much to check. Defaults to everything.
2864	*/
2865	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
2866	{
2867	unsigned cErrors = 0;
2868
2869	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2870
2871	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2872	# error "Invalid shadow mode for 32-bit guest paging."
2873	# endif
2874
2875	PPGM pPGM = &pVM->pgm.s;
2876	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
2877	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
2878	RTHCPHYS HCPhys; /* general usage. */
2879	int rc;
2880
2881	/*
2882	* Check that the Guest CR3 and all it's mappings are correct.
2883	*/
2884	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & X86_CR3_PAGE_MASK),
2885	("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
2886	false);
2887	rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
2888	AssertRCReturn(rc, 1);
2889	HCPhys = NIL_RTHCPHYS;
2890	rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & X86_CR3_PAGE_MASK, &HCPhys);
2891	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
2892	# ifdef IN_RING3
2893	RTGCPHYS GCPhys;
2894	rc = PGMR3DbgHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
2895	AssertRCReturn(rc, 1);
2896	AssertMsgReturn((cr3 & X86_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
2897	# endif
2898	const X86PD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
2899
2900	/*
2901	* Get and check the Shadow CR3.
2902	*/
2903	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2904	const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
2905	unsigned cPDEs = ELEMENTS(pPDDst->a);
2906	# else
2907	const X86PDPAE pPDDst = pPGM->CTXMID(ap,PaePDs[0]); / use it as a 2048 entry PD */
2908	unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
2909	# endif
2910	if (cb != ~(RTGCUINTPTR)0)
2911	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
2912
2913	/** @todo call the other two PGMAssert() functions. /
2914
2915	/*
2916	* Iterate the shadow page directory.
2917	*/
2918	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
2919	unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
2920	cPDEs += iPDDst;
2921	for (;
2922	iPDDst < cPDEs;
2923	iPDDst++, GCPtr += _4G / cPDEs)
2924	{
2925	const SHWPDE PdeDst = pPDDst->a[iPDDst];
2926	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2927	{
2928	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2929	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
2930	{
2931	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
2932	cErrors++;
2933	continue;
2934	}
2935	}
2936	else if ( (PdeDst.u & X86_PDE_P)
2937	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
2938	)
2939	{
2940	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
2941	PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
2942	if (!pPoolPage)
2943	{
2944	AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
2945	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
2946	cErrors++;
2947	continue;
2948	}
2949	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
2950
2951	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
2952	{
2953	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
2954	GCPtr, (uint64_t)PdeDst.u));
2955	cErrors++;
2956	}
2957
2958	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
2959	{
2960	AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
2961	GCPtr, (uint64_t)PdeDst.u));
2962	cErrors++;
2963	}
2964
2965	const X86PDE PdeSrc = pPDSrc->a[iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)];
2966	if (!PdeSrc.n.u1Present)
2967	{
2968	AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
2969	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
2970	cErrors++;
2971	continue;
2972	}
2973
2974	if ( !PdeSrc.b.u1Size
2975	\|\| !(cr4 & X86_CR4_PSE))
2976	{
2977	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
2978	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2979	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2980	# endif
2981	}
2982	else
2983	{
2984	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
2985	{
2986	AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
2987	GCPtr, (uint64_t)PdeSrc.u));
2988	cErrors++;
2989	continue;
2990	}
2991	GCPhysGst = PdeSrc.u & GST_PDE4M_PG_MASK;
2992	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2993	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
2994	# endif
2995	}
2996
2997	if ( pPoolPage->enmKind
2998	!= (!PdeSrc.b.u1Size \|\| !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
2999	{
3000	AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
3001	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3002	cErrors++;
3003	}
3004
3005	PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3006	if (!pPhysPage)
3007	{
3008	AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3009	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3010	cErrors++;
3011	continue;
3012	}
3013
3014	if (GCPhysGst != pPoolPage->GCPhys)
3015	{
3016	AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
3017	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3018	cErrors++;
3019	continue;
3020	}
3021
3022	if ( !PdeSrc.b.u1Size
3023	\|\| !(cr4 & X86_CR4_PSE))
3024	{
3025	/*
3026	* Page Table.
3027	*/
3028	const GSTPT *pPTSrc;
3029	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3030	if (VBOX_FAILURE(rc))
3031	{
3032	AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3033	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3034	cErrors++;
3035	continue;
3036	}
3037	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3038	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3039	{
3040	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3041	// (This problem will go away when/if we shadow multiple CR3s.)
3042	AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3043	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3044	cErrors++;
3045	continue;
3046	}
3047	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3048	{
3049	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
3050	GCPtr, (uint64_t)PdeDst.u));
3051	cErrors++;
3052	continue;
3053	}
3054
3055	/* iterate the page table. */
3056	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3057	const unsigned offPTSrc = 0;
3058	# else
3059	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3060	# endif
3061	for (unsigned iPT = 0, off = 0;
3062	iPT < ELEMENTS(pPTDst->a);
3063	iPT++, off += PAGE_SIZE)
3064	{
3065	const SHWPTE PteDst = pPTDst->a[iPT];
3066
3067	/* skip not-present entries. */
3068	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3069	continue;
3070	Assert(PteDst.n.u1Present);
3071
3072	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3073	if (!PteSrc.n.u1Present)
3074	{
3075	#ifdef IN_RING3
3076	PGMAssertHandlerAndFlagsInSync(pVM);
3077	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3078	#endif
3079	AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
3080	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3081	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3082	cErrors++;
3083	continue;
3084	}
3085
3086	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3087	# if 1 /** @todo sync accessed bit properly... */
3088	fIgnoreFlags \|= X86_PTE_A;
3089	# endif
3090
3091	/* match the physical addresses */
3092	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3093	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3094
3095	# ifdef IN_RING3
3096	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3097	if (VBOX_FAILURE(rc))
3098	{
3099	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3100	{
3101	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3102	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3103	cErrors++;
3104	continue;
3105	}
3106	}
3107	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3108	{
3109	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3110	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3111	cErrors++;
3112	continue;
3113	}
3114	# endif
3115
3116	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3117	if (!pPhysPage)
3118	{
3119	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3120	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3121	{
3122	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3123	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3124	cErrors++;
3125	continue;
3126	}
3127	# endif
3128	if (PteDst.n.u1Write)
3129	{
3130	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3131	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3132	cErrors++;
3133	}
3134	fIgnoreFlags \|= X86_PTE_RW;
3135	}
3136	else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
3137	{
3138	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3139	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3140	cErrors++;
3141	continue;
3142	}
3143
3144	/* flags */
3145	if (PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPhysPage))
3146	{
3147	if (!PGM_PAGE_HAVE_ACTIVE_ALL_HANDLERS(pPhysPage))
3148	{
3149	if (PteDst.n.u1Write)
3150	{
3151	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
3152	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3153	cErrors++;
3154	continue;
3155	}
3156	fIgnoreFlags \|= X86_PTE_RW;
3157	}
3158	else
3159	{
3160	if (PteDst.n.u1Present)
3161	{
3162	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
3163	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3164	cErrors++;
3165	continue;
3166	}
3167	fIgnoreFlags \|= X86_PTE_P;
3168	}
3169	}
3170	else
3171	{
3172	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3173	{
3174	if (PteDst.n.u1Write)
3175	{
3176	AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3177	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3178	cErrors++;
3179	continue;
3180	}
3181	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3182	{
3183	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3184	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3185	cErrors++;
3186	continue;
3187	}
3188	if (PteDst.n.u1Dirty)
3189	{
3190	AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3191	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3192	cErrors++;
3193	}
3194	# if 0 /** @todo sync access bit properly... */
3195	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3196	{
3197	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3198	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3199	cErrors++;
3200	}
3201	fIgnoreFlags \|= X86_PTE_RW;
3202	# else
3203	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3204	# endif
3205	}
3206	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3207	{
3208	/* access bit emulation (not implemented). */
3209	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3210	{
3211	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3212	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3213	cErrors++;
3214	continue;
3215	}
3216	if (!PteDst.n.u1Accessed)
3217	{
3218	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3219	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3220	cErrors++;
3221	}
3222	fIgnoreFlags \|= X86_PTE_P;
3223	}
3224	# ifdef DEBUG_sandervl
3225	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3226	# endif
3227	}
3228
3229	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3230	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3231	)
3232	{
3233	AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3234	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3235	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3236	cErrors++;
3237	continue;
3238	}
3239	} /* foreach PTE */
3240	}
3241	else
3242	{
3243	/*
3244	* Big Page.
3245	*/
3246	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| X86_PDE_PAE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3247	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3248	{
3249	if (PdeDst.n.u1Write)
3250	{
3251	AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3252	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3253	cErrors++;
3254	continue;
3255	}
3256	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3257	{
3258	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3259	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3260	cErrors++;
3261	continue;
3262	}
3263	# if 0 /** @todo sync access bit properly... */
3264	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3265	{
3266	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3267	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3268	cErrors++;
3269	}
3270	fIgnoreFlags \|= X86_PTE_RW;
3271	# else
3272	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3273	# endif
3274	}
3275	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3276	{
3277	/* access bit emulation (not implemented). */
3278	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3279	{
3280	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3281	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3282	cErrors++;
3283	continue;
3284	}
3285	if (!PdeDst.n.u1Accessed)
3286	{
3287	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3288	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3289	cErrors++;
3290	}
3291	fIgnoreFlags \|= X86_PTE_P;
3292	}
3293
3294	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3295	{
3296	AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3297	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3298	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3299	cErrors++;
3300	}
3301
3302	/* iterate the page table. */
3303	for (unsigned iPT = 0, off = 0;
3304	iPT < ELEMENTS(pPTDst->a);
3305	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3306	{
3307	const SHWPTE PteDst = pPTDst->a[iPT];
3308
3309	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3310	{
3311	AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3312	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3313	cErrors++;
3314	}
3315
3316	/* skip not-present entries. */
3317	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3318	continue;
3319
3320	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3321
3322	/* match the physical addresses */
3323	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3324
3325	# ifdef IN_RING3
3326	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3327	if (VBOX_FAILURE(rc))
3328	{
3329	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3330	{
3331	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3332	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3333	cErrors++;
3334	}
3335	}
3336	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3337	{
3338	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3339	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3340	cErrors++;
3341	continue;
3342	}
3343	# endif
3344
3345	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3346	if (!pPhysPage)
3347	{
3348	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3349	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3350	{
3351	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3352	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3353	cErrors++;
3354	continue;
3355	}
3356	# endif
3357	if (PteDst.n.u1Write)
3358	{
3359	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3360	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3361	cErrors++;
3362	}
3363	fIgnoreFlags \|= X86_PTE_RW;
3364	}
3365	else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
3366	{
3367	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3368	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3369	cErrors++;
3370	continue;
3371	}
3372
3373	/* flags */
3374	if (PGM_PAGE_HAVE_ACTIVE_HANDLERS(pPhysPage))
3375	{
3376	if (!PGM_PAGE_HAVE_ACTIVE_ALL_HANDLERS(pPhysPage))
3377	{
3378	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
3379	{
3380	if (PteDst.n.u1Write)
3381	{
3382	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3383	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3384	cErrors++;
3385	continue;
3386	}
3387	fIgnoreFlags \|= X86_PTE_RW;
3388	}
3389	}
3390	else
3391	{
3392	if (PteDst.n.u1Present)
3393	{
3394	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3395	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3396	cErrors++;
3397	continue;
3398	}
3399	fIgnoreFlags \|= X86_PTE_P;
3400	}
3401	}
3402
3403	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3404	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3405	)
3406	{
3407	AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3408	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3409	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3410	cErrors++;
3411	continue;
3412	}
3413	} /* foreach PTE */
3414	}
3415	}
3416	/* not present */
3417
3418	} /* forearch PDE */
3419
3420	# ifdef DEBUG
3421	if (cErrors)
3422	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3423	# endif
3424
3425	#elif PGM_GST_TYPE == PGM_TYPE_PAE
3426	//# error not implemented
3427
3428
3429	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3430	//# error not implemented
3431
3432	/#else: guest real and protected mode /
3433	#endif
3434	return cErrors;
3435	}
3436	#endif /* VBOX_STRICT */
3437

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 6927

Download in other formats: