PGMAllBth.h@ 2004

Last change on this file since 2004 was 1908, checked in by vboxsync, 18 years ago
Backed out part of 20091 to check if that's the cause of the major performance regression seen on testboxlin.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 142.0 KB

Line
1	/* $Id: PGMAllBth.h 1908 2007-04-04 07:49:50Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006 InnoTek Systemberatung GmbH
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License as published by the Free Software Foundation,
15	* in version 2 as it comes in the "COPYING" file of the VirtualBox OSE
16	* distribution. VirtualBox OSE is distributed in the hope that it will
17	* be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* If you received this file as part of a commercial VirtualBox
20	* distribution, then only the terms of your commercial VirtualBox
21	* license agreement apply instead of the previous paragraph.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVM pVM, VBOXPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PVBOXPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PVBOXPD pPDSrc, RTGCUINTPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	__END_DECLS
43
44
45	/**
46	* #PF Handler for raw-mode guest execution.
47	*
48	* @returns VBox status code (appropriate for trap handling and GC return).
49	* @param pVM VM Handle.
50	* @param uErr The trap error code.
51	* @param pRegFrame Trap register frame.
52	* @param pvFault The fault address.
53	*/
54	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
55	{
56	#if PGM_GST_TYPE == PGM_TYPE_32BIT
57	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
58	# error "32-bit guest mode is only implemented for 32-bit and PAE shadow modes."
59	# endif
60
61	# ifdef PGM_CACHE_VERY_STRICT
62	pgmCacheCheckPD(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM));
63	# endif
64
65	# if PGM_SHW_TYPE == PGM_TYPE_PAE
66	/*
67	* Hide the instruction fetch trap indicator for now.
68	*/
69	/** @todo NXE will change this and we must fix NXE in the switcher too! */
70	if (uErr & X86_TRAP_PF_ID)
71	{
72	uErr &= ~X86_TRAP_PF_ID;
73	TRPMSetErrorCode(pVM, uErr);
74	}
75	# endif
76
77	/*
78	* Get PDs.
79	*/
80	int rc;
81	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
82	const unsigned iPDSrc = (uintptr_t)pvFault >> GST_PD_SHIFT;
83	const unsigned iPDDst = (uintptr_t)pvFault >> SHW_PD_SHIFT;
84	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
85	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
86	# else /* PAE */
87	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries. */
88	# endif
89
90	/* Determine current privilege level */
91	uint32_t cpl = CPUMGetGuestCPL(pVM, pRegFrame);
92
93	# ifdef PGM_SYNC_DIRTY_BIT
94	/*
95	* If we successfully correct the write protection fault due to dirty bit
96	* tracking, or this page fault is a genuine one, then return immediately.
97	*/
98	STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
99	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
100	STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
101	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
102	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
103	{
104	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
105	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
106	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
107	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
108	}
109	# endif
110	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
111
112	/*
113	* A common case is the not-present error caused by lazy page table syncing.
114	*
115	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
116	* so we can safely assume that the shadow PT is present when calling SyncPage later.
117	*
118	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
119	* of mapping conflict and defer to SyncCR3 in R3.
120	* (Again, we do NOT support access handlers for non-present guest pages.)
121	*
122	*/
123	VBOXPDE PdeSrc = pPDSrc->a[iPDSrc];
124	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
125	&& !pPDDst->a[iPDDst].n.u1Present
126	&& PdeSrc.n.u1Present)
127
128	{
129	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
130	STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
131	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
132	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
133	if (VBOX_SUCCESS(rc))
134	{
135	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
136	return rc;
137	}
138	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
139	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
140	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
141	return VINF_PGM_SYNC_CR3;
142	}
143
144	/*
145	* Check if this address is within any of our mappings.
146	*
147	* This is very fast and it's gonna save us a bit of effort below and prevent
148	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
149	* (BTW, it's impossible to have physical access handlers in a mapping.)
150	*/
151	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
152	{
153	STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
154	PPGMMAPPING pMapping = CTXSUFF(pVM->pgm.s.pMappings);
155	for ( ; pMapping; pMapping = CTXSUFF(pMapping->pNext))
156	{
157	if ((uintptr_t)pvFault < (uintptr_t)pMapping->GCPtr)
158	break;
159	if ((uintptr_t)pvFault - (uintptr_t)pMapping->GCPtr < pMapping->cb)
160	{
161	/*
162	* The first thing we check is if we've got an undetected conflict.
163	*/
164	if (!pVM->pgm.s.fMappingsFixed)
165	{
166	unsigned iPT = pMapping->cPTs;
167	while (iPT-- > 0)
168	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
169	{
170	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
171	Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
172	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
173	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
174	return VINF_PGM_SYNC_CR3;
175	}
176	}
177
178	/*
179	* Check if the fault address is in a virtual page access handler range.
180	*/
181	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
182	if ( pCur
183	&& pCur->enmType != PGMVIRTHANDLERTYPE_EIP
184	&& (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb
185	&& ( uErr & X86_TRAP_PF_RW
186	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
187	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** r=bird: <- this is probably wrong. */
188	{
189	#ifdef IN_GC
190	STAM_PROFILE_START(&pCur->Stat, h);
191	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr);
192	STAM_PROFILE_STOP(&pCur->Stat, h);
193	#else
194	AssertFailed();
195	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
196	#endif
197	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
198	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
199	return rc;
200	}
201
202	/*
203	* Check if the EIP is in a virtual page access handler range.
204	*/
205	if (cpl == 0)
206	{
207	RTGCPTR pvEIP;
208	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &pvEIP);
209	if (VBOX_SUCCESS(rc))
210	{
211	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvEIP);
212	if ( pCur
213	&& pCur->enmType == PGMVIRTHANDLERTYPE_EIP
214	&& (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr < pCur->cb)
215	{
216	#ifdef IN_GC
217	STAM_PROFILE_START(&pCur->Stat, h);
218	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr);
219	STAM_PROFILE_STOP(&pCur->Stat, h);
220	#else
221	AssertFailed();
222	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
223	#endif
224	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
225	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
226	return rc;
227	}
228	}
229	}
230
231	/*
232	* Pretend we're not here and let the guest handle the trap.
233	*/
234	TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
235	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
236	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
237	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
238	return VINF_EM_RAW_GUEST_TRAP;
239	}
240	}
241	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
242	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
243
244	/*
245	* Check if this fault address is flagged for special treatment,
246	* which means we'll have to figure out the physical address and
247	* check flags associated with it.
248	*
249	* ASSUME that we can limit any special access handling to pages
250	* in page tables which the guest believes to be present.
251	*/
252	if (PdeSrc.n.u1Present)
253	{
254	RTGCPHYS GCPhys = ~0U;
255	uint32_t cr4 = CPUMGetGuestCR4(pVM);
256	if ( PdeSrc.b.u1Size
257	&& (cr4 & X86_CR4_PSE))
258	GCPhys = (PdeSrc.u & X86_PDE4M_PG_MASK)
259	\| ((RTGCPHYS)pvFault & (PAGE_OFFSET_MASK_BIG ^ PAGE_OFFSET_MASK));
260	else
261	{
262	PVBOXPT pPTSrc;
263	# ifdef IN_GC
264	rc = PGMGCDynMapGCPage(pVM, PdeSrc.u & X86_PDE_PG_MASK, (void **)&pPTSrc);
265	# else
266	pPTSrc = (PVBOXPT)MMPhysGCPhys2HCVirt(pVM, PdeSrc.u & X86_PDE_PG_MASK, sizeof(*pPTSrc));
267	if (pPTSrc == 0)
268	rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
269	# endif
270	if (VBOX_SUCCESS(rc))
271	{
272	unsigned iPTESrc = ((uintptr_t)pvFault >> PAGE_SHIFT) & PTE_MASK;
273	if (pPTSrc->a[iPTESrc].n.u1Present)
274	GCPhys = pPTSrc->a[iPTESrc].u & X86_PTE_PG_MASK;
275	}
276	}
277
278	/*
279	* If we have a GC address we'll check if it has any flags set.
280	*/
281	if (GCPhys != ~0U)
282	{
283	STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
284
285	RTHCPHYS HCPhys;
286	rc = PGMRamGCPhys2HCPhysWithFlags(&pVM->pgm.s, GCPhys, &HCPhys);
287	if (VBOX_SUCCESS(rc))
288	{
289	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_HANDLER \| MM_RAM_FLAGS_VIRTUAL_HANDLER))
290	{
291	if (HCPhys & MM_RAM_FLAGS_PHYSICAL_HANDLER)
292	{
293	/*
294	* Physical page access handler.
295	*/
296	const RTGCPHYS GCPhysFault = GCPhys \| ((uintptr_t)pvFault & PAGE_OFFSET_MASK);
297	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
298	if (pCur)
299	{
300	# ifdef PGM_SYNC_N_PAGES
301	/*
302	* If the region is write protected and we got a page not present fault, then sync
303	* the pages. If the fault was caused by a read, then restart the instruction.
304	* In case of write access continue to the GC write handler.
305	*
306	* ASSUMES that there is only one handler per page or that they have similar write properties.
307	*/
308	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
309	&& !(uErr & X86_TRAP_PF_P))
310	{
311	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
312	if ( VBOX_FAILURE(rc)
313	\|\| !(uErr & X86_TRAP_PF_RW)
314	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
315	{
316	AssertRC(rc);
317	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
318	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
319	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
320	return rc;
321	}
322	}
323	# endif
324
325	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
326	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
327	("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, HCPhys, uErr, pCur->enmType));
328
329	#ifdef IN_GC
330	Assert(CTXSUFF(pCur->pfnHandler));
331	STAM_PROFILE_START(&pCur->Stat, h);
332	rc = pCur->CTXSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXSUFF(pCur->pvUser));
333	STAM_PROFILE_STOP(&pCur->Stat, h);
334	#elif IN_RING0
335	if (CTXALLSUFF(pCur->pfnHandler))
336	{
337	STAM_PROFILE_START(&pCur->Stat, h);
338	rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
339	STAM_PROFILE_STOP(&pCur->Stat, h);
340	}
341	else
342	rc = VINF_EM_RAW_EMULATE_INSTR;
343	#else
344	rc = VINF_EM_RAW_EMULATE_INSTR;
345	#endif
346	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
347	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
348	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
349	return rc;
350	}
351	}
352	else
353	{
354	# ifdef PGM_SYNC_N_PAGES
355	/*
356	* If the region is write protected and we got a page not present fault, then sync
357	* the pages. If the fault was caused by a read, then restart the instruction.
358	* In case of write access continue to the GC write handler.
359	*/
360	if ( (HCPhys & (MM_RAM_FLAGS_VIRTUAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_ALL)) == MM_RAM_FLAGS_VIRTUAL_WRITE
361	&& !(uErr & X86_TRAP_PF_P))
362	{
363	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
364	if ( VBOX_FAILURE(rc)
365	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
366	\|\| !(uErr & X86_TRAP_PF_RW))
367	{
368	AssertRC(rc);
369	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
370	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
371	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
372	return rc;
373	}
374	}
375	# endif
376	/*
377	* Ok, it's an virtual page access handler.
378	*
379	* Since it's faster to search by address, we'll do that first
380	* and then retry by GCPhys if that fails.
381	*/
382	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
383	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
384	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
385	*/
386	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
387	if (pCur)
388	{
389	AssertMsg(!((uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb)
390	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
391	\|\| !(uErr & X86_TRAP_PF_P)
392	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
393	("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, HCPhys, uErr, pCur->enmType));
394
395	if ( pCur->enmType != PGMVIRTHANDLERTYPE_EIP
396	&& (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb
397	&& ( uErr & X86_TRAP_PF_RW
398	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
399	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** @todo r=bird: _HYPERVISOR is impossible here because of mapping check. */
400	{
401	#ifdef IN_GC
402	STAM_PROFILE_START(&pCur->Stat, h);
403	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr);
404	STAM_PROFILE_STOP(&pCur->Stat, h);
405	#else
406	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
407	#endif
408	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
409	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
410	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
411	return rc;
412	}
413	/* Unhandled part of a monitored page */
414	}
415	else
416	{
417	/* Check by physical address. */
418	PPGMVIRTHANDLER pCur;
419	unsigned iPage;
420	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((uintptr_t)pvFault & PAGE_OFFSET_MASK),
421	&pCur, &iPage);
422	Assert(VBOX_SUCCESS(rc) \|\| !pCur);
423	if ( pCur
424	&& pCur->enmType != PGMVIRTHANDLERTYPE_EIP
425	&& ( uErr & X86_TRAP_PF_RW
426	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
427	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) )
428	{
429	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
430	#ifdef IN_GC
431	uintptr_t off = (iPage << PAGE_SHIFT) + ((uintptr_t)pvFault & PAGE_OFFSET_MASK) - ((uintptr_t)pCur->GCPtr & PAGE_OFFSET_MASK);
432	Assert(off < pCur->cb);
433	STAM_PROFILE_START(&pCur->Stat, h);
434	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
435	STAM_PROFILE_STOP(&pCur->Stat, h);
436	#else
437	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
438	#endif
439	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
440	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
441	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
442	return rc;
443	}
444	}
445	}
446
447	/*
448	* There is a handled area of the page, but this fault doesn't belong to it.
449	* We must emulate the instruction.
450	*
451	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
452	* we first check if this was a page-not-present fault for a page with only
453	* write access handlers. Restart the instruction if it wasn't a write access.
454	*/
455	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
456
457	if ( !(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL))
458	&& !(uErr & X86_TRAP_PF_P))
459	{
460	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
461	if ( VBOX_FAILURE(rc)
462	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
463	\|\| !(uErr & X86_TRAP_PF_RW))
464	{
465	AssertRC(rc);
466	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
467	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
468	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
469	return rc;
470	}
471	}
472
473	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
474	* It's writing to an unhandled part of the LDT page several million times.
475	*/
476	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
477	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%VHp%s%s\n",
478	rc, HCPhys, HCPhys & MM_RAM_FLAGS_PHYSICAL_HANDLER ? " phys" : "",
479	HCPhys & MM_RAM_FLAGS_VIRTUAL_HANDLER ? " virt" : ""));
480	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
481	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
482	return rc;
483	} /* if any kind of handler */
484
485	if (uErr & X86_TRAP_PF_P)
486	{
487	/*
488	* The page isn't marked, but it might still be monitored by a virtual page access handler.
489	* (ASSUMES no temporary disabling of virtual handlers.)
490	*/
491	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
492	* we should correct both the shadow page table and physical memory flags, and not only check for
493	* accesses within the handler region but for access to pages with virtual handlers. */
494	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
495	if (pCur)
496	{
497	AssertMsg( !((uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb)
498	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
499	\|\| !(uErr & X86_TRAP_PF_P)
500	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
501	("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, HCPhys, uErr, pCur->enmType));
502
503	if ( pCur->enmType != PGMVIRTHANDLERTYPE_EIP
504	&& (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb
505	&& ( uErr & X86_TRAP_PF_RW
506	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
507	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** @todo r=bird: _HYPERVISOR is impossible here because of mapping check. */
508	{
509	#ifdef IN_GC
510	STAM_PROFILE_START(&pCur->Stat, h);
511	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr);
512	STAM_PROFILE_STOP(&pCur->Stat, h);
513	#else
514	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
515	#endif
516	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
517	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
518	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
519	return rc;
520	}
521	}
522	}
523	}
524	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
525
526	# ifdef PGM_OUT_OF_SYNC_IN_GC
527	/*
528	* We are here only if page is present in Guest page tables and trap is not handled
529	* by our handlers.
530	* Check it for page out-of-sync situation.
531	*/
532	STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
533
534	if (!(uErr & X86_TRAP_PF_P))
535	{
536	/*
537	* Page is not present in our page tables.
538	* Try to sync it!
539	* BTW, fPageShw is invalid in this branch!
540	*/
541	if (uErr & X86_TRAP_PF_US)
542	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
543	else /* supervisor */
544	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
545
546	# if defined(LOG_ENABLED) && !defined(IN_RING0)
547	RTGCPHYS GCPhys;
548	uint64_t fPageGst;
549	PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
550	Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
551	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
552	# endif /* LOG_ENABLED */
553
554	# ifndef IN_RING0
555	if (cpl == 0)
556	{
557	uint64_t fPageGst;
558	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
559	if ( VBOX_SUCCESS(rc)
560	&& !(fPageGst & X86_PTE_US))
561	{
562	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
563	if ( pvFault == (RTGCPTR)pRegFrame->eip
564	\|\| (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
565	#ifdef CSAM_DETECT_NEW_CODE_PAGES
566	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
567	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
568	#endif /* CSAM_DETECT_NEW_CODE_PAGES */
569	)
570	{
571	LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
572	rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
573	if (rc != VINF_SUCCESS)
574	{
575	/*
576	* CSAM needs to perform a job in ring 3.
577	*
578	* Sync the page before going to the host context; otherwise we'll end up in a loop if
579	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
580	*/
581	LogFlow(("CSAM ring 3 job\n"));
582	int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
583	AssertRC(rc2);
584
585	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
586	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
587	return rc;
588	}
589	}
590	#ifdef CSAM_DETECT_NEW_CODE_PAGES
591	else
592	if ( uErr == X86_TRAP_PF_RW
593	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
594	&& pRegFrame->ecx < 0x10000
595	)
596	{
597	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
598	* to detect loading of new code pages.
599	*/
600
601	/*
602	* Decode the instruction.
603	*/
604	RTGCPTR PC;
605	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
606	if (rc == VINF_SUCCESS)
607	{
608	DISCPUSTATE Cpu;
609	uint32_t cbOp;
610	rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
611
612	/* For now we'll restrict this to rep movsw/d instructions */
613	if ( rc == VINF_SUCCESS
614	&& Cpu.pCurInstr->opcode == OP_MOVSWD
615	&& (Cpu.prefix & PREFIX_REP))
616	{
617	CSAMMarkPossibleCodePage(pVM, pvFault);
618	}
619	}
620	}
621	#endif /* CSAM_DETECT_NEW_CODE_PAGES */
622
623	/*
624	* Mark this page as safe.
625	*/
626	/** @todo not correct for pages that contain both code and data!! */
627	Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
628	CSAMMarkPage(pVM, pvFault, true);
629	}
630	}
631	# endif
632	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
633	if (VBOX_SUCCESS(rc))
634	{
635	/* The page was successfully synced, return to the guest. */
636	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
637	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
638	return VINF_SUCCESS;
639	}
640	}
641	else
642	{
643	/*
644	* A side effect of not flushing global PDEs are out of sync pages due
645	* to physical monitored regions, that are no longer valid.
646	* Assume for now it only applies to the read/write flag
647	*/
648	if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
649	{
650	if (uErr & X86_TRAP_PF_US)
651	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
652	else /* supervisor */
653	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
654
655
656	/*
657	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
658	*/
659	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
660	if (VBOX_SUCCESS(rc))
661	{
662	/*
663	* Page was successfully synced, return to guest.
664	*/
665	# ifdef VBOX_STRICT
666	RTGCPHYS GCPhys;
667	uint64_t fPageGst;
668	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
669	Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
670	LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
671
672	uint64_t fPageShw;
673	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
674	Assert(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW);
675	# endif /* VBOX_STRICT */
676	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
677	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
678	return VINF_SUCCESS;
679	}
680	}
681
682	# ifdef VBOX_STRICT
683	/*
684	* Check for VMM page flags vs. Guest page flags consistency.
685	* Currently only for debug purposes.
686	*/
687	if (VBOX_SUCCESS(rc))
688	{
689	/* Get guest page flags. */
690	uint64_t fPageGst;
691	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
692	if (VBOX_SUCCESS(rc))
693	{
694	uint64_t fPageShw;
695	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
696
697	/*
698	* Compare page flags.
699	* Note: we have AVL, A, D bits desynched.
700	*/
701	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
702	("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
703	}
704	else
705	AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
706	}
707	else
708	AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
709	# endif /* VBOX_STRICT */
710	}
711	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
712	# endif /* PGM_OUT_OF_SYNC_IN_GC */
713	}
714	else
715	{
716	/*
717	* Page not present in Guest OS or invalid page table address.
718	* This is potential virtual page access handler food.
719	*
720	* For the present we'll say that our access handlers don't
721	* work for this case - we've already discarded the page table
722	* not present case which is identical to this.
723	*
724	* When we perchance find we need this, we will probably have AVL
725	* trees (offset based) to operate on and we can measure their speed
726	* agains mapping a page table and probably rearrange this handling
727	* a bit. (Like, searching virtual ranges before checking the
728	* physical address.)
729	*/
730	}
731	}
732
733
734	/*
735	* Check if it's in a EIP based virtual page access handler range.
736	* This is only used for supervisor pages in flat mode.
737	*/
738	/** @todo this stuff is completely broken by the out-of-sync stuff. since we don't use this stuff, that's not really a problem yet. */
739	STAM_PROFILE_START(&pVM->pgm.s.StatEIPHandlers, d);
740	if (cpl == 0)
741	{
742	RTGCPTR pvEIP;
743	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &pvEIP);
744	if ( VBOX_SUCCESS(rc)
745	&& pvEIP == (RTGCPTR)pRegFrame->eip)
746	{
747	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvEIP);
748	if ( pCur
749	&& pCur->enmType == PGMVIRTHANDLERTYPE_EIP
750	&& (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr < pCur->cb)
751	{
752	LogFlow(("EIP handler\n"));
753	#ifdef IN_GC
754	STAM_PROFILE_START(&pCur->Stat, h);
755	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr);
756	STAM_PROFILE_STOP(&pCur->Stat, h);
757	#else
758	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
759	#endif
760	STAM_PROFILE_STOP(&pVM->pgm.s.StatEIPHandlers, d);
761	return rc;
762	}
763	}
764	}
765	STAM_PROFILE_STOP(&pVM->pgm.s.StatEIPHandlers, d);
766
767	/*
768	* Conclusion, this is a guest trap.
769	*/
770	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
771	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
772	return VINF_EM_RAW_GUEST_TRAP;
773
774	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
775
776	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
777	return VERR_INTERNAL_ERROR;
778	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
779	}
780
781
782	/**
783	* Emulation of the invlpg instruction.
784	*
785	*
786	* @returns VBox status code.
787	*
788	* @param pVM VM handle.
789	* @param GCPtrPage Page to invalidate.
790	*
791	* @remark ASSUMES that the guest is updating before invalidating. This order
792	* isn't required by the CPU, so this is speculative and could cause
793	* trouble.
794	*
795	* @todo Flush page or page directory only if necessary!
796	* @todo Add a #define for simply invalidating the page.
797	*/
798	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
799	{
800	#if PGM_GST_TYPE == PGM_TYPE_32BIT
801
802	LogFlow(("InvalidatePage %x\n", GCPtrPage));
803	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
804	/*
805	* Get the shadow PD entry and skip out if this PD isn't present.
806	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
807	*/
808	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
809	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
810	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
811	# else
812	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
813	# endif
814	const SHWPDE PdeDst = *pPdeDst;
815	if (!PdeDst.n.u1Present)
816	{
817	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
818	return VINF_SUCCESS;
819	}
820
821	/*
822	* Get the guest PD entry and calc big page.
823	*/
824	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
825	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
826	VBOXPDE PdeSrc = pPDSrc->a[iPDSrc];
827	const uint32_t cr4 = CPUMGetGuestCR4(pVM);
828	const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
829
830	# ifdef IN_RING3
831	/*
832	* If a CR3 Sync is pending we may ignore the invalidate page operation
833	* depending on the kind of sync and if it's a global page or not.
834	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
835	*/
836	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
837	if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
838	\|\| ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
839	&& fIsBigPage
840	&& PdeSrc.b.u1Global
841	&& (cr4 & X86_CR4_PGE)
842	)
843	)
844	# else
845	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
846	# endif
847	{
848	STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
849	return VINF_SUCCESS;
850	}
851	# endif /* IN_RING3 */
852
853
854	/*
855	* Deal with the Guest PDE.
856	*/
857	int rc = VINF_SUCCESS;
858	if (PdeSrc.n.u1Present)
859	{
860	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
861	{
862	/*
863	* Conflict - Let SyncPT deal with it to avoid duplicate code.
864	*/
865	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
866	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
867	}
868	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
869	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
870	{
871	/*
872	* Mark not present so we can resync the PDE when it's used.
873	*/
874	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
875	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
876	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
877	pPdeDst->u = 0;
878	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
879	PGM_INVL_GUEST_TLBS();
880	}
881	# ifdef PGM_SYNC_ACCESSED_BIT
882	else if (!PdeSrc.n.u1Accessed)
883	{
884	/*
885	* Mark not present so we can set the accessed bit.
886	*/
887	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
888	pPdeDst->u = 0;
889	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
890	PGM_INVL_GUEST_TLBS();
891	}
892	# endif
893	else if (!fIsBigPage)
894	{
895	/*
896	* 4KB - page.
897	*/
898	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
899	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
900	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
901	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
902	# endif
903	if (pShwPage->GCPhys == GCPhys)
904	{
905	#if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
906	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
907	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
908	if (pPT->a[iPTEDst].n.u1Present)
909	{
910	# ifdef PGMPOOL_WITH_USER_TRACKING
911	/* This is very unlikely with caching/monitoring enabled. */
912	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
913	# endif
914	pPT->a[iPTEDst].u = 0;
915	}
916	#else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
917	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
918	if (VBOX_SUCCESS(rc))
919	rc = VINF_SUCCESS;
920	#endif
921	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
922	PGM_INVL_PG(GCPtrPage);
923	}
924	else
925	{
926	/*
927	* The page table address changed.
928	*/
929	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
930	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
931	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
932	pPdeDst->u = 0;
933	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
934	PGM_INVL_GUEST_TLBS();
935	}
936	}
937	else
938	{
939	/*
940	* 4MB - page.
941	*/
942	/* Before freeing the page, check if anything really changed. */
943	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
944	RTGCPHYS GCPhys = PdeSrc.u & X86_PDE4M_PG_MASK;
945	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
946	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
947	# endif
948	if ( pShwPage->GCPhys == GCPhys
949	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
950	{
951	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
952	/** @todo PAT */
953	# ifdef PGM_SYNC_DIRTY_BIT
954	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
955	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
956	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
957	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
958	# else
959	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
960	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD)))
961	# endif
962	{
963	LogFlow(("Skipping flush for big page containing %VGv (PD=%X)-> nothing has changed!\n", GCPtrPage, iPDSrc));
964	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
965	return VINF_SUCCESS;
966	}
967	}
968
969	/*
970	* Ok, the page table is present and it's been changed in the guest.
971	* If we're in host context, we'll just mark it as not present taking the lazy approach.
972	* We could do this for some flushes in GC too, but we need an algorithm for
973	* deciding which 4MB pages containing code likely to be executed very soon.
974	*/
975	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
976	pPdeDst->u = 0;
977	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
978	DUMP_PDE_BIG("PGMInvalidatePage", iPDSrc, PdeSrc);
979	PGM_INVL_BIG_PG(GCPtrPage);
980	}
981	}
982	else
983	{
984	/*
985	* Page directory is not present, mark shadow PDE not present.
986	*/
987	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
988	{
989	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
990	pPdeDst->u = 0;
991	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
992	PGM_INVL_PG(GCPtrPage);
993	}
994	else
995	{
996	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
997	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
998	}
999	}
1000
1001	return rc;
1002
1003	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1004	# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
1005	# endif
1006	return VINF_SUCCESS;
1007
1008	#elif PGM_GST_TYPE == PGM_TYPE_PAE
1009	# if PGM_SHW_TYPE == PGM_TYPE_PAE
1010	//# error not implemented
1011	return VERR_INTERNAL_ERROR;
1012
1013	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1014	# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
1015	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1016
1017	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
1018	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1019	//# error not implemented
1020	return VERR_INTERNAL_ERROR;
1021
1022	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1023	# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
1024	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1025
1026	#else /* guest real and protected mode */
1027
1028	return VINF_SUCCESS;
1029	#endif
1030	}
1031
1032
1033	#ifdef PGMPOOL_WITH_USER_TRACKING
1034	/**
1035	* Update the tracking of shadowed pages.
1036	*
1037	* @param pVM The VM handle.
1038	* @param pShwPage The shadow page.
1039	* @param HCPhys The physical page we is being dereferenced.
1040	*/
1041	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1042	{
1043	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1044	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1045	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1046
1047	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1048	* 1. have a medium sized HCPhys -> GCPhys cache (hash?)
1049	* 2. write protect all shadowed pages. I.e. implement caching.
1050	*/
1051	/*
1052	* Find the guest address.
1053	*/
1054	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTXSUFF(pRamRanges);
1055	pRam;
1056	pRam = pRam->CTXSUFF(pNext))
1057	{
1058	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1059	while (iPage-- > 0)
1060	{
1061	if ((pRam->aHCPhys[iPage] & X86_PTE_PAE_PG_MASK) == HCPhys)
1062	{
1063	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1064	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aHCPhys[iPage]);
1065	pShwPage->cPresent--;
1066	pPool->cPresent--;
1067	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1068	return;
1069	}
1070	}
1071	}
1072
1073	for (;;)
1074	AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1075	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1076	pShwPage->cPresent--;
1077	pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1078	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1079	}
1080
1081
1082	/**
1083	* Update the tracking of shadowed pages.
1084	*
1085	* @param pVM The VM handle.
1086	* @param pShwPage The shadow page.
1087	* @param u16 The top 16-bit of the *pHCPhys.
1088	* @param pHCPhys Pointer to the ram range physical page entry.
1089	* @param iPTDst The index into the shadow table.
1090	*/
1091	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PRTHCPHYS pHCPhys, const unsigned iPTDst)
1092	{
1093	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1094	/*
1095	* We're making certain assumptions about the placement of cRef and idx.
1096	*/
1097	Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1098	Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1099
1100	/*
1101	* Just deal with the simple first time here.
1102	*/
1103	if (!u16)
1104	{
1105	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1106	u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) \| pShwPage->idx;
1107	}
1108	else
1109	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1110
1111	/* write back, trying to be clever... */
1112	Log2(("SyncPageWorkerTrackAddRef: u16=%#x *pHCPhys=%VHp->%VHp iPTDst=%#x\n",
1113	u16, pHCPhys, (pHCPhys & MM_RAM_FLAGS_NO_REFS_MASK) \| ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1114	((uint16_t )pHCPhys + 3) = u16;
1115	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1116
1117	/* update statistics. */
1118	pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1119	pShwPage->cPresent++;
1120	if (pShwPage->iFirstPresent > iPTDst)
1121	pShwPage->iFirstPresent = iPTDst;
1122	}
1123	#endif /* PGMPOOL_WITH_USER_TRACKING */
1124
1125
1126	/**
1127	* Creates a 4K shadow page for a guest page.
1128	*
1129	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1130	* physical address. The PdeSrc argument only the flags are used. No page structured
1131	* will be mapped in this function.
1132	*
1133	* @param pVM VM handle.
1134	* @param pPteDst Destination page table entry.
1135	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1136	* Can safely assume that only the flags are being used.
1137	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1138	* @param pShwPage Pointer to the shadow page.
1139	* @param iPTDst The index into the shadow table.
1140	*
1141	* @remark Not used for 2/4MB pages!
1142	*/
1143	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, VBOXPDE PdeSrc, VBOXPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1144	{
1145	if (PteSrc.n.u1Present)
1146	{
1147	/*
1148	* Find the ram range.
1149	*/
1150	PRTHCPHYS pHCPhys;
1151	int rc = PGMRamGCPhys2PagePtr(&pVM->pgm.s, PteSrc.u & X86_PTE_PG_MASK, &pHCPhys);
1152	if (VBOX_SUCCESS(rc))
1153	{
1154	/** @todo investiage PWT, PCD and PAT. */
1155	/*
1156	* Make page table entry.
1157	*/
1158	const RTHCPHYS HCPhys = *pHCPhys;
1159	SHWPTE PteDst;
1160	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
1161	{
1162	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1163	if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL)))
1164	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1165	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1166	else
1167	PteDst.u = 0;
1168	/** @todo count these two kinds. */
1169	}
1170	else
1171	{
1172	#ifdef PGM_SYNC_DIRTY_BIT
1173	# ifdef PGM_SYNC_ACCESSED_BIT
1174	/*
1175	* If the page or page directory entry is not marked accessed,
1176	* we mark the page not present.
1177	*/
1178	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1179	{
1180	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1181	PteDst.u = 0;
1182	}
1183	else
1184	# endif
1185	/*
1186	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1187	* when the page is modified.
1188	*/
1189	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1190	{
1191	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1192	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1193	\| (HCPhys & X86_PTE_PAE_PG_MASK)
1194	\| PGM_PTFLAGS_TRACK_DIRTY;
1195	}
1196	else
1197	{
1198	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1199	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1200	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1201	}
1202	#endif
1203	}
1204
1205	#ifdef PGMPOOL_WITH_USER_TRACKING
1206	/*
1207	* Keep user track up to date.
1208	*/
1209	if (PteDst.n.u1Present)
1210	{
1211	if (!pPteDst->n.u1Present)
1212	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pHCPhys, iPTDst);
1213	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1214	{
1215	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1216	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1217	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pHCPhys, iPTDst);
1218	}
1219	}
1220	else if (pPteDst->n.u1Present)
1221	{
1222	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1223	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1224	}
1225	#endif /* PGMPOOL_WITH_USER_TRACKING */
1226
1227	/*
1228	* Update statistics and commit the entry.
1229	*/
1230	if (!PteSrc.n.u1Global)
1231	pShwPage->fSeenNonGlobal = true;
1232	*pPteDst = PteDst;
1233	}
1234	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1235	/** @todo count these. */
1236	}
1237	else
1238	{
1239	/*
1240	* Page not-present.
1241	*/
1242	#ifdef PGMPOOL_WITH_USER_TRACKING
1243	/* Keep user track up to date. */
1244	if (pPteDst->n.u1Present)
1245	{
1246	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1247	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1248	}
1249	#endif /* PGMPOOL_WITH_USER_TRACKING */
1250	pPteDst->u = 0;
1251	/** @todo count these. */
1252	}
1253	}
1254
1255
1256	/**
1257	* Syncs a guest OS page.
1258	*
1259	* There are no conflicts at this point, neither is there any need for
1260	* page table allocations.
1261	*
1262	* @returns VBox status code.
1263	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1264	* @param pVM VM handle.
1265	* @param PdeSrc Page directory entry of the guest.
1266	* @param GCPtrPage Guest context page address.
1267	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1268	* @param uErr Fault error (X86_TRAP_PF_*).
1269	*/
1270	PGM_BTH_DECL(int, SyncPage)(PVM pVM, VBOXPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1271	{
1272	LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1273
1274	#if PGM_GST_TYPE == PGM_TYPE_32BIT
1275
1276	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1277	# error "Invalid shadow mode for 32-bit guest mode!"
1278	# endif
1279
1280	/*
1281	* Assert preconditions.
1282	*/
1283	# if GC_ARCH_BITS != 32
1284	Assert(GCPtrPage < _4G); //???
1285	# endif
1286	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> X86_PD_SHIFT) & X86_PD_MASK]);
1287	Assert(PdeSrc.n.u1Present);
1288	Assert(cPages);
1289
1290	/*
1291	* Get the shadow PDE, find the shadow page table in the pool.
1292	*/
1293	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1294	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1295	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1296	# else /* PAE */
1297	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1298	# endif
1299	Assert(PdeDst.n.u1Present);
1300	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1301
1302	/*
1303	* Check that the page is present and that the shadow PDE isn't out of sync.
1304	*/
1305	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1306	RTGCPHYS GCPhys;
1307	if (!fBigPage)
1308	{
1309	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1310	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1311	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1312	# endif
1313	}
1314	else
1315	{
1316	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1317	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1318	GCPhys \|= GCPtrPage & X86_PAGE_2M_SIZE;
1319	# endif
1320	}
1321	if ( pShwPage->GCPhys == GCPhys
1322	&& PdeSrc.n.u1Present
1323	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1324	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1325	)
1326	{
1327	# ifdef PGM_SYNC_ACCESSED_BIT
1328	/*
1329	* Check that the PDE is marked accessed already.
1330	* Since we set the accessed bit before getting here on a #PF, this
1331	* check is only meant for dealing with non-#PF'ing paths.
1332	*/
1333	if (PdeSrc.n.u1Accessed)
1334	# endif
1335	{
1336	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1337	if (!fBigPage)
1338	{
1339	/*
1340	* 4KB Page - Map the guest page table.
1341	*/
1342	PVBOXPT pPTSrc;
1343	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & X86_PDE_PG_MASK, &pPTSrc);
1344	if (VBOX_SUCCESS(rc))
1345	{
1346	# ifdef PGM_SYNC_N_PAGES
1347	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1348	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1349	{
1350	/*
1351	* This code path is currently only taken when the caller is PGMTrap0eHandler
1352	* for non-present pages!
1353	*
1354	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1355	* deal with locality.
1356	*/
1357	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1358	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1359	const unsigned offPTSrc = 0;
1360	# else
1361	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1362	# endif
1363	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1364	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1365	iPTDst = 0;
1366	else
1367	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1368	for (; iPTDst < iPTDstEnd; iPTDst++)
1369	{
1370	if (!pPTDst->a[iPTDst].n.u1Present)
1371	{
1372	VBOXPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1373	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(X86_PT_MASK << X86_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1374	NOREF(GCPtrCurPage);
1375	#ifndef IN_RING0
1376	/*
1377	* Assuming kernel code will be marked as supervisor - and not as user level
1378	* and executed using a conforming code selector - And marked as readonly.
1379	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1380	*/
1381	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1382	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1383	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1384	\|\| PGMRamTestFlags(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK,
1385	MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE)
1386	)
1387	#endif
1388	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1389	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1390	GCPtrCurPage, PteSrc.n.u1Present,
1391	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1392	PteSrc.n.u1User & PdeSrc.n.u1User,
1393	(uint64_t)PteSrc.u,
1394	(uint64_t)pPTDst->a[iPTDst].u,
1395	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1396	}
1397	}
1398	}
1399	else
1400	# endif /* PGM_SYNC_N_PAGES */
1401	{
1402	const unsigned iPTSrc = (GCPtrPage >> X86_PT_SHIFT) & X86_PT_MASK;
1403	VBOXPTE PteSrc = pPTSrc->a[iPTSrc];
1404	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1405	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1406	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1407	GCPtrPage, PteSrc.n.u1Present,
1408	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1409	PteSrc.n.u1User & PdeSrc.n.u1User,
1410	(uint64_t)PteSrc.u,
1411	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1412	}
1413	}
1414	else /* MMIO or invalid page: emulated in #PF handler. */
1415	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1416	}
1417	else
1418	{
1419	/*
1420	* 4/2MB page - lazy syncing shadow 4K pages.
1421	* (There are many causes of getting here, it's no longer only CSAM.)
1422	*/
1423	/* Calculate the GC physical address of this 4KB shadow page. */
1424	RTGCPHYS GCPhys = (PdeSrc.u & X86_PDE4M_PAE_PG_MASK) \| ((RTGCUINTPTR)GCPtrPage & PAGE_OFFSET_MASK_BIG);
1425	/* Find ram range. */
1426	PRTHCPHYS pHCPhys;
1427	int rc = PGMRamGCPhys2PagePtr(&pVM->pgm.s, GCPhys, &pHCPhys);
1428	if (VBOX_SUCCESS(rc))
1429	{
1430	/*
1431	* Make shadow PTE entry.
1432	*/
1433	RTHCPHYS HCPhys = *pHCPhys;
1434	SHWPTE PteDst;
1435	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1436	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1437	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
1438	{
1439	if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL)))
1440	PteDst.n.u1Write = 0;
1441	else
1442	PteDst.u = 0;
1443	}
1444	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1445	# ifdef PGMPOOL_WITH_USER_TRACKING
1446	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1447	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pHCPhys, iPTDst);
1448	# endif
1449	pPTDst->a[iPTDst] = PteDst;
1450
1451
1452	# ifdef PGM_SYNC_DIRTY_BIT
1453	/*
1454	* If the page is not flagged as dirty and is writable, then make it read-only
1455	* at PD level, so we can set the dirty bit when the page is modified.
1456	*
1457	* ASSUMES that page access handlers are implemented on page table entry level.
1458	* Thus we will first catch the dirty access and set PDE.D and restart. If
1459	* there is an access handler, we'll trap again and let it work on the problem.
1460	*/
1461	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1462	* As for invlpg, it simply frees the whole shadow PT.
1463	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1464	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1465	{
1466	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1467	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1468	PdeDst.n.u1Write = 0;
1469	}
1470	else
1471	{
1472	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1473	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1474	}
1475	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1476	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1477	# else /* PAE */
1478	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1479	# endif
1480	# endif /* PGM_SYNC_DIRTY_BIT */
1481	Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1482	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1483	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1484	}
1485	}
1486	return VINF_SUCCESS;
1487	}
1488	# ifdef PGM_SYNC_ACCESSED_BIT
1489	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1490	#endif
1491	}
1492	else
1493	{
1494	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1495	Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1496	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1497	}
1498
1499	/*
1500	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1501	* Yea, I'm lazy.
1502	*/
1503	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1504	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1505	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1506	# else /* PAE */
1507	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1508	# endif
1509	PGM_INVL_GUEST_TLBS();
1510	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1511
1512	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1513
1514	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1515	return VERR_INTERNAL_ERROR;
1516	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1517	}
1518
1519
1520
1521	#ifdef PGM_SYNC_DIRTY_BIT
1522
1523	/**
1524	* Investigate page fault and handle write protection page faults caused by
1525	* dirty bit tracking.
1526	*
1527	* @returns VBox status code.
1528	* @param pVM VM handle.
1529	* @param uErr Page fault error code.
1530	* @param pPdeDst Shadow page directory entry.
1531	* @param pPdeSrc Guest page directory entry.
1532	* @param GCPtrPage Guest context page address.
1533	*/
1534	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PVBOXPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1535	{
1536	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1537	LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1538
1539	/*
1540	* Real page fault?
1541	*/
1542	if ( (uErr & X86_TRAP_PF_RSVD)
1543	\|\| !pPdeSrc->n.u1Present
1544	\|\| ((uErr & X86_TRAP_PF_RW) && !pPdeSrc->n.u1Write)
1545	\|\| ((uErr & X86_TRAP_PF_US) && !pPdeSrc->n.u1User) )
1546	{
1547	#ifdef IN_GC
1548	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1549	#endif
1550	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1551	LogFlow(("CheckPageFault: real page fault at %VGv (1)\n", GCPtrPage));
1552
1553	if (pPdeSrc->n.u1Present)
1554	{
1555	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1556	* See the 2nd case below as well.
1557	*/
1558	if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1559	{
1560	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1561	}
1562	else
1563	{
1564	/*
1565	* Map the guest page table.
1566	*/
1567	PVBOXPT pPTSrc;
1568	int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & X86_PDE_PG_MASK, &pPTSrc);
1569	if (VBOX_SUCCESS(rc))
1570	{
1571	PVBOXPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & PTE_MASK];
1572	const VBOXPTE PteSrc = *pPteSrc;
1573	if (pPteSrc->n.u1Present)
1574	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1575	}
1576	AssertRC(rc);
1577	}
1578	}
1579	return VINF_EM_RAW_GUEST_TRAP;
1580	}
1581
1582	/*
1583	* First check the easy case where the page directory has been marked read-only to track
1584	* the dirty bit of an emulated BIG page
1585	*/
1586	if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1587	{
1588	/* Mark guest page directory as accessed */
1589	pPdeSrc->b.u1Accessed = 1;
1590
1591	/*
1592	* Only write protection page faults are relevant here.
1593	*/
1594	if (uErr & X86_TRAP_PF_RW)
1595	{
1596	/* Mark guest page directory as dirty (BIG page only). */
1597	pPdeSrc->b.u1Dirty = 1;
1598
1599	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1600	{
1601	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1602
1603	Assert(pPdeSrc->b.u1Write);
1604
1605	pPdeDst->n.u1Write = 1;
1606	pPdeDst->n.u1Accessed = 1;
1607	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1608	PGM_INVL_BIG_PG(GCPtrPage);
1609	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1610	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1611	}
1612	}
1613	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1614	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1615	}
1616	/* else: 4KB page table */
1617
1618	/*
1619	* Map the guest page table.
1620	*/
1621	PVBOXPT pPTSrc;
1622	int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & X86_PDE_PG_MASK, &pPTSrc);
1623	if (VBOX_SUCCESS(rc))
1624	{
1625	/*
1626	* Real page fault?
1627	*/
1628	PVBOXPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & PTE_MASK];
1629	const VBOXPTE PteSrc = *pPteSrc;
1630	if ( !PteSrc.n.u1Present
1631	\|\| ((uErr & X86_TRAP_PF_RW) && !PteSrc.n.u1Write)
1632	\|\| ((uErr & X86_TRAP_PF_US) && !PteSrc.n.u1User)
1633	)
1634	{
1635	#ifdef IN_GC
1636	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1637	#endif
1638	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1639	LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1640
1641	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1642	* See the 2nd case above as well.
1643	*/
1644	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1645	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1646
1647	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1648	return VINF_EM_RAW_GUEST_TRAP;
1649	}
1650	LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1651
1652	/*
1653	* Set the accessed bits in the page directory and the page table.
1654	*/
1655	pPdeSrc->n.u1Accessed = 1;
1656	pPteSrc->n.u1Accessed = 1;
1657
1658	/*
1659	* Only write protection page faults are relevant here.
1660	*/
1661	if (uErr & X86_TRAP_PF_RW)
1662	{
1663	/* Write access, so mark guest entry as dirty. */
1664	#if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1665	if (!pPteSrc->n.u1Dirty)
1666	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1667	else
1668	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1669	#endif
1670	pPteSrc->n.u1Dirty = 1;
1671
1672	if (pPdeDst->n.u1Present)
1673	{
1674	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1675	* Our individual shadow handlers will provide more information and force a fatal exit.
1676	*/
1677	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1678	{
1679	LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1680	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1681	return VINF_SUCCESS;
1682	}
1683
1684	/*
1685	* Map shadow page table.
1686	*/
1687	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1688	if (pShwPage)
1689	{
1690	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1691	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1692	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1693	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1694	{
1695	LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1696	#ifdef VBOX_STRICT
1697	RTHCPHYS HCPhys;
1698	rc = PGMRamGCPhys2HCPhysWithFlags(&pVM->pgm.s, pPteSrc->u & X86_PTE_PG_MASK, &HCPhys);
1699	if (VBOX_SUCCESS(rc))
1700	AssertMsg(!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE)),
1701	("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1702	#endif
1703	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1704
1705	Assert(pPteSrc->n.u1Write);
1706
1707	pPteDst->n.u1Write = 1;
1708	pPteDst->n.u1Dirty = 1;
1709	pPteDst->n.u1Accessed = 1;
1710	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1711	PGM_INVL_PG(GCPtrPage);
1712
1713	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1714	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1715	}
1716	}
1717	else
1718	AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
1719	}
1720	}
1721	/** @todo Optimize accessed bit emulation? */
1722	#ifdef VBOX_STRICT
1723	/*
1724	* Sanity check.
1725	*/
1726	else if ( !pPteSrc->n.u1Dirty
1727	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
1728	&& pPdeDst->n.u1Present)
1729	{
1730	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1731	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1732	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1733	if ( pPteDst->n.u1Present
1734	&& pPteDst->n.u1Write)
1735	LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
1736	}
1737	#endif /* VBOX_STRICT */
1738	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1739	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1740	}
1741	AssertRC(rc);
1742	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1743	return rc;
1744	}
1745
1746	#endif
1747
1748
1749	/**
1750	* Sync a shadow page table.
1751	*
1752	* The shadow page table is not present. This includes the case where
1753	* there is a conflict with a mapping.
1754	*
1755	* @returns VBox status code.
1756	* @param pVM VM handle.
1757	* @param iPD Page directory index.
1758	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
1759	* Assume this is a temporary mapping.
1760	* @param GCPtrPage GC Pointer of the page that caused the fault
1761	*/
1762	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PVBOXPD pPDSrc, RTGCUINTPTR GCPtrPage)
1763	{
1764	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1765	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
1766	LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
1767
1768	#if PGM_GST_TYPE == PGM_TYPE_32BIT
1769
1770	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1771	# error "Invalid shadow mode for 32-bit guest mode!"
1772	# endif
1773
1774	/*
1775	* Validate input a little bit.
1776	*/
1777	Assert(iPDSrc == (GCPtrPage >> GST_PD_SHIFT));
1778	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1779	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
1780	# else
1781	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
1782	# endif
1783	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1784	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
1785	SHWPDE PdeDst = *pPdeDst;
1786
1787	/*
1788	* Check for conflicts.
1789	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
1790	* HC: Simply resolve the conflict.
1791	*/
1792	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1793	{
1794	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1795	# ifndef IN_RING3
1796	Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
1797	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1798	return VERR_ADDRESS_CONFLICT;
1799	# else
1800	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
1801	Assert(pMapping);
1802	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPDSrc);
1803	if (VBOX_FAILURE(rc))
1804	{
1805	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1806	return rc;
1807	}
1808	PdeDst = *pPdeDst;
1809	# endif
1810	}
1811	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
1812
1813	/*
1814	* Sync page directory entry.
1815	*/
1816	int rc = VINF_SUCCESS;
1817	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1818	if (PdeSrc.n.u1Present)
1819	{
1820	/*
1821	* Allocate & map the page table.
1822	*/
1823	PSHWPT pPTDst;
1824	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1825	PPGMPOOLPAGE pShwPage;
1826	RTGCPHYS GCPhys;
1827	if (fPageTable)
1828	{
1829	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1830	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1831	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1832	# endif
1833	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1834	}
1835	else
1836	{
1837	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1838	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1839	GCPhys \|= GCPtrPage & BIT(X86_PAGE_2M_SHIFT);
1840	# endif
1841	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1842	}
1843	if (rc == VINF_SUCCESS)
1844	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1845	else if (rc == VINF_PGM_CACHED_PAGE)
1846	{
1847	/*
1848	* The PT was cached, just hook it up.
1849	*/
1850	if (fPageTable)
1851	PdeDst.u = pShwPage->Core.Key
1852	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1853	else
1854	{
1855	PdeDst.u = pShwPage->Core.Key
1856	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1857	# ifdef PGM_SYNC_DIRTY_BIT /* (see explanation and assumtions further down.) */
1858	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1859	{
1860	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1861	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1862	PdeDst.b.u1Write = 0;
1863	}
1864	# endif
1865	}
1866	*pPdeDst = PdeDst;
1867	return VINF_SUCCESS;
1868	}
1869	else if (rc == VERR_PGM_POOL_FLUSHED)
1870	return VINF_PGM_SYNC_CR3;
1871	else
1872	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
1873	PdeDst.u &= X86_PDE_AVL_MASK;
1874	PdeDst.u \|= pShwPage->Core.Key;
1875
1876	# ifdef PGM_SYNC_DIRTY_BIT
1877	/*
1878	* Page directory has been accessed (this is a fault situation, remember).
1879	*/
1880	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
1881	# endif
1882
1883	if (fPageTable)
1884	{
1885	/*
1886	* Page table - 4KB.
1887	*
1888	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
1889	*/
1890	Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
1891	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
1892	PGSTPT pPTSrc;
1893	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1894	if (VBOX_SUCCESS(rc))
1895	{
1896	/*
1897	* Start by syncing the page directory entry so CSAM's TLB trick works.
1898	*/
1899	PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK))
1900	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1901	*pPdeDst = PdeDst;
1902
1903	/*
1904	* Directory/page user or supervisor privilege: (same goes for read/write)
1905	*
1906	* Directory Page Combined
1907	* U/S U/S U/S
1908	* 0 0 0
1909	* 0 1 0
1910	* 1 0 0
1911	* 1 1 1
1912	*
1913	* Simple AND operation. Table listed for completeness.
1914	*
1915	*/
1916	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
1917	# ifdef PGM_SYNC_N_PAGES
1918	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1919	unsigned iPTDst = iPTBase;
1920	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1921	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
1922	iPTDst = 0;
1923	else
1924	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1925	# else /* !PGM_SYNC_N_PAGES */
1926	unsigned iPTDst = 0;
1927	const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
1928	# endif /* !PGM_SYNC_N_PAGES */
1929	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1930	const unsigned offPTSrc = 0;
1931	# else
1932	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1933	# endif
1934	for (; iPTDst < iPTDstEnd; iPTDst++)
1935	{
1936	const unsigned iPTSrc = iPTDst + offPTSrc;
1937	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1938
1939	if (PteSrc.n.u1Present) /* we've already cleared it above */
1940	{
1941	#ifndef IN_RING0
1942	/*
1943	* Assuming kernel code will be marked as supervisor - and not as user level
1944	* and executed using a conforming code selector - And marked as readonly.
1945	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1946	*/
1947	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
1948	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
1949	\|\| PGMRamTestFlags(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK,
1950	MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE)
1951	)
1952	#endif
1953	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1954	Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
1955	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
1956	PteSrc.n.u1Present,
1957	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1958	PteSrc.n.u1User & PdeSrc.n.u1User,
1959	(uint64_t)PteSrc.u,
1960	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
1961	(PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
1962	}
1963	} /* for PTEs */
1964	}
1965	}
1966	else
1967	{
1968	/*
1969	* Big page - 2/4MB.
1970	*
1971	* We'll walk the ram range list in parallel and optimize lookups.
1972	* We will only sync on shadow page table at a time.
1973	*/
1974	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
1975
1976	/**
1977	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
1978	*/
1979
1980	/*
1981	* Start by syncing the page directory entry.
1982	*/
1983	PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
1984	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1985
1986	# ifdef PGM_SYNC_DIRTY_BIT
1987	/*
1988	* If the page is not flagged as dirty and is writable, then make it read-only
1989	* at PD level, so we can set the dirty bit when the page is modified.
1990	*
1991	* ASSUMES that page access handlers are implemented on page table entry level.
1992	* Thus we will first catch the dirty access and set PDE.D and restart. If
1993	* there is an access handler, we'll trap again and let it work on the problem.
1994	*/
1995	/** @todo move the above stuff to a section in the PGM documentation. */
1996	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
1997	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1998	{
1999	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2000	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2001	PdeDst.b.u1Write = 0;
2002	}
2003	# endif /* PGM_SYNC_DIRTY_BIT */
2004	*pPdeDst = PdeDst;
2005
2006	/*
2007	* Fill the shadow page table.
2008	*/
2009	/* Get address and flags from the source PDE. */
2010	SHWPTE PteDstBase;
2011	PteDstBase.u = PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2012
2013	/* Loop thru the entries in the shadow PT. */
2014	const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2015	Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2016	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2017	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2018	PPGMRAMRANGE pRam = CTXSUFF(pVM->pgm.s.pRamRanges);
2019	unsigned iPTDst = 0;
2020	while (iPTDst < ELEMENTS(pPTDst->a))
2021	{
2022	/* Advance ram range list. */
2023	while (pRam && GCPhys > pRam->GCPhysLast)
2024	pRam = CTXSUFF(pRam->pNext);
2025	if (pRam && GCPhys >= pRam->GCPhys)
2026	{
2027	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2028	do
2029	{
2030	/* Make shadow PTE. */
2031	RTHCPHYS HCPhys = pRam->aHCPhys[iHCPage];
2032	SHWPTE PteDst;
2033
2034	/* Make sure the RAM has already been allocated. */
2035	if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC)
2036	{
2037	if (RT_UNLIKELY(!(pRam->aHCPhys[iHCPage] & X86_PTE_PAE_PG_MASK)))
2038	{
2039	#ifdef IN_RING3
2040	int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2041	#else
2042	int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2043	#endif
2044	if (rc != VINF_SUCCESS)
2045	return rc;
2046
2047	HCPhys = pRam->aHCPhys[iHCPage];
2048	}
2049	}
2050
2051	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
2052	{
2053	if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL)))
2054	{
2055	PteDst.u = (HCPhys & X86_PTE_PAE_PG_MASK) \| PteDstBase.u;
2056	PteDst.n.u1Write = 0;
2057	}
2058	else
2059	PteDst.u = 0;
2060	}
2061	#ifndef IN_RING0
2062	/*
2063	* Assuming kernel code will be marked as supervisor and not as user level and executed
2064	* using a conforming code selector. Don't check for readonly, as that implies the whole
2065	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2066	*/
2067	else if ( !PdeSrc.n.u1User
2068	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2069	PteDst.u = 0;
2070	#endif
2071	else
2072	PteDst.u = (HCPhys & X86_PTE_PAE_PG_MASK) \| PteDstBase.u;
2073	# ifdef PGMPOOL_WITH_USER_TRACKING
2074	if (PteDst.n.u1Present)
2075	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, &pRam->aHCPhys[iHCPage], iPTDst);
2076	# endif
2077	/* commit it */
2078	pPTDst->a[iPTDst] = PteDst;
2079	Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2080	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2081	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2082
2083	/* advance */
2084	GCPhys += PAGE_SIZE;
2085	iHCPage++;
2086	iPTDst++;
2087	} while ( iPTDst < ELEMENTS(pPTDst->a)
2088	&& GCPhys <= pRam->GCPhysLast);
2089	}
2090	else if (pRam)
2091	{
2092	Log(("Invalid pages at %VGp\n", GCPhys));
2093	do
2094	{
2095	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2096	GCPhys += PAGE_SIZE;
2097	iPTDst++;
2098	} while ( iPTDst < ELEMENTS(pPTDst->a)
2099	&& GCPhys < pRam->GCPhys);
2100	}
2101	else
2102	{
2103	Log(("Invalid pages at %VGp (2)\n", GCPhys));
2104	for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2105	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2106	}
2107	} /* while more PTEs */
2108	} /* 4KB / 4MB */
2109	}
2110	else
2111	AssertRelease(!PdeDst.n.u1Present);
2112
2113	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2114	# ifdef IN_GC
2115	if (VBOX_FAILURE(rc))
2116	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2117	# endif
2118	return rc;
2119
2120	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2121
2122	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2123	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2124	return VERR_INTERNAL_ERROR;
2125	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2126	}
2127
2128
2129
2130	/**
2131	* Prefetch a page/set of pages.
2132	*
2133	* Typically used to sync commonly used pages before entering raw mode
2134	* after a CR3 reload.
2135	*
2136	* @returns VBox status code.
2137	* @param pVM VM handle.
2138	* @param GCPtrPage Page to invalidate.
2139	*/
2140	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2141	{
2142	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2143
2144	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2145	# error "Invalid shadow mode for 32-bit guest mode!"
2146	# endif
2147
2148	/*
2149	* Check that all Guest levels thru the PDE are present, getting the
2150	* PD and PDE in the processes.
2151	*/
2152	int rc = VINF_SUCCESS;
2153	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2154	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
2155	const VBOXPDE PdeSrc = pPDSrc->a[iPDSrc];
2156	# ifdef PGM_SYNC_ACCESSED_BIT
2157	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2158	# else
2159	if (PdeSrc.n.u1Present)
2160	# endif
2161	{
2162	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2163	const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> X86_PD_SHIFT];
2164	# else
2165	const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> X86_PD_PAE_SHIFT];
2166	# endif
2167	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2168	{
2169	if (!PdeDst.n.u1Present)
2170	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2171	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2172	else
2173	{
2174	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2175	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2176	* makes no sense to prefetch more than one page.
2177	*/
2178	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2179	if (VBOX_SUCCESS(rc))
2180	rc = VINF_SUCCESS;
2181	}
2182	}
2183	}
2184	return rc;
2185
2186	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2187
2188	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2189	return VERR_INTERNAL_ERROR;
2190	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2191	}
2192
2193
2194
2195
2196	/**
2197	* Syncs a page during a PGMVerifyAccess() call.
2198	*
2199	* @returns VBox status code (informational included).
2200	* @param GCPtrPage The address of the page to sync.
2201	* @param fPage The effective guest page flags.
2202	* @param uErr The trap error code.
2203	*/
2204	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2205	{
2206	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2207
2208	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2209
2210	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2211	# error "Invalid shadow mode for 32-bit guest mode!"
2212	# endif
2213
2214	#ifndef IN_RING0
2215	if (!(fPage & X86_PTE_US))
2216	{
2217	/*
2218	* Mark this page as safe.
2219	*/
2220	/** @todo not correct for pages that contain both code and data!! */
2221	Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2222	CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2223	}
2224	#endif
2225	/*
2226	* Get guest PD and index.
2227	*/
2228	unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
2229	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2230	int rc = VINF_SUCCESS;
2231
2232	# ifdef PGM_SYNC_DIRTY_BIT
2233	/*
2234	* First check if the page fault was caused by dirty bit tracking
2235	*/
2236	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2237	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> X86_PD_SHIFT];
2238	# else
2239	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> X86_PD_PAE_SHIFT];
2240	# endif
2241	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2242	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2243	Log(("PGMVerifyAccess: success (dirty)\n"));
2244	else
2245	# endif /* PGM_SYNC_DIRTY_BIT */
2246	{
2247	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2248	if (uErr & X86_TRAP_PF_US)
2249	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2250	else /* supervisor */
2251	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2252
2253	rc = PGM_BTH_NAME(SyncPage)(pVM, pPDSrc->a[iPDSrc], GCPtrPage, 1, 0);
2254	if (VBOX_SUCCESS(rc))
2255	{
2256	/* Page was successfully synced */
2257	Log(("PGMVerifyAccess: success (sync)\n"));
2258	rc = VINF_SUCCESS;
2259	}
2260	else
2261	{
2262	Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2263	return VINF_EM_RAW_GUEST_TRAP;
2264	}
2265	}
2266	return rc;
2267
2268	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2269
2270	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2271	return VERR_INTERNAL_ERROR;
2272	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2273	}
2274
2275
2276	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2277	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2278	/**
2279	* Figures out which kind of shadow page this guest PDE warrants.
2280	*
2281	* @returns Shadow page kind.
2282	* @param pPdeSrc The guest PDE in question.
2283	* @param cr4 The current guest cr4 value.
2284	*/
2285	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const VBOXPDE *pPdeSrc, uint32_t cr4)
2286	{
2287	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
2288	return BTH_PGMPOOLKIND_PT_FOR_PT;
2289	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
2290	//{
2291	// case 0:
2292	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2293	// case X86_PDE4M_RW:
2294	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2295	// case X86_PDE4M_US:
2296	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2297	// case X86_PDE4M_RW \| X86_PDE4M_US:
2298	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2299	# if 0
2300	// case X86_PDE4M_PAE_NX:
2301	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2302	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
2303	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2304	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2305	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2306	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2307	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2308	# endif
2309	return BTH_PGMPOOLKIND_PT_FOR_BIG;
2310	//}
2311	}
2312	# endif
2313	#endif
2314
2315	#undef MY_STAM_COUNTER_INC
2316	#define MY_STAM_COUNTER_INC(a) do { } while (0)
2317
2318
2319	/**
2320	* Syncs the paging hierarchy starting at CR3.
2321	*
2322	* @returns VBox status code, no specials.
2323	* @param pVM The virtual machine.
2324	* @param cr0 Guest context CR0 register
2325	* @param cr3 Guest context CR3 register
2326	* @param cr4 Guest context CR4 register
2327	* @param fGlobal Including global page directories or not
2328	*/
2329	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal)
2330	{
2331	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2332	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2333	/*
2334	* Inform the PGM PD Cache Manager about the pending sync.
2335	*/
2336	if (fGlobal \|\| VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2337	{
2338	# if 0 /** @todo what the heck is this about? */
2339	/* Don't cause an additional global CR3 reload the next time (the flag is cleared in PGMSyncCR3). */
2340	VM_FF_CLEAR(pVM, VM_FF_PGM_SYNC_CR3);
2341	# endif
2342
2343	/* Change this CR3 reload to be a global one. */
2344	fGlobal = true;
2345	}
2346	# endif
2347	#endif
2348
2349	/*
2350	* Update page access handlers.
2351	* The virtual are always flushed, while the physical are only on demand.
2352	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2353	* have to look into that later because it will have a bad influence on the performance.
2354	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
2355	* bird: Yes, but that won't work for aliases.
2356	*/
2357	/** @todo this MUST go away. See #1557. */
2358	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2359	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2360	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2361
2362	#ifdef PGMPOOL_WITH_MONITORING
2363	/*
2364	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2365	* Occationally we will have to clear all the shadow page tables because we wanted
2366	* to monitor a page which was mapped by too many shadowed page tables. This operation
2367	* sometimes refered to as a 'lightweight flush'.
2368	*/
2369	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2370	pgmPoolMonitorModifiedClearAll(pVM);
2371	else
2372	{
2373	# ifdef IN_RING3
2374	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2375	pgmPoolClearAll(pVM);
2376	# else
2377	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2378	return VINF_PGM_SYNC_CR3;
2379	# endif
2380	}
2381	#endif
2382
2383	Assert(fGlobal \|\| (cr4 & X86_CR4_PGE));
2384	MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2385
2386	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2387	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2388	/*
2389	* Get page directory addresses.
2390	*/
2391	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2392	PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2393	# else
2394	PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2395	# endif
2396	PVBOXPD pPDSrc = pVM->pgm.s.CTXSUFF(pGuestPD);
2397
2398	Assert(pPDSrc);
2399	#ifndef IN_GC
2400	Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2401	#endif
2402
2403	/*
2404	* Iterate the page directory.
2405	*/
2406	PPGMMAPPING pMapping;
2407	unsigned iPdNoMapping;
2408	const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2409	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2410
2411	/* Only check mappings if they are supposed to be put into the shadow page table. */
2412	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2413	{
2414	pMapping = pVM->pgm.s.CTXSUFF(pMappings);
2415	iPdNoMapping = (pMapping) ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2416	}
2417	else
2418	{
2419	pMapping = 0;
2420	iPdNoMapping = ~0U;
2421	}
2422
2423	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2424	{
2425	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2426	Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2427	# else
2428	Assert(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst);
2429	# endif
2430	register VBOXPDE PdeSrc = pPDSrc->a[iPD];
2431	if ( PdeSrc.n.u1Present
2432	&& (PdeSrc.n.u1User \|\| fRawR0Enabled))
2433	{
2434	/*
2435	* Check for conflicts with GC mappings.
2436	*/
2437	if (iPD == iPdNoMapping)
2438	{
2439	if (pVM->pgm.s.fMappingsFixed)
2440	{
2441	/* It's fixed, just skip the mapping. */
2442	const unsigned cPTs = pMapping->cPTs;
2443	iPD += cPTs - 1;
2444	pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2445	pMapping = pMapping->CTXSUFF(pNext);
2446	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2447	continue;
2448	}
2449
2450	#ifdef IN_RING3
2451	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2452	if (VBOX_FAILURE(rc))
2453	return rc;
2454
2455	/*
2456	* Update iPdNoMapping and pMapping.
2457	*/
2458	pMapping = pVM->pgm.s.pMappingsHC;
2459	while (pMapping && pMapping->GCPtr < (iPD << PGDIR_SHIFT))
2460	pMapping = pMapping->pNextHC;
2461	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2462	#else
2463	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2464	return VINF_PGM_SYNC_CR3;
2465	#endif
2466	}
2467
2468	/*
2469	* Sync page directory entry.
2470	*
2471	* The current approach is to allocated the page table but to set
2472	* the entry to not-present and postpone the page table synching till
2473	* it's actually used.
2474	*/
2475	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2476	const unsigned iPdShw = iPD; NOREF(iPdShw);
2477	# else
2478	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2479	# endif
2480	{
2481	SHWPDE PdeDst = *pPDEDst;
2482	if (PdeDst.n.u1Present)
2483	{
2484	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2485	RTGCPHYS GCPhys;
2486	if ( !PdeSrc.b.u1Size
2487	\|\| !(cr4 & X86_CR4_PSE))
2488	{
2489	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2490	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2491	GCPhys \|= i * (PAGE_SIZE / 2);
2492	# endif
2493	}
2494	else
2495	{
2496	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
2497	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2498	GCPhys \|= i * X86_PAGE_2M_SIZE;
2499	# endif
2500	}
2501
2502	if ( pShwPage->GCPhys == GCPhys
2503	&& pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2504	&& ( pShwPage->fCached
2505	\|\| ( !fGlobal
2506	&& ( false
2507	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2508	\|\| ( (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2509	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE)) /* global 2/4MB page. */
2510	\|\| ( !pShwPage->fSeenNonGlobal
2511	&& (cr4 & X86_CR4_PGE))
2512	# endif
2513	)
2514	)
2515	)
2516	&& ( (PdeSrc.u & (X86_PDE_US \| X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US \| X86_PDE_RW))
2517	\|\| ( (cr4 & X86_CR4_PSE)
2518	&& ((PdeSrc.u & (X86_PDE_US \| X86_PDE4M_PS \| X86_PDE4M_D)) \| PGM_PDFLAGS_TRACK_DIRTY)
2519	== ((PdeDst.u & (X86_PDE_US \| X86_PDE_RW \| PGM_PDFLAGS_TRACK_DIRTY)) \| X86_PDE4M_PS))
2520	)
2521	)
2522	{
2523	# ifdef VBOX_WITH_STATISTICS
2524	if ( !fGlobal
2525	&& (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2526	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE))
2527	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2528	else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2529	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2530	else
2531	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2532	# endif /* VBOX_WITH_STATISTICS */
2533	/** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2534	* The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2535	//# ifdef PGMPOOL_WITH_CACHE
2536	// pgmPoolCacheUsed(pPool, pShwPage);
2537	//# endif
2538	}
2539	else
2540	{
2541	pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
2542	pPDEDst->u = 0;
2543	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
2544	}
2545	}
2546	else
2547	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
2548	pPDEDst++;
2549	}
2550	}
2551	else if (iPD != iPdNoMapping)
2552	{
2553	/*
2554	* Check if there is any page directory to mark not present here.
2555	*/
2556	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2557	const unsigned iPdShw = iPD; NOREF(iPdShw);
2558	# else
2559	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2560	# endif
2561	{
2562	if (pPDEDst->n.u1Present)
2563	{
2564	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
2565	pPDEDst->u = 0;
2566	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
2567	}
2568	pPDEDst++;
2569	}
2570	}
2571	else
2572	{
2573	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2574	const unsigned cPTs = pMapping->cPTs;
2575	if (pVM->pgm.s.fMappingsFixed)
2576	{
2577	/* It's fixed, just skip the mapping. */
2578	pMapping = pMapping->CTXSUFF(pNext);
2579	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2580	}
2581	else
2582	{
2583	/*
2584	* Check for conflicts for subsequent pagetables
2585	* and advance to the next mapping.
2586	*/
2587	iPdNoMapping = ~0U;
2588	unsigned iPT = cPTs;
2589	while (iPT-- > 1)
2590	{
2591	if ( pPDSrc->a[iPD + iPT].n.u1Present
2592	&& (pPDSrc->a[iPD + iPT].n.u1User \|\| fRawR0Enabled))
2593	{
2594	# ifdef IN_RING3
2595	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2596	if (VBOX_FAILURE(rc))
2597	return rc;
2598
2599	/*
2600	* Update iPdNoMapping and pMapping.
2601	*/
2602	pMapping = pVM->pgm.s.CTXSUFF(pMappings);
2603	while (pMapping && pMapping->GCPtr < (iPD << PGDIR_SHIFT))
2604	pMapping = pMapping->CTXSUFF(pNext);
2605	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2606	break;
2607	# else
2608	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2609	return VINF_PGM_SYNC_CR3;
2610	# endif
2611	}
2612	}
2613	if (iPdNoMapping == ~0U && pMapping)
2614	{
2615	pMapping = pMapping->CTXSUFF(pNext);
2616	if (pMapping)
2617	iPdNoMapping = pMapping->GCPtr >> PGDIR_SHIFT;
2618	}
2619	}
2620	/* advance. */
2621	iPD += cPTs - 1;
2622	pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2623	}
2624
2625	} /* for iPD */
2626	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2627	# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
2628	# endif
2629
2630	return VINF_SUCCESS;
2631
2632	#elif PGM_GST_TYPE == PGM_TYPE_PAE
2633	# if PGM_SHW_TYPE == PGM_TYPE_PAE
2634	//# error not implemented
2635	return VERR_INTERNAL_ERROR;
2636
2637	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2638	# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
2639	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2640
2641	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
2642	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
2643	//# error not implemented
2644	return VERR_INTERNAL_ERROR;
2645
2646	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2647	# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
2648	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2649
2650	#else /* guest real and protected mode */
2651
2652	return VINF_SUCCESS;
2653	#endif
2654	}
2655
2656
2657
2658
2659	#ifdef VBOX_STRICT
2660	#ifdef IN_GC
2661	# undef AssertMsgFailed
2662	# define AssertMsgFailed Log
2663	#endif
2664	#ifdef IN_RING3
2665	# include <VBox/dbgf.h>
2666
2667	/**
2668	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
2669	*
2670	* @returns VBox status code (VINF_SUCCESS).
2671	* @param pVM The VM handle.
2672	* @param cr3 The root of the hierarchy.
2673	* @param crr The cr4, only PAE and PSE is currently used.
2674	* @param fLongMode Set if long mode, false if not long mode.
2675	* @param cMaxDepth Number of levels to dump.
2676	* @param pHlp Pointer to the output functions.
2677	*/
2678	__BEGIN_DECLS
2679	PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
2680	__END_DECLS
2681
2682	#endif
2683
2684	/**
2685	* Checks that the shadow page table is in sync with the guest one.
2686	*
2687	* @returns The number of errors.
2688	* @param pVM The virtual machine.
2689	* @param cr3 Guest context CR3 register
2690	* @param cr4 Guest context CR4 register
2691	* @param GCPtr Where to start. Defaults to 0.
2692	* @param cb How much to check. Defaults to everything.
2693	*/
2694	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
2695	{
2696	unsigned cErrors = 0;
2697
2698	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2699
2700	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2701	# error "Invalid shadow mode for 32-bit guest paging."
2702	# endif
2703
2704	PPGM pPGM = &pVM->pgm.s;
2705	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
2706	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
2707	RTHCPHYS HCPhys; /* general usage. */
2708	int rc;
2709
2710	/*
2711	* Check that the Guest CR3 and all it's mappings are correct.
2712	*/
2713	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & X86_CR3_PAGE_MASK),
2714	("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
2715	false);
2716	rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
2717	AssertRCReturn(rc, 1);
2718	rc = PGMRamGCPhys2HCPhys(pPGM, cr3 & X86_CR3_PAGE_MASK, &HCPhys);
2719	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
2720	# ifndef IN_GC
2721	RTGCPHYS GCPhys;
2722	rc = PGMPhysHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
2723	AssertRCReturn(rc, 1);
2724	AssertMsgReturn((cr3 & X86_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
2725	# endif
2726	const X86PD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
2727
2728	/*
2729	* Get and check the Shadow CR3.
2730	*/
2731	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2732	const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
2733	unsigned cPDEs = ELEMENTS(pPDDst->a);
2734	# else
2735	const X86PDPAE pPDDst = pPGM->CTXMID(ap,PaePDs[0]); / use it as a 2048 entry PD */
2736	unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
2737	# endif
2738	if (cb != ~(RTGCUINTPTR)0)
2739	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
2740
2741	/** @todo call the other two PGMAssert() functions. /
2742
2743	/*
2744	* Iterate the shadow page directory.
2745	*/
2746	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
2747	unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
2748	cPDEs += iPDDst;
2749	for (;
2750	iPDDst < cPDEs;
2751	iPDDst++, GCPtr += _4G / cPDEs)
2752	{
2753	const SHWPDE PdeDst = pPDDst->a[iPDDst];
2754	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2755	{
2756	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2757	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
2758	{
2759	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
2760	cErrors++;
2761	continue;
2762	}
2763	}
2764	else if ( (PdeDst.u & X86_PDE_P)
2765	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
2766	)
2767	{
2768	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
2769	PPGMPOOLPAGE pPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
2770	if (!pPage)
2771	{
2772	AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
2773	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
2774	cErrors++;
2775	continue;
2776	}
2777	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPage);
2778
2779	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
2780	{
2781	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
2782	GCPtr, (uint64_t)PdeDst.u));
2783	cErrors++;
2784	}
2785
2786	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
2787	{
2788	AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
2789	GCPtr, (uint64_t)PdeDst.u));
2790	cErrors++;
2791	}
2792
2793	const X86PDE PdeSrc = pPDSrc->a[iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)];
2794	if (!PdeSrc.n.u1Present)
2795	{
2796	AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
2797	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
2798	cErrors++;
2799	continue;
2800	}
2801
2802	if ( !PdeSrc.b.u1Size
2803	\|\| !(cr4 & X86_CR4_PSE))
2804	{
2805	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
2806	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2807	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2808	# endif
2809	}
2810	else
2811	{
2812	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
2813	{
2814	AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
2815	GCPtr, (uint64_t)PdeSrc.u));
2816	cErrors++;
2817	continue;
2818	}
2819	GCPhysGst = PdeSrc.u & GST_PDE4M_PG_MASK;
2820	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2821	GCPhysGst \|= GCPtr & BIT(X86_PAGE_2M_SHIFT);
2822	# endif
2823	}
2824
2825	if ( pPage->enmKind
2826	!= (!PdeSrc.b.u1Size \|\| !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
2827	{
2828	AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
2829	pPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
2830	cErrors++;
2831	}
2832
2833	rc = PGMRamGCPhys2HCPhysWithFlags(pPGM, GCPhysGst, &HCPhys);
2834	if (VBOX_FAILURE(rc))
2835	{
2836	AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
2837	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
2838	cErrors++;
2839	continue;
2840	}
2841
2842	if (GCPhysGst != pPage->GCPhys)
2843	{
2844	AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
2845	GCPhysGst, pPage->GCPhys, GCPtr));
2846	cErrors++;
2847	continue;
2848	}
2849
2850	if ( !PdeSrc.b.u1Size
2851	\|\| !(cr4 & X86_CR4_PSE))
2852	{
2853	/*
2854	* Page Table.
2855	*/
2856	const GSTPT *pPTSrc;
2857	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
2858	if (VBOX_FAILURE(rc))
2859	{
2860	AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
2861	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
2862	cErrors++;
2863	continue;
2864	}
2865	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
2866	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
2867	{
2868	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
2869	// (This problem will go away when/if we shadow multiple CR3s.)
2870	AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
2871	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
2872	cErrors++;
2873	continue;
2874	}
2875	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
2876	{
2877	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
2878	GCPtr, (uint64_t)PdeDst.u));
2879	cErrors++;
2880	continue;
2881	}
2882
2883	/* iterate the page table. */
2884	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2885	const unsigned offPTSrc = 0;
2886	# else
2887	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
2888	# endif
2889	for (unsigned iPT = 0, off = 0;
2890	iPT < ELEMENTS(pPTDst->a);
2891	iPT++, off += PAGE_SIZE)
2892	{
2893	const SHWPTE PteDst = pPTDst->a[iPT];
2894
2895	/* skip not-present entries. */
2896	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
2897	continue;
2898	Assert(PteDst.n.u1Present);
2899
2900	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
2901	if (!PteSrc.n.u1Present)
2902	{
2903	#ifdef IN_RING3
2904	PGMAssertHandlerAndFlagsInSync(pVM);
2905	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
2906	#endif
2907	AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
2908	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
2909	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
2910	cErrors++;
2911	continue;
2912	}
2913
2914	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
2915	# if 1 /** @todo sync accessed bit properly... */
2916	fIgnoreFlags \|= X86_PTE_A;
2917	# endif
2918
2919	/* match the physical addresses */
2920	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
2921	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
2922
2923	# ifdef IN_RING3
2924	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
2925	if (VBOX_FAILURE(rc))
2926	{
2927	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
2928	{
2929	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
2930	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2931	cErrors++;
2932	continue;
2933	}
2934	}
2935	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
2936	{
2937	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
2938	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2939	cErrors++;
2940	continue;
2941	}
2942	# endif
2943
2944	rc = PGMRamGCPhys2HCPhysWithFlags(pPGM, GCPhysGst, &HCPhys);
2945	if (VBOX_FAILURE(rc))
2946	{
2947	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
2948	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
2949	{
2950	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
2951	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2952	cErrors++;
2953	continue;
2954	}
2955	# endif
2956	if (PteDst.n.u1Write)
2957	{
2958	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
2959	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2960	cErrors++;
2961	}
2962	fIgnoreFlags \|= X86_PTE_RW;
2963	}
2964	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
2965	{
2966	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
2967	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2968	cErrors++;
2969	continue;
2970	}
2971
2972	/* flags */
2973	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
2974	{
2975	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
2976	{
2977	if (PteDst.n.u1Write)
2978	{
2979	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
2980	GCPtr + off, HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2981	cErrors++;
2982	continue;
2983	}
2984	fIgnoreFlags \|= X86_PTE_RW;
2985	}
2986	else
2987	{
2988	if (PteDst.n.u1Present)
2989	{
2990	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
2991	GCPtr + off, HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2992	cErrors++;
2993	continue;
2994	}
2995	fIgnoreFlags \|= X86_PTE_P;
2996	}
2997	}
2998	else
2999	{
3000	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3001	{
3002	if (PteDst.n.u1Write)
3003	{
3004	AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3005	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3006	cErrors++;
3007	continue;
3008	}
3009	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3010	{
3011	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3012	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3013	cErrors++;
3014	continue;
3015	}
3016	if (PteDst.n.u1Dirty)
3017	{
3018	AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3019	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3020	cErrors++;
3021	}
3022	# if 0 /** @todo sync access bit properly... */
3023	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3024	{
3025	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3026	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3027	cErrors++;
3028	}
3029	fIgnoreFlags \|= X86_PTE_RW;
3030	# else
3031	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3032	# endif
3033	}
3034	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3035	{
3036	/* access bit emulation (not implemented). */
3037	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3038	{
3039	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3040	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3041	cErrors++;
3042	continue;
3043	}
3044	if (!PteDst.n.u1Accessed)
3045	{
3046	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3047	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3048	cErrors++;
3049	}
3050	fIgnoreFlags \|= X86_PTE_P;
3051	}
3052	# ifdef DEBUG_sandervl
3053	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3054	# endif
3055	}
3056
3057	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3058	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3059	)
3060	{
3061	AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3062	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3063	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3064	cErrors++;
3065	continue;
3066	}
3067	} /* foreach PTE */
3068	}
3069	else
3070	{
3071	/*
3072	* Big Page.
3073	*/
3074	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| X86_PDE_PAE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3075	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3076	{
3077	if (PdeDst.n.u1Write)
3078	{
3079	AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3080	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3081	cErrors++;
3082	continue;
3083	}
3084	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3085	{
3086	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3087	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3088	cErrors++;
3089	continue;
3090	}
3091	# if 0 /** @todo sync access bit properly... */
3092	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3093	{
3094	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3095	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3096	cErrors++;
3097	}
3098	fIgnoreFlags \|= X86_PTE_RW;
3099	# else
3100	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3101	# endif
3102	}
3103	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3104	{
3105	/* access bit emulation (not implemented). */
3106	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3107	{
3108	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3109	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3110	cErrors++;
3111	continue;
3112	}
3113	if (!PdeDst.n.u1Accessed)
3114	{
3115	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3116	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3117	cErrors++;
3118	}
3119	fIgnoreFlags \|= X86_PTE_P;
3120	}
3121
3122	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3123	{
3124	AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3125	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3126	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3127	cErrors++;
3128	}
3129
3130	/* iterate the page table. */
3131	for (unsigned iPT = 0, off = 0;
3132	iPT < ELEMENTS(pPTDst->a);
3133	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3134	{
3135	const SHWPTE PteDst = pPTDst->a[iPT];
3136
3137	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3138	{
3139	AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3140	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3141	cErrors++;
3142	}
3143
3144	/* skip not-present entries. */
3145	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3146	continue;
3147
3148	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3149
3150	/* match the physical addresses */
3151	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3152
3153	# ifdef IN_RING3
3154	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3155	if (VBOX_FAILURE(rc))
3156	{
3157	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3158	{
3159	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3160	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3161	cErrors++;
3162	}
3163	}
3164	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3165	{
3166	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3167	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3168	cErrors++;
3169	continue;
3170	}
3171	# endif
3172
3173	rc = PGMRamGCPhys2HCPhysWithFlags(pPGM, GCPhysGst, &HCPhys);
3174	if (VBOX_FAILURE(rc))
3175	{
3176	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3177	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3178	{
3179	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3180	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3181	cErrors++;
3182	continue;
3183	}
3184	# endif
3185	if (PteDst.n.u1Write)
3186	{
3187	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3188	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3189	cErrors++;
3190	}
3191	fIgnoreFlags \|= X86_PTE_RW;
3192	}
3193	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3194	{
3195	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3196	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3197	cErrors++;
3198	continue;
3199	}
3200
3201	/* flags */
3202	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
3203	{
3204	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
3205	{
3206	if (!(HCPhys & MM_RAM_FLAGS_PHYSICAL_TEMP_OFF))
3207	{
3208	if (PteDst.n.u1Write)
3209	{
3210	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3211	GCPtr + off, HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3212	cErrors++;
3213	continue;
3214	}
3215	fIgnoreFlags \|= X86_PTE_RW;
3216	}
3217	}
3218	else
3219	{
3220	if (PteDst.n.u1Present)
3221	{
3222	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3223	GCPtr + off, HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3224	cErrors++;
3225	continue;
3226	}
3227	fIgnoreFlags \|= X86_PTE_P;
3228	}
3229	}
3230
3231	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3232	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3233	)
3234	{
3235	AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3236	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3237	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3238	cErrors++;
3239	continue;
3240	}
3241	} /* foreach PTE */
3242	}
3243	}
3244	/* not present */
3245
3246	} /* forearch PDE */
3247
3248	# ifdef DEBUG
3249	if (cErrors)
3250	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3251	# endif
3252
3253	#elif PGM_GST_TYPE == PGM_TYPE_PAE
3254	//# error not implemented
3255
3256
3257	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3258	//# error not implemented
3259
3260	/#else: guest real and protected mode /
3261	#endif
3262	return cErrors;
3263	}
3264	#endif /* VBOX_STRICT */
3265

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 2004

Download in other formats: