PGMAllBth.h@ 2096

Last change on this file since 2096 was 2078, checked in by vboxsync, 18 years ago
Removed obsolete code
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 141.9 KB

Line
1	/* $Id: PGMAllBth.h 2078 2007-04-13 14:37:31Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006 InnoTek Systemberatung GmbH
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License as published by the Free Software Foundation,
15	* in version 2 as it comes in the "COPYING" file of the VirtualBox OSE
16	* distribution. VirtualBox OSE is distributed in the hope that it will
17	* be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* If you received this file as part of a commercial VirtualBox
20	* distribution, then only the terms of your commercial VirtualBox
21	* license agreement apply instead of the previous paragraph.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVM pVM, VBOXPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PVBOXPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PVBOXPD pPDSrc, RTGCUINTPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	__END_DECLS
43
44
45	/**
46	* #PF Handler for raw-mode guest execution.
47	*
48	* @returns VBox status code (appropriate for trap handling and GC return).
49	* @param pVM VM Handle.
50	* @param uErr The trap error code.
51	* @param pRegFrame Trap register frame.
52	* @param pvFault The fault address.
53	*/
54	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
55	{
56	#if PGM_GST_TYPE == PGM_TYPE_32BIT
57	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
58	# error "32-bit guest mode is only implemented for 32-bit and PAE shadow modes."
59	# endif
60
61	# if PGM_SHW_TYPE == PGM_TYPE_PAE
62	/*
63	* Hide the instruction fetch trap indicator for now.
64	*/
65	/** @todo NXE will change this and we must fix NXE in the switcher too! */
66	if (uErr & X86_TRAP_PF_ID)
67	{
68	uErr &= ~X86_TRAP_PF_ID;
69	TRPMSetErrorCode(pVM, uErr);
70	}
71	# endif
72
73	/*
74	* Get PDs.
75	*/
76	int rc;
77	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
78	const unsigned iPDSrc = (uintptr_t)pvFault >> GST_PD_SHIFT;
79	const unsigned iPDDst = (uintptr_t)pvFault >> SHW_PD_SHIFT;
80	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
81	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
82	# else /* PAE */
83	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries. */
84	# endif
85
86	/* Determine current privilege level */
87	uint32_t cpl = CPUMGetGuestCPL(pVM, pRegFrame);
88
89	# ifdef PGM_SYNC_DIRTY_BIT
90	/*
91	* If we successfully correct the write protection fault due to dirty bit
92	* tracking, or this page fault is a genuine one, then return immediately.
93	*/
94	STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
95	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
96	STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
97	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
98	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
99	{
100	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
101	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
102	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
103	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
104	}
105	# endif
106	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
107
108	/*
109	* A common case is the not-present error caused by lazy page table syncing.
110	*
111	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
112	* so we can safely assume that the shadow PT is present when calling SyncPage later.
113	*
114	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
115	* of mapping conflict and defer to SyncCR3 in R3.
116	* (Again, we do NOT support access handlers for non-present guest pages.)
117	*
118	*/
119	VBOXPDE PdeSrc = pPDSrc->a[iPDSrc];
120	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
121	&& !pPDDst->a[iPDDst].n.u1Present
122	&& PdeSrc.n.u1Present)
123
124	{
125	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
126	STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
127	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
128	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
129	if (VBOX_SUCCESS(rc))
130	{
131	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
132	return rc;
133	}
134	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
135	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
136	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
137	return VINF_PGM_SYNC_CR3;
138	}
139
140	/*
141	* Check if this address is within any of our mappings.
142	*
143	* This is very fast and it's gonna save us a bit of effort below and prevent
144	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
145	* (BTW, it's impossible to have physical access handlers in a mapping.)
146	*/
147	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
148	{
149	STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
150	PPGMMAPPING pMapping = CTXSUFF(pVM->pgm.s.pMappings);
151	for ( ; pMapping; pMapping = CTXSUFF(pMapping->pNext))
152	{
153	if ((uintptr_t)pvFault < (uintptr_t)pMapping->GCPtr)
154	break;
155	if ((uintptr_t)pvFault - (uintptr_t)pMapping->GCPtr < pMapping->cb)
156	{
157	/*
158	* The first thing we check is if we've got an undetected conflict.
159	*/
160	if (!pVM->pgm.s.fMappingsFixed)
161	{
162	unsigned iPT = pMapping->cPTs;
163	while (iPT-- > 0)
164	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
165	{
166	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
167	Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
168	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
169	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
170	return VINF_PGM_SYNC_CR3;
171	}
172	}
173
174	/*
175	* Check if the fault address is in a virtual page access handler range.
176	*/
177	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
178	if ( pCur
179	&& pCur->enmType != PGMVIRTHANDLERTYPE_EIP
180	&& (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb
181	&& ( uErr & X86_TRAP_PF_RW
182	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
183	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** r=bird: <- this is probably wrong. */
184	{
185	#ifdef IN_GC
186	STAM_PROFILE_START(&pCur->Stat, h);
187	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr);
188	STAM_PROFILE_STOP(&pCur->Stat, h);
189	#else
190	AssertFailed();
191	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
192	#endif
193	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
194	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
195	return rc;
196	}
197
198	/*
199	* Check if the EIP is in a virtual page access handler range.
200	*/
201	if (cpl == 0)
202	{
203	RTGCPTR pvEIP;
204	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &pvEIP);
205	if (VBOX_SUCCESS(rc))
206	{
207	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvEIP);
208	if ( pCur
209	&& pCur->enmType == PGMVIRTHANDLERTYPE_EIP
210	&& (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr < pCur->cb)
211	{
212	#ifdef IN_GC
213	STAM_PROFILE_START(&pCur->Stat, h);
214	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr);
215	STAM_PROFILE_STOP(&pCur->Stat, h);
216	#else
217	AssertFailed();
218	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
219	#endif
220	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
221	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
222	return rc;
223	}
224	}
225	}
226
227	/*
228	* Pretend we're not here and let the guest handle the trap.
229	*/
230	TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
231	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
232	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
233	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
234	return VINF_EM_RAW_GUEST_TRAP;
235	}
236	}
237	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
238	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
239
240	/*
241	* Check if this fault address is flagged for special treatment,
242	* which means we'll have to figure out the physical address and
243	* check flags associated with it.
244	*
245	* ASSUME that we can limit any special access handling to pages
246	* in page tables which the guest believes to be present.
247	*/
248	if (PdeSrc.n.u1Present)
249	{
250	RTGCPHYS GCPhys = ~0U;
251	uint32_t cr4 = CPUMGetGuestCR4(pVM);
252	if ( PdeSrc.b.u1Size
253	&& (cr4 & X86_CR4_PSE))
254	GCPhys = (PdeSrc.u & X86_PDE4M_PG_MASK)
255	\| ((RTGCPHYS)pvFault & (PAGE_OFFSET_MASK_BIG ^ PAGE_OFFSET_MASK));
256	else
257	{
258	PVBOXPT pPTSrc;
259	# ifdef IN_GC
260	rc = PGMGCDynMapGCPage(pVM, PdeSrc.u & X86_PDE_PG_MASK, (void **)&pPTSrc);
261	# else
262	pPTSrc = (PVBOXPT)MMPhysGCPhys2HCVirt(pVM, PdeSrc.u & X86_PDE_PG_MASK, sizeof(*pPTSrc));
263	if (pPTSrc == 0)
264	rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
265	# endif
266	if (VBOX_SUCCESS(rc))
267	{
268	unsigned iPTESrc = ((uintptr_t)pvFault >> PAGE_SHIFT) & PTE_MASK;
269	if (pPTSrc->a[iPTESrc].n.u1Present)
270	GCPhys = pPTSrc->a[iPTESrc].u & X86_PTE_PG_MASK;
271	}
272	}
273
274	/*
275	* If we have a GC address we'll check if it has any flags set.
276	*/
277	if (GCPhys != ~0U)
278	{
279	STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
280
281	RTHCPHYS HCPhys;
282	rc = PGMRamGCPhys2HCPhysWithFlags(&pVM->pgm.s, GCPhys, &HCPhys);
283	if (VBOX_SUCCESS(rc))
284	{
285	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_HANDLER \| MM_RAM_FLAGS_VIRTUAL_HANDLER))
286	{
287	if (HCPhys & MM_RAM_FLAGS_PHYSICAL_HANDLER)
288	{
289	/*
290	* Physical page access handler.
291	*/
292	const RTGCPHYS GCPhysFault = GCPhys \| ((uintptr_t)pvFault & PAGE_OFFSET_MASK);
293	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
294	if (pCur)
295	{
296	# ifdef PGM_SYNC_N_PAGES
297	/*
298	* If the region is write protected and we got a page not present fault, then sync
299	* the pages. If the fault was caused by a read, then restart the instruction.
300	* In case of write access continue to the GC write handler.
301	*
302	* ASSUMES that there is only one handler per page or that they have similar write properties.
303	*/
304	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
305	&& !(uErr & X86_TRAP_PF_P))
306	{
307	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
308	if ( VBOX_FAILURE(rc)
309	\|\| !(uErr & X86_TRAP_PF_RW)
310	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
311	{
312	AssertRC(rc);
313	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
314	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
315	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
316	return rc;
317	}
318	}
319	# endif
320
321	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
322	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
323	("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, HCPhys, uErr, pCur->enmType));
324
325	#ifdef IN_GC
326	Assert(CTXSUFF(pCur->pfnHandler));
327	STAM_PROFILE_START(&pCur->Stat, h);
328	rc = pCur->CTXSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXSUFF(pCur->pvUser));
329	STAM_PROFILE_STOP(&pCur->Stat, h);
330	#elif IN_RING0
331	if (CTXALLSUFF(pCur->pfnHandler))
332	{
333	STAM_PROFILE_START(&pCur->Stat, h);
334	rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
335	STAM_PROFILE_STOP(&pCur->Stat, h);
336	}
337	else
338	rc = VINF_EM_RAW_EMULATE_INSTR;
339	#else
340	rc = VINF_EM_RAW_EMULATE_INSTR;
341	#endif
342	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
343	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
344	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
345	return rc;
346	}
347	}
348	else
349	{
350	# ifdef PGM_SYNC_N_PAGES
351	/*
352	* If the region is write protected and we got a page not present fault, then sync
353	* the pages. If the fault was caused by a read, then restart the instruction.
354	* In case of write access continue to the GC write handler.
355	*/
356	if ( (HCPhys & (MM_RAM_FLAGS_VIRTUAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_ALL)) == MM_RAM_FLAGS_VIRTUAL_WRITE
357	&& !(uErr & X86_TRAP_PF_P))
358	{
359	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
360	if ( VBOX_FAILURE(rc)
361	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
362	\|\| !(uErr & X86_TRAP_PF_RW))
363	{
364	AssertRC(rc);
365	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
366	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
367	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
368	return rc;
369	}
370	}
371	# endif
372	/*
373	* Ok, it's an virtual page access handler.
374	*
375	* Since it's faster to search by address, we'll do that first
376	* and then retry by GCPhys if that fails.
377	*/
378	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
379	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
380	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
381	*/
382	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
383	if (pCur)
384	{
385	AssertMsg(!((uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb)
386	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
387	\|\| !(uErr & X86_TRAP_PF_P)
388	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
389	("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, HCPhys, uErr, pCur->enmType));
390
391	if ( pCur->enmType != PGMVIRTHANDLERTYPE_EIP
392	&& (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb
393	&& ( uErr & X86_TRAP_PF_RW
394	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
395	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** @todo r=bird: _HYPERVISOR is impossible here because of mapping check. */
396	{
397	#ifdef IN_GC
398	STAM_PROFILE_START(&pCur->Stat, h);
399	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr);
400	STAM_PROFILE_STOP(&pCur->Stat, h);
401	#else
402	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
403	#endif
404	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
405	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
406	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
407	return rc;
408	}
409	/* Unhandled part of a monitored page */
410	}
411	else
412	{
413	/* Check by physical address. */
414	PPGMVIRTHANDLER pCur;
415	unsigned iPage;
416	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((uintptr_t)pvFault & PAGE_OFFSET_MASK),
417	&pCur, &iPage);
418	Assert(VBOX_SUCCESS(rc) \|\| !pCur);
419	if ( pCur
420	&& pCur->enmType != PGMVIRTHANDLERTYPE_EIP
421	&& ( uErr & X86_TRAP_PF_RW
422	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
423	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) )
424	{
425	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
426	#ifdef IN_GC
427	uintptr_t off = (iPage << PAGE_SHIFT) + ((uintptr_t)pvFault & PAGE_OFFSET_MASK) - ((uintptr_t)pCur->GCPtr & PAGE_OFFSET_MASK);
428	Assert(off < pCur->cb);
429	STAM_PROFILE_START(&pCur->Stat, h);
430	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
431	STAM_PROFILE_STOP(&pCur->Stat, h);
432	#else
433	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
434	#endif
435	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
436	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
437	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
438	return rc;
439	}
440	}
441	}
442
443	/*
444	* There is a handled area of the page, but this fault doesn't belong to it.
445	* We must emulate the instruction.
446	*
447	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
448	* we first check if this was a page-not-present fault for a page with only
449	* write access handlers. Restart the instruction if it wasn't a write access.
450	*/
451	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
452
453	if ( !(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL))
454	&& !(uErr & X86_TRAP_PF_P))
455	{
456	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
457	if ( VBOX_FAILURE(rc)
458	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
459	\|\| !(uErr & X86_TRAP_PF_RW))
460	{
461	AssertRC(rc);
462	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
463	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
464	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
465	return rc;
466	}
467	}
468
469	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
470	* It's writing to an unhandled part of the LDT page several million times.
471	*/
472	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
473	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%VHp%s%s\n",
474	rc, HCPhys, HCPhys & MM_RAM_FLAGS_PHYSICAL_HANDLER ? " phys" : "",
475	HCPhys & MM_RAM_FLAGS_VIRTUAL_HANDLER ? " virt" : ""));
476	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
477	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
478	return rc;
479	} /* if any kind of handler */
480
481	if (uErr & X86_TRAP_PF_P)
482	{
483	/*
484	* The page isn't marked, but it might still be monitored by a virtual page access handler.
485	* (ASSUMES no temporary disabling of virtual handlers.)
486	*/
487	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
488	* we should correct both the shadow page table and physical memory flags, and not only check for
489	* accesses within the handler region but for access to pages with virtual handlers. */
490	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
491	if (pCur)
492	{
493	AssertMsg( !((uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb)
494	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
495	\|\| !(uErr & X86_TRAP_PF_P)
496	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
497	("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, HCPhys, uErr, pCur->enmType));
498
499	if ( pCur->enmType != PGMVIRTHANDLERTYPE_EIP
500	&& (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr < pCur->cb
501	&& ( uErr & X86_TRAP_PF_RW
502	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
503	&& pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** @todo r=bird: _HYPERVISOR is impossible here because of mapping check. */
504	{
505	#ifdef IN_GC
506	STAM_PROFILE_START(&pCur->Stat, h);
507	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvFault - (uintptr_t)pCur->GCPtr);
508	STAM_PROFILE_STOP(&pCur->Stat, h);
509	#else
510	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
511	#endif
512	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
513	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
514	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
515	return rc;
516	}
517	}
518	}
519	}
520	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
521
522	# ifdef PGM_OUT_OF_SYNC_IN_GC
523	/*
524	* We are here only if page is present in Guest page tables and trap is not handled
525	* by our handlers.
526	* Check it for page out-of-sync situation.
527	*/
528	STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
529
530	if (!(uErr & X86_TRAP_PF_P))
531	{
532	/*
533	* Page is not present in our page tables.
534	* Try to sync it!
535	* BTW, fPageShw is invalid in this branch!
536	*/
537	if (uErr & X86_TRAP_PF_US)
538	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
539	else /* supervisor */
540	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
541
542	# if defined(LOG_ENABLED) && !defined(IN_RING0)
543	RTGCPHYS GCPhys;
544	uint64_t fPageGst;
545	PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
546	Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
547	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
548	# endif /* LOG_ENABLED */
549
550	# ifndef IN_RING0
551	if (cpl == 0)
552	{
553	uint64_t fPageGst;
554	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
555	if ( VBOX_SUCCESS(rc)
556	&& !(fPageGst & X86_PTE_US))
557	{
558	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
559	if ( pvFault == (RTGCPTR)pRegFrame->eip
560	\|\| (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
561	#ifdef CSAM_DETECT_NEW_CODE_PAGES
562	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
563	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
564	#endif /* CSAM_DETECT_NEW_CODE_PAGES */
565	)
566	{
567	LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
568	rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
569	if (rc != VINF_SUCCESS)
570	{
571	/*
572	* CSAM needs to perform a job in ring 3.
573	*
574	* Sync the page before going to the host context; otherwise we'll end up in a loop if
575	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
576	*/
577	LogFlow(("CSAM ring 3 job\n"));
578	int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
579	AssertRC(rc2);
580
581	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
582	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
583	return rc;
584	}
585	}
586	#ifdef CSAM_DETECT_NEW_CODE_PAGES
587	else
588	if ( uErr == X86_TRAP_PF_RW
589	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
590	&& pRegFrame->ecx < 0x10000
591	)
592	{
593	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
594	* to detect loading of new code pages.
595	*/
596
597	/*
598	* Decode the instruction.
599	*/
600	RTGCPTR PC;
601	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
602	if (rc == VINF_SUCCESS)
603	{
604	DISCPUSTATE Cpu;
605	uint32_t cbOp;
606	rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
607
608	/* For now we'll restrict this to rep movsw/d instructions */
609	if ( rc == VINF_SUCCESS
610	&& Cpu.pCurInstr->opcode == OP_MOVSWD
611	&& (Cpu.prefix & PREFIX_REP))
612	{
613	CSAMMarkPossibleCodePage(pVM, pvFault);
614	}
615	}
616	}
617	#endif /* CSAM_DETECT_NEW_CODE_PAGES */
618
619	/*
620	* Mark this page as safe.
621	*/
622	/** @todo not correct for pages that contain both code and data!! */
623	Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
624	CSAMMarkPage(pVM, pvFault, true);
625	}
626	}
627	# endif
628	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
629	if (VBOX_SUCCESS(rc))
630	{
631	/* The page was successfully synced, return to the guest. */
632	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
633	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
634	return VINF_SUCCESS;
635	}
636	}
637	else
638	{
639	/*
640	* A side effect of not flushing global PDEs are out of sync pages due
641	* to physical monitored regions, that are no longer valid.
642	* Assume for now it only applies to the read/write flag
643	*/
644	if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
645	{
646	if (uErr & X86_TRAP_PF_US)
647	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
648	else /* supervisor */
649	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
650
651
652	/*
653	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
654	*/
655	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
656	if (VBOX_SUCCESS(rc))
657	{
658	/*
659	* Page was successfully synced, return to guest.
660	*/
661	# ifdef VBOX_STRICT
662	RTGCPHYS GCPhys;
663	uint64_t fPageGst;
664	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
665	Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
666	LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
667
668	uint64_t fPageShw;
669	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
670	Assert(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW);
671	# endif /* VBOX_STRICT */
672	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
673	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
674	return VINF_SUCCESS;
675	}
676	}
677
678	# ifdef VBOX_STRICT
679	/*
680	* Check for VMM page flags vs. Guest page flags consistency.
681	* Currently only for debug purposes.
682	*/
683	if (VBOX_SUCCESS(rc))
684	{
685	/* Get guest page flags. */
686	uint64_t fPageGst;
687	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
688	if (VBOX_SUCCESS(rc))
689	{
690	uint64_t fPageShw;
691	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
692
693	/*
694	* Compare page flags.
695	* Note: we have AVL, A, D bits desynched.
696	*/
697	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
698	("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
699	}
700	else
701	AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
702	}
703	else
704	AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
705	# endif /* VBOX_STRICT */
706	}
707	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
708	# endif /* PGM_OUT_OF_SYNC_IN_GC */
709	}
710	else
711	{
712	/*
713	* Page not present in Guest OS or invalid page table address.
714	* This is potential virtual page access handler food.
715	*
716	* For the present we'll say that our access handlers don't
717	* work for this case - we've already discarded the page table
718	* not present case which is identical to this.
719	*
720	* When we perchance find we need this, we will probably have AVL
721	* trees (offset based) to operate on and we can measure their speed
722	* agains mapping a page table and probably rearrange this handling
723	* a bit. (Like, searching virtual ranges before checking the
724	* physical address.)
725	*/
726	}
727	}
728
729
730	/*
731	* Check if it's in a EIP based virtual page access handler range.
732	* This is only used for supervisor pages in flat mode.
733	*/
734	/** @todo this stuff is completely broken by the out-of-sync stuff. since we don't use this stuff, that's not really a problem yet. */
735	STAM_PROFILE_START(&pVM->pgm.s.StatEIPHandlers, d);
736	if (cpl == 0)
737	{
738	RTGCPTR pvEIP;
739	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &pvEIP);
740	if ( VBOX_SUCCESS(rc)
741	&& pvEIP == (RTGCPTR)pRegFrame->eip)
742	{
743	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvEIP);
744	if ( pCur
745	&& pCur->enmType == PGMVIRTHANDLERTYPE_EIP
746	&& (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr < pCur->cb)
747	{
748	LogFlow(("EIP handler\n"));
749	#ifdef IN_GC
750	STAM_PROFILE_START(&pCur->Stat, h);
751	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (uintptr_t)pvEIP - (uintptr_t)pCur->GCPtr);
752	STAM_PROFILE_STOP(&pCur->Stat, h);
753	#else
754	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
755	#endif
756	STAM_PROFILE_STOP(&pVM->pgm.s.StatEIPHandlers, d);
757	return rc;
758	}
759	}
760	}
761	STAM_PROFILE_STOP(&pVM->pgm.s.StatEIPHandlers, d);
762
763	/*
764	* Conclusion, this is a guest trap.
765	*/
766	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
767	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
768	return VINF_EM_RAW_GUEST_TRAP;
769
770	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
771
772	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
773	return VERR_INTERNAL_ERROR;
774	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
775	}
776
777
778	/**
779	* Emulation of the invlpg instruction.
780	*
781	*
782	* @returns VBox status code.
783	*
784	* @param pVM VM handle.
785	* @param GCPtrPage Page to invalidate.
786	*
787	* @remark ASSUMES that the guest is updating before invalidating. This order
788	* isn't required by the CPU, so this is speculative and could cause
789	* trouble.
790	*
791	* @todo Flush page or page directory only if necessary!
792	* @todo Add a #define for simply invalidating the page.
793	*/
794	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
795	{
796	#if PGM_GST_TYPE == PGM_TYPE_32BIT
797
798	LogFlow(("InvalidatePage %x\n", GCPtrPage));
799	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
800	/*
801	* Get the shadow PD entry and skip out if this PD isn't present.
802	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
803	*/
804	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
805	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
806	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
807	# else
808	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
809	# endif
810	const SHWPDE PdeDst = *pPdeDst;
811	if (!PdeDst.n.u1Present)
812	{
813	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
814	return VINF_SUCCESS;
815	}
816
817	/*
818	* Get the guest PD entry and calc big page.
819	*/
820	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
821	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
822	VBOXPDE PdeSrc = pPDSrc->a[iPDSrc];
823	const uint32_t cr4 = CPUMGetGuestCR4(pVM);
824	const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
825
826	# ifdef IN_RING3
827	/*
828	* If a CR3 Sync is pending we may ignore the invalidate page operation
829	* depending on the kind of sync and if it's a global page or not.
830	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
831	*/
832	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
833	if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
834	\|\| ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
835	&& fIsBigPage
836	&& PdeSrc.b.u1Global
837	&& (cr4 & X86_CR4_PGE)
838	)
839	)
840	# else
841	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
842	# endif
843	{
844	STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
845	return VINF_SUCCESS;
846	}
847	# endif /* IN_RING3 */
848
849
850	/*
851	* Deal with the Guest PDE.
852	*/
853	int rc = VINF_SUCCESS;
854	if (PdeSrc.n.u1Present)
855	{
856	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
857	{
858	/*
859	* Conflict - Let SyncPT deal with it to avoid duplicate code.
860	*/
861	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
862	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
863	}
864	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
865	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
866	{
867	/*
868	* Mark not present so we can resync the PDE when it's used.
869	*/
870	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
871	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
872	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
873	pPdeDst->u = 0;
874	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
875	PGM_INVL_GUEST_TLBS();
876	}
877	# ifdef PGM_SYNC_ACCESSED_BIT
878	else if (!PdeSrc.n.u1Accessed)
879	{
880	/*
881	* Mark not present so we can set the accessed bit.
882	*/
883	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
884	pPdeDst->u = 0;
885	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
886	PGM_INVL_GUEST_TLBS();
887	}
888	# endif
889	else if (!fIsBigPage)
890	{
891	/*
892	* 4KB - page.
893	*/
894	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
895	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
896	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
897	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
898	# endif
899	if (pShwPage->GCPhys == GCPhys)
900	{
901	#if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
902	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
903	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
904	if (pPT->a[iPTEDst].n.u1Present)
905	{
906	# ifdef PGMPOOL_WITH_USER_TRACKING
907	/* This is very unlikely with caching/monitoring enabled. */
908	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
909	# endif
910	pPT->a[iPTEDst].u = 0;
911	}
912	#else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
913	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
914	if (VBOX_SUCCESS(rc))
915	rc = VINF_SUCCESS;
916	#endif
917	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
918	PGM_INVL_PG(GCPtrPage);
919	}
920	else
921	{
922	/*
923	* The page table address changed.
924	*/
925	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
926	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
927	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
928	pPdeDst->u = 0;
929	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
930	PGM_INVL_GUEST_TLBS();
931	}
932	}
933	else
934	{
935	/*
936	* 4MB - page.
937	*/
938	/* Before freeing the page, check if anything really changed. */
939	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
940	RTGCPHYS GCPhys = PdeSrc.u & X86_PDE4M_PG_MASK;
941	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
942	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
943	# endif
944	if ( pShwPage->GCPhys == GCPhys
945	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
946	{
947	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
948	/** @todo PAT */
949	# ifdef PGM_SYNC_DIRTY_BIT
950	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
951	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
952	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
953	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
954	# else
955	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
956	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD)))
957	# endif
958	{
959	LogFlow(("Skipping flush for big page containing %VGv (PD=%X)-> nothing has changed!\n", GCPtrPage, iPDSrc));
960	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
961	return VINF_SUCCESS;
962	}
963	}
964
965	/*
966	* Ok, the page table is present and it's been changed in the guest.
967	* If we're in host context, we'll just mark it as not present taking the lazy approach.
968	* We could do this for some flushes in GC too, but we need an algorithm for
969	* deciding which 4MB pages containing code likely to be executed very soon.
970	*/
971	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
972	pPdeDst->u = 0;
973	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
974	DUMP_PDE_BIG("PGMInvalidatePage", iPDSrc, PdeSrc);
975	PGM_INVL_BIG_PG(GCPtrPage);
976	}
977	}
978	else
979	{
980	/*
981	* Page directory is not present, mark shadow PDE not present.
982	*/
983	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
984	{
985	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
986	pPdeDst->u = 0;
987	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
988	PGM_INVL_PG(GCPtrPage);
989	}
990	else
991	{
992	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
993	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
994	}
995	}
996
997	return rc;
998
999	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1000	# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
1001	# endif
1002	return VINF_SUCCESS;
1003
1004	#elif PGM_GST_TYPE == PGM_TYPE_PAE
1005	# if PGM_SHW_TYPE == PGM_TYPE_PAE
1006	//# error not implemented
1007	return VERR_INTERNAL_ERROR;
1008
1009	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1010	# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
1011	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1012
1013	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
1014	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1015	//# error not implemented
1016	return VERR_INTERNAL_ERROR;
1017
1018	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1019	# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
1020	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1021
1022	#else /* guest real and protected mode */
1023
1024	return VINF_SUCCESS;
1025	#endif
1026	}
1027
1028
1029	#ifdef PGMPOOL_WITH_USER_TRACKING
1030	/**
1031	* Update the tracking of shadowed pages.
1032	*
1033	* @param pVM The VM handle.
1034	* @param pShwPage The shadow page.
1035	* @param HCPhys The physical page we is being dereferenced.
1036	*/
1037	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1038	{
1039	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1040	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1041	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1042
1043	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1044	* 1. have a medium sized HCPhys -> GCPhys cache (hash?)
1045	* 2. write protect all shadowed pages. I.e. implement caching.
1046	*/
1047	/*
1048	* Find the guest address.
1049	*/
1050	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTXSUFF(pRamRanges);
1051	pRam;
1052	pRam = pRam->CTXSUFF(pNext))
1053	{
1054	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1055	while (iPage-- > 0)
1056	{
1057	if ((pRam->aHCPhys[iPage] & X86_PTE_PAE_PG_MASK) == HCPhys)
1058	{
1059	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1060	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aHCPhys[iPage]);
1061	pShwPage->cPresent--;
1062	pPool->cPresent--;
1063	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1064	return;
1065	}
1066	}
1067	}
1068
1069	for (;;)
1070	AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1071	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1072	pShwPage->cPresent--;
1073	pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1074	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1075	}
1076
1077
1078	/**
1079	* Update the tracking of shadowed pages.
1080	*
1081	* @param pVM The VM handle.
1082	* @param pShwPage The shadow page.
1083	* @param u16 The top 16-bit of the *pHCPhys.
1084	* @param pHCPhys Pointer to the ram range physical page entry.
1085	* @param iPTDst The index into the shadow table.
1086	*/
1087	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PRTHCPHYS pHCPhys, const unsigned iPTDst)
1088	{
1089	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1090	/*
1091	* We're making certain assumptions about the placement of cRef and idx.
1092	*/
1093	Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1094	Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1095
1096	/*
1097	* Just deal with the simple first time here.
1098	*/
1099	if (!u16)
1100	{
1101	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1102	u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) \| pShwPage->idx;
1103	}
1104	else
1105	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1106
1107	/* write back, trying to be clever... */
1108	Log2(("SyncPageWorkerTrackAddRef: u16=%#x *pHCPhys=%VHp->%VHp iPTDst=%#x\n",
1109	u16, pHCPhys, (pHCPhys & MM_RAM_FLAGS_NO_REFS_MASK) \| ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1110	((uint16_t )pHCPhys + 3) = u16;
1111	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1112
1113	/* update statistics. */
1114	pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1115	pShwPage->cPresent++;
1116	if (pShwPage->iFirstPresent > iPTDst)
1117	pShwPage->iFirstPresent = iPTDst;
1118	}
1119	#endif /* PGMPOOL_WITH_USER_TRACKING */
1120
1121
1122	/**
1123	* Creates a 4K shadow page for a guest page.
1124	*
1125	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1126	* physical address. The PdeSrc argument only the flags are used. No page structured
1127	* will be mapped in this function.
1128	*
1129	* @param pVM VM handle.
1130	* @param pPteDst Destination page table entry.
1131	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1132	* Can safely assume that only the flags are being used.
1133	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1134	* @param pShwPage Pointer to the shadow page.
1135	* @param iPTDst The index into the shadow table.
1136	*
1137	* @remark Not used for 2/4MB pages!
1138	*/
1139	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, VBOXPDE PdeSrc, VBOXPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1140	{
1141	if (PteSrc.n.u1Present)
1142	{
1143	/*
1144	* Find the ram range.
1145	*/
1146	PRTHCPHYS pHCPhys;
1147	int rc = PGMRamGCPhys2PagePtr(&pVM->pgm.s, PteSrc.u & X86_PTE_PG_MASK, &pHCPhys);
1148	if (VBOX_SUCCESS(rc))
1149	{
1150	/** @todo investiage PWT, PCD and PAT. */
1151	/*
1152	* Make page table entry.
1153	*/
1154	const RTHCPHYS HCPhys = *pHCPhys;
1155	SHWPTE PteDst;
1156	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
1157	{
1158	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1159	if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL)))
1160	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1161	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1162	else
1163	PteDst.u = 0;
1164	/** @todo count these two kinds. */
1165	}
1166	else
1167	{
1168	#ifdef PGM_SYNC_DIRTY_BIT
1169	# ifdef PGM_SYNC_ACCESSED_BIT
1170	/*
1171	* If the page or page directory entry is not marked accessed,
1172	* we mark the page not present.
1173	*/
1174	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1175	{
1176	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1177	PteDst.u = 0;
1178	}
1179	else
1180	# endif
1181	/*
1182	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1183	* when the page is modified.
1184	*/
1185	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1186	{
1187	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1188	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1189	\| (HCPhys & X86_PTE_PAE_PG_MASK)
1190	\| PGM_PTFLAGS_TRACK_DIRTY;
1191	}
1192	else
1193	{
1194	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1195	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1196	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1197	}
1198	#endif
1199	}
1200
1201	#ifdef PGMPOOL_WITH_USER_TRACKING
1202	/*
1203	* Keep user track up to date.
1204	*/
1205	if (PteDst.n.u1Present)
1206	{
1207	if (!pPteDst->n.u1Present)
1208	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pHCPhys, iPTDst);
1209	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1210	{
1211	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1212	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1213	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pHCPhys, iPTDst);
1214	}
1215	}
1216	else if (pPteDst->n.u1Present)
1217	{
1218	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1219	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1220	}
1221	#endif /* PGMPOOL_WITH_USER_TRACKING */
1222
1223	/*
1224	* Update statistics and commit the entry.
1225	*/
1226	if (!PteSrc.n.u1Global)
1227	pShwPage->fSeenNonGlobal = true;
1228	*pPteDst = PteDst;
1229	}
1230	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1231	/** @todo count these. */
1232	}
1233	else
1234	{
1235	/*
1236	* Page not-present.
1237	*/
1238	#ifdef PGMPOOL_WITH_USER_TRACKING
1239	/* Keep user track up to date. */
1240	if (pPteDst->n.u1Present)
1241	{
1242	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1243	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1244	}
1245	#endif /* PGMPOOL_WITH_USER_TRACKING */
1246	pPteDst->u = 0;
1247	/** @todo count these. */
1248	}
1249	}
1250
1251
1252	/**
1253	* Syncs a guest OS page.
1254	*
1255	* There are no conflicts at this point, neither is there any need for
1256	* page table allocations.
1257	*
1258	* @returns VBox status code.
1259	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1260	* @param pVM VM handle.
1261	* @param PdeSrc Page directory entry of the guest.
1262	* @param GCPtrPage Guest context page address.
1263	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1264	* @param uErr Fault error (X86_TRAP_PF_*).
1265	*/
1266	PGM_BTH_DECL(int, SyncPage)(PVM pVM, VBOXPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1267	{
1268	LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1269
1270	#if PGM_GST_TYPE == PGM_TYPE_32BIT
1271
1272	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1273	# error "Invalid shadow mode for 32-bit guest mode!"
1274	# endif
1275
1276	/*
1277	* Assert preconditions.
1278	*/
1279	# if GC_ARCH_BITS != 32
1280	Assert(GCPtrPage < _4G); //???
1281	# endif
1282	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> X86_PD_SHIFT) & X86_PD_MASK]);
1283	Assert(PdeSrc.n.u1Present);
1284	Assert(cPages);
1285
1286	/*
1287	* Get the shadow PDE, find the shadow page table in the pool.
1288	*/
1289	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1290	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1291	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1292	# else /* PAE */
1293	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1294	# endif
1295	Assert(PdeDst.n.u1Present);
1296	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1297
1298	/*
1299	* Check that the page is present and that the shadow PDE isn't out of sync.
1300	*/
1301	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1302	RTGCPHYS GCPhys;
1303	if (!fBigPage)
1304	{
1305	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1306	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1307	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1308	# endif
1309	}
1310	else
1311	{
1312	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1313	# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1314	GCPhys \|= GCPtrPage & X86_PAGE_2M_SIZE;
1315	# endif
1316	}
1317	if ( pShwPage->GCPhys == GCPhys
1318	&& PdeSrc.n.u1Present
1319	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1320	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1321	)
1322	{
1323	# ifdef PGM_SYNC_ACCESSED_BIT
1324	/*
1325	* Check that the PDE is marked accessed already.
1326	* Since we set the accessed bit before getting here on a #PF, this
1327	* check is only meant for dealing with non-#PF'ing paths.
1328	*/
1329	if (PdeSrc.n.u1Accessed)
1330	# endif
1331	{
1332	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1333	if (!fBigPage)
1334	{
1335	/*
1336	* 4KB Page - Map the guest page table.
1337	*/
1338	PVBOXPT pPTSrc;
1339	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & X86_PDE_PG_MASK, &pPTSrc);
1340	if (VBOX_SUCCESS(rc))
1341	{
1342	# ifdef PGM_SYNC_N_PAGES
1343	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1344	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1345	{
1346	/*
1347	* This code path is currently only taken when the caller is PGMTrap0eHandler
1348	* for non-present pages!
1349	*
1350	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1351	* deal with locality.
1352	*/
1353	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1354	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1355	const unsigned offPTSrc = 0;
1356	# else
1357	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1358	# endif
1359	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1360	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1361	iPTDst = 0;
1362	else
1363	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1364	for (; iPTDst < iPTDstEnd; iPTDst++)
1365	{
1366	if (!pPTDst->a[iPTDst].n.u1Present)
1367	{
1368	VBOXPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1369	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(X86_PT_MASK << X86_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1370	NOREF(GCPtrCurPage);
1371	#ifndef IN_RING0
1372	/*
1373	* Assuming kernel code will be marked as supervisor - and not as user level
1374	* and executed using a conforming code selector - And marked as readonly.
1375	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1376	*/
1377	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1378	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1379	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1380	\|\| PGMRamTestFlags(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK,
1381	MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE)
1382	)
1383	#endif
1384	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1385	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1386	GCPtrCurPage, PteSrc.n.u1Present,
1387	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1388	PteSrc.n.u1User & PdeSrc.n.u1User,
1389	(uint64_t)PteSrc.u,
1390	(uint64_t)pPTDst->a[iPTDst].u,
1391	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1392	}
1393	}
1394	}
1395	else
1396	# endif /* PGM_SYNC_N_PAGES */
1397	{
1398	const unsigned iPTSrc = (GCPtrPage >> X86_PT_SHIFT) & X86_PT_MASK;
1399	VBOXPTE PteSrc = pPTSrc->a[iPTSrc];
1400	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1401	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1402	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1403	GCPtrPage, PteSrc.n.u1Present,
1404	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1405	PteSrc.n.u1User & PdeSrc.n.u1User,
1406	(uint64_t)PteSrc.u,
1407	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1408	}
1409	}
1410	else /* MMIO or invalid page: emulated in #PF handler. */
1411	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1412	}
1413	else
1414	{
1415	/*
1416	* 4/2MB page - lazy syncing shadow 4K pages.
1417	* (There are many causes of getting here, it's no longer only CSAM.)
1418	*/
1419	/* Calculate the GC physical address of this 4KB shadow page. */
1420	RTGCPHYS GCPhys = (PdeSrc.u & X86_PDE4M_PAE_PG_MASK) \| ((RTGCUINTPTR)GCPtrPage & PAGE_OFFSET_MASK_BIG);
1421	/* Find ram range. */
1422	PRTHCPHYS pHCPhys;
1423	int rc = PGMRamGCPhys2PagePtr(&pVM->pgm.s, GCPhys, &pHCPhys);
1424	if (VBOX_SUCCESS(rc))
1425	{
1426	/*
1427	* Make shadow PTE entry.
1428	*/
1429	RTHCPHYS HCPhys = *pHCPhys;
1430	SHWPTE PteDst;
1431	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1432	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1433	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
1434	{
1435	if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL)))
1436	PteDst.n.u1Write = 0;
1437	else
1438	PteDst.u = 0;
1439	}
1440	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1441	# ifdef PGMPOOL_WITH_USER_TRACKING
1442	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1443	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pHCPhys, iPTDst);
1444	# endif
1445	pPTDst->a[iPTDst] = PteDst;
1446
1447
1448	# ifdef PGM_SYNC_DIRTY_BIT
1449	/*
1450	* If the page is not flagged as dirty and is writable, then make it read-only
1451	* at PD level, so we can set the dirty bit when the page is modified.
1452	*
1453	* ASSUMES that page access handlers are implemented on page table entry level.
1454	* Thus we will first catch the dirty access and set PDE.D and restart. If
1455	* there is an access handler, we'll trap again and let it work on the problem.
1456	*/
1457	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1458	* As for invlpg, it simply frees the whole shadow PT.
1459	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1460	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1461	{
1462	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1463	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1464	PdeDst.n.u1Write = 0;
1465	}
1466	else
1467	{
1468	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1469	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1470	}
1471	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1472	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1473	# else /* PAE */
1474	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1475	# endif
1476	# endif /* PGM_SYNC_DIRTY_BIT */
1477	Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1478	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1479	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1480	}
1481	}
1482	return VINF_SUCCESS;
1483	}
1484	# ifdef PGM_SYNC_ACCESSED_BIT
1485	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1486	#endif
1487	}
1488	else
1489	{
1490	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1491	Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1492	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1493	}
1494
1495	/*
1496	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1497	* Yea, I'm lazy.
1498	*/
1499	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1500	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1501	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1502	# else /* PAE */
1503	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1504	# endif
1505	PGM_INVL_GUEST_TLBS();
1506	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1507
1508	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1509
1510	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1511	return VERR_INTERNAL_ERROR;
1512	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1513	}
1514
1515
1516
1517	#ifdef PGM_SYNC_DIRTY_BIT
1518
1519	/**
1520	* Investigate page fault and handle write protection page faults caused by
1521	* dirty bit tracking.
1522	*
1523	* @returns VBox status code.
1524	* @param pVM VM handle.
1525	* @param uErr Page fault error code.
1526	* @param pPdeDst Shadow page directory entry.
1527	* @param pPdeSrc Guest page directory entry.
1528	* @param GCPtrPage Guest context page address.
1529	*/
1530	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PVBOXPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1531	{
1532	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1533	LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1534
1535	/*
1536	* Real page fault?
1537	*/
1538	if ( (uErr & X86_TRAP_PF_RSVD)
1539	\|\| !pPdeSrc->n.u1Present
1540	\|\| ((uErr & X86_TRAP_PF_RW) && !pPdeSrc->n.u1Write)
1541	\|\| ((uErr & X86_TRAP_PF_US) && !pPdeSrc->n.u1User) )
1542	{
1543	#ifdef IN_GC
1544	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1545	#endif
1546	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1547	LogFlow(("CheckPageFault: real page fault at %VGv (1)\n", GCPtrPage));
1548
1549	if (pPdeSrc->n.u1Present)
1550	{
1551	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1552	* See the 2nd case below as well.
1553	*/
1554	if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1555	{
1556	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1557	}
1558	else
1559	{
1560	/*
1561	* Map the guest page table.
1562	*/
1563	PVBOXPT pPTSrc;
1564	int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & X86_PDE_PG_MASK, &pPTSrc);
1565	if (VBOX_SUCCESS(rc))
1566	{
1567	PVBOXPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & PTE_MASK];
1568	const VBOXPTE PteSrc = *pPteSrc;
1569	if (pPteSrc->n.u1Present)
1570	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1571	}
1572	AssertRC(rc);
1573	}
1574	}
1575	return VINF_EM_RAW_GUEST_TRAP;
1576	}
1577
1578	/*
1579	* First check the easy case where the page directory has been marked read-only to track
1580	* the dirty bit of an emulated BIG page
1581	*/
1582	if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1583	{
1584	/* Mark guest page directory as accessed */
1585	pPdeSrc->b.u1Accessed = 1;
1586
1587	/*
1588	* Only write protection page faults are relevant here.
1589	*/
1590	if (uErr & X86_TRAP_PF_RW)
1591	{
1592	/* Mark guest page directory as dirty (BIG page only). */
1593	pPdeSrc->b.u1Dirty = 1;
1594
1595	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1596	{
1597	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1598
1599	Assert(pPdeSrc->b.u1Write);
1600
1601	pPdeDst->n.u1Write = 1;
1602	pPdeDst->n.u1Accessed = 1;
1603	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1604	PGM_INVL_BIG_PG(GCPtrPage);
1605	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1606	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1607	}
1608	}
1609	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1610	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1611	}
1612	/* else: 4KB page table */
1613
1614	/*
1615	* Map the guest page table.
1616	*/
1617	PVBOXPT pPTSrc;
1618	int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & X86_PDE_PG_MASK, &pPTSrc);
1619	if (VBOX_SUCCESS(rc))
1620	{
1621	/*
1622	* Real page fault?
1623	*/
1624	PVBOXPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & PTE_MASK];
1625	const VBOXPTE PteSrc = *pPteSrc;
1626	if ( !PteSrc.n.u1Present
1627	\|\| ((uErr & X86_TRAP_PF_RW) && !PteSrc.n.u1Write)
1628	\|\| ((uErr & X86_TRAP_PF_US) && !PteSrc.n.u1User)
1629	)
1630	{
1631	#ifdef IN_GC
1632	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1633	#endif
1634	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1635	LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1636
1637	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1638	* See the 2nd case above as well.
1639	*/
1640	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1641	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1642
1643	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1644	return VINF_EM_RAW_GUEST_TRAP;
1645	}
1646	LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1647
1648	/*
1649	* Set the accessed bits in the page directory and the page table.
1650	*/
1651	pPdeSrc->n.u1Accessed = 1;
1652	pPteSrc->n.u1Accessed = 1;
1653
1654	/*
1655	* Only write protection page faults are relevant here.
1656	*/
1657	if (uErr & X86_TRAP_PF_RW)
1658	{
1659	/* Write access, so mark guest entry as dirty. */
1660	#if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1661	if (!pPteSrc->n.u1Dirty)
1662	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1663	else
1664	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1665	#endif
1666	pPteSrc->n.u1Dirty = 1;
1667
1668	if (pPdeDst->n.u1Present)
1669	{
1670	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1671	* Our individual shadow handlers will provide more information and force a fatal exit.
1672	*/
1673	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1674	{
1675	LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1676	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1677	return VINF_SUCCESS;
1678	}
1679
1680	/*
1681	* Map shadow page table.
1682	*/
1683	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1684	if (pShwPage)
1685	{
1686	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1687	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1688	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1689	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1690	{
1691	LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1692	#ifdef VBOX_STRICT
1693	RTHCPHYS HCPhys;
1694	rc = PGMRamGCPhys2HCPhysWithFlags(&pVM->pgm.s, pPteSrc->u & X86_PTE_PG_MASK, &HCPhys);
1695	if (VBOX_SUCCESS(rc))
1696	AssertMsg(!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE)),
1697	("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1698	#endif
1699	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1700
1701	Assert(pPteSrc->n.u1Write);
1702
1703	pPteDst->n.u1Write = 1;
1704	pPteDst->n.u1Dirty = 1;
1705	pPteDst->n.u1Accessed = 1;
1706	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1707	PGM_INVL_PG(GCPtrPage);
1708
1709	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1710	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1711	}
1712	}
1713	else
1714	AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
1715	}
1716	}
1717	/** @todo Optimize accessed bit emulation? */
1718	#ifdef VBOX_STRICT
1719	/*
1720	* Sanity check.
1721	*/
1722	else if ( !pPteSrc->n.u1Dirty
1723	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
1724	&& pPdeDst->n.u1Present)
1725	{
1726	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1727	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1728	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1729	if ( pPteDst->n.u1Present
1730	&& pPteDst->n.u1Write)
1731	LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
1732	}
1733	#endif /* VBOX_STRICT */
1734	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1735	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1736	}
1737	AssertRC(rc);
1738	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1739	return rc;
1740	}
1741
1742	#endif
1743
1744
1745	/**
1746	* Sync a shadow page table.
1747	*
1748	* The shadow page table is not present. This includes the case where
1749	* there is a conflict with a mapping.
1750	*
1751	* @returns VBox status code.
1752	* @param pVM VM handle.
1753	* @param iPD Page directory index.
1754	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
1755	* Assume this is a temporary mapping.
1756	* @param GCPtrPage GC Pointer of the page that caused the fault
1757	*/
1758	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PVBOXPD pPDSrc, RTGCUINTPTR GCPtrPage)
1759	{
1760	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1761	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
1762	LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
1763
1764	#if PGM_GST_TYPE == PGM_TYPE_32BIT
1765
1766	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1767	# error "Invalid shadow mode for 32-bit guest mode!"
1768	# endif
1769
1770	/*
1771	* Validate input a little bit.
1772	*/
1773	Assert(iPDSrc == (GCPtrPage >> GST_PD_SHIFT));
1774	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1775	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
1776	# else
1777	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
1778	# endif
1779	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1780	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
1781	SHWPDE PdeDst = *pPdeDst;
1782
1783	/*
1784	* Check for conflicts.
1785	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
1786	* HC: Simply resolve the conflict.
1787	*/
1788	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1789	{
1790	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1791	# ifndef IN_RING3
1792	Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
1793	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1794	return VERR_ADDRESS_CONFLICT;
1795	# else
1796	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
1797	Assert(pMapping);
1798	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPDSrc);
1799	if (VBOX_FAILURE(rc))
1800	{
1801	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1802	return rc;
1803	}
1804	PdeDst = *pPdeDst;
1805	# endif
1806	}
1807	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
1808
1809	/*
1810	* Sync page directory entry.
1811	*/
1812	int rc = VINF_SUCCESS;
1813	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1814	if (PdeSrc.n.u1Present)
1815	{
1816	/*
1817	* Allocate & map the page table.
1818	*/
1819	PSHWPT pPTDst;
1820	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1821	PPGMPOOLPAGE pShwPage;
1822	RTGCPHYS GCPhys;
1823	if (fPageTable)
1824	{
1825	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1826	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1827	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1828	# endif
1829	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1830	}
1831	else
1832	{
1833	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1834	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1835	GCPhys \|= GCPtrPage & BIT(X86_PAGE_2M_SHIFT);
1836	# endif
1837	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1838	}
1839	if (rc == VINF_SUCCESS)
1840	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1841	else if (rc == VINF_PGM_CACHED_PAGE)
1842	{
1843	/*
1844	* The PT was cached, just hook it up.
1845	*/
1846	if (fPageTable)
1847	PdeDst.u = pShwPage->Core.Key
1848	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1849	else
1850	{
1851	PdeDst.u = pShwPage->Core.Key
1852	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1853	# ifdef PGM_SYNC_DIRTY_BIT /* (see explanation and assumtions further down.) */
1854	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1855	{
1856	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1857	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1858	PdeDst.b.u1Write = 0;
1859	}
1860	# endif
1861	}
1862	*pPdeDst = PdeDst;
1863	return VINF_SUCCESS;
1864	}
1865	else if (rc == VERR_PGM_POOL_FLUSHED)
1866	return VINF_PGM_SYNC_CR3;
1867	else
1868	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
1869	PdeDst.u &= X86_PDE_AVL_MASK;
1870	PdeDst.u \|= pShwPage->Core.Key;
1871
1872	# ifdef PGM_SYNC_DIRTY_BIT
1873	/*
1874	* Page directory has been accessed (this is a fault situation, remember).
1875	*/
1876	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
1877	# endif
1878
1879	if (fPageTable)
1880	{
1881	/*
1882	* Page table - 4KB.
1883	*
1884	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
1885	*/
1886	Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
1887	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
1888	PGSTPT pPTSrc;
1889	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1890	if (VBOX_SUCCESS(rc))
1891	{
1892	/*
1893	* Start by syncing the page directory entry so CSAM's TLB trick works.
1894	*/
1895	PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK))
1896	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1897	*pPdeDst = PdeDst;
1898
1899	/*
1900	* Directory/page user or supervisor privilege: (same goes for read/write)
1901	*
1902	* Directory Page Combined
1903	* U/S U/S U/S
1904	* 0 0 0
1905	* 0 1 0
1906	* 1 0 0
1907	* 1 1 1
1908	*
1909	* Simple AND operation. Table listed for completeness.
1910	*
1911	*/
1912	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
1913	# ifdef PGM_SYNC_N_PAGES
1914	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1915	unsigned iPTDst = iPTBase;
1916	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1917	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
1918	iPTDst = 0;
1919	else
1920	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1921	# else /* !PGM_SYNC_N_PAGES */
1922	unsigned iPTDst = 0;
1923	const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
1924	# endif /* !PGM_SYNC_N_PAGES */
1925	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1926	const unsigned offPTSrc = 0;
1927	# else
1928	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1929	# endif
1930	for (; iPTDst < iPTDstEnd; iPTDst++)
1931	{
1932	const unsigned iPTSrc = iPTDst + offPTSrc;
1933	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1934
1935	if (PteSrc.n.u1Present) /* we've already cleared it above */
1936	{
1937	#ifndef IN_RING0
1938	/*
1939	* Assuming kernel code will be marked as supervisor - and not as user level
1940	* and executed using a conforming code selector - And marked as readonly.
1941	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1942	*/
1943	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
1944	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
1945	\|\| PGMRamTestFlags(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK,
1946	MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE)
1947	)
1948	#endif
1949	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1950	Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
1951	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
1952	PteSrc.n.u1Present,
1953	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1954	PteSrc.n.u1User & PdeSrc.n.u1User,
1955	(uint64_t)PteSrc.u,
1956	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
1957	(PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
1958	}
1959	} /* for PTEs */
1960	}
1961	}
1962	else
1963	{
1964	/*
1965	* Big page - 2/4MB.
1966	*
1967	* We'll walk the ram range list in parallel and optimize lookups.
1968	* We will only sync on shadow page table at a time.
1969	*/
1970	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
1971
1972	/**
1973	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
1974	*/
1975
1976	/*
1977	* Start by syncing the page directory entry.
1978	*/
1979	PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
1980	\| (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
1981
1982	# ifdef PGM_SYNC_DIRTY_BIT
1983	/*
1984	* If the page is not flagged as dirty and is writable, then make it read-only
1985	* at PD level, so we can set the dirty bit when the page is modified.
1986	*
1987	* ASSUMES that page access handlers are implemented on page table entry level.
1988	* Thus we will first catch the dirty access and set PDE.D and restart. If
1989	* there is an access handler, we'll trap again and let it work on the problem.
1990	*/
1991	/** @todo move the above stuff to a section in the PGM documentation. */
1992	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
1993	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1994	{
1995	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1996	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1997	PdeDst.b.u1Write = 0;
1998	}
1999	# endif /* PGM_SYNC_DIRTY_BIT */
2000	*pPdeDst = PdeDst;
2001
2002	/*
2003	* Fill the shadow page table.
2004	*/
2005	/* Get address and flags from the source PDE. */
2006	SHWPTE PteDstBase;
2007	PteDstBase.u = PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2008
2009	/* Loop thru the entries in the shadow PT. */
2010	const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2011	Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2012	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2013	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2014	PPGMRAMRANGE pRam = CTXSUFF(pVM->pgm.s.pRamRanges);
2015	unsigned iPTDst = 0;
2016	while (iPTDst < ELEMENTS(pPTDst->a))
2017	{
2018	/* Advance ram range list. */
2019	while (pRam && GCPhys > pRam->GCPhysLast)
2020	pRam = CTXSUFF(pRam->pNext);
2021	if (pRam && GCPhys >= pRam->GCPhys)
2022	{
2023	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2024	do
2025	{
2026	/* Make shadow PTE. */
2027	RTHCPHYS HCPhys = pRam->aHCPhys[iHCPage];
2028	SHWPTE PteDst;
2029
2030	/* Make sure the RAM has already been allocated. */
2031	if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC)
2032	{
2033	if (RT_UNLIKELY(!(pRam->aHCPhys[iHCPage] & X86_PTE_PAE_PG_MASK)))
2034	{
2035	#ifdef IN_RING3
2036	int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2037	#else
2038	int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2039	#endif
2040	if (rc != VINF_SUCCESS)
2041	return rc;
2042
2043	HCPhys = pRam->aHCPhys[iHCPage];
2044	}
2045	}
2046
2047	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
2048	{
2049	if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL)))
2050	{
2051	PteDst.u = (HCPhys & X86_PTE_PAE_PG_MASK) \| PteDstBase.u;
2052	PteDst.n.u1Write = 0;
2053	}
2054	else
2055	PteDst.u = 0;
2056	}
2057	#ifndef IN_RING0
2058	/*
2059	* Assuming kernel code will be marked as supervisor and not as user level and executed
2060	* using a conforming code selector. Don't check for readonly, as that implies the whole
2061	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2062	*/
2063	else if ( !PdeSrc.n.u1User
2064	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2065	PteDst.u = 0;
2066	#endif
2067	else
2068	PteDst.u = (HCPhys & X86_PTE_PAE_PG_MASK) \| PteDstBase.u;
2069	# ifdef PGMPOOL_WITH_USER_TRACKING
2070	if (PteDst.n.u1Present)
2071	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, &pRam->aHCPhys[iHCPage], iPTDst);
2072	# endif
2073	/* commit it */
2074	pPTDst->a[iPTDst] = PteDst;
2075	Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2076	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2077	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2078
2079	/* advance */
2080	GCPhys += PAGE_SIZE;
2081	iHCPage++;
2082	iPTDst++;
2083	} while ( iPTDst < ELEMENTS(pPTDst->a)
2084	&& GCPhys <= pRam->GCPhysLast);
2085	}
2086	else if (pRam)
2087	{
2088	Log(("Invalid pages at %VGp\n", GCPhys));
2089	do
2090	{
2091	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2092	GCPhys += PAGE_SIZE;
2093	iPTDst++;
2094	} while ( iPTDst < ELEMENTS(pPTDst->a)
2095	&& GCPhys < pRam->GCPhys);
2096	}
2097	else
2098	{
2099	Log(("Invalid pages at %VGp (2)\n", GCPhys));
2100	for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2101	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2102	}
2103	} /* while more PTEs */
2104	} /* 4KB / 4MB */
2105	}
2106	else
2107	AssertRelease(!PdeDst.n.u1Present);
2108
2109	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2110	# ifdef IN_GC
2111	if (VBOX_FAILURE(rc))
2112	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2113	# endif
2114	return rc;
2115
2116	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2117
2118	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2119	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2120	return VERR_INTERNAL_ERROR;
2121	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2122	}
2123
2124
2125
2126	/**
2127	* Prefetch a page/set of pages.
2128	*
2129	* Typically used to sync commonly used pages before entering raw mode
2130	* after a CR3 reload.
2131	*
2132	* @returns VBox status code.
2133	* @param pVM VM handle.
2134	* @param GCPtrPage Page to invalidate.
2135	*/
2136	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2137	{
2138	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2139
2140	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2141	# error "Invalid shadow mode for 32-bit guest mode!"
2142	# endif
2143
2144	/*
2145	* Check that all Guest levels thru the PDE are present, getting the
2146	* PD and PDE in the processes.
2147	*/
2148	int rc = VINF_SUCCESS;
2149	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2150	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
2151	const VBOXPDE PdeSrc = pPDSrc->a[iPDSrc];
2152	# ifdef PGM_SYNC_ACCESSED_BIT
2153	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2154	# else
2155	if (PdeSrc.n.u1Present)
2156	# endif
2157	{
2158	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2159	const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> X86_PD_SHIFT];
2160	# else
2161	const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> X86_PD_PAE_SHIFT];
2162	# endif
2163	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2164	{
2165	if (!PdeDst.n.u1Present)
2166	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2167	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2168	else
2169	{
2170	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2171	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2172	* makes no sense to prefetch more than one page.
2173	*/
2174	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2175	if (VBOX_SUCCESS(rc))
2176	rc = VINF_SUCCESS;
2177	}
2178	}
2179	}
2180	return rc;
2181
2182	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2183
2184	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2185	return VERR_INTERNAL_ERROR;
2186	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2187	}
2188
2189
2190
2191
2192	/**
2193	* Syncs a page during a PGMVerifyAccess() call.
2194	*
2195	* @returns VBox status code (informational included).
2196	* @param GCPtrPage The address of the page to sync.
2197	* @param fPage The effective guest page flags.
2198	* @param uErr The trap error code.
2199	*/
2200	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2201	{
2202	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2203
2204	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2205
2206	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2207	# error "Invalid shadow mode for 32-bit guest mode!"
2208	# endif
2209
2210	#ifndef IN_RING0
2211	if (!(fPage & X86_PTE_US))
2212	{
2213	/*
2214	* Mark this page as safe.
2215	*/
2216	/** @todo not correct for pages that contain both code and data!! */
2217	Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2218	CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2219	}
2220	#endif
2221	/*
2222	* Get guest PD and index.
2223	*/
2224	unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
2225	PVBOXPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2226	int rc = VINF_SUCCESS;
2227
2228	# ifdef PGM_SYNC_DIRTY_BIT
2229	/*
2230	* First check if the page fault was caused by dirty bit tracking
2231	*/
2232	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2233	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> X86_PD_SHIFT];
2234	# else
2235	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> X86_PD_PAE_SHIFT];
2236	# endif
2237	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2238	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2239	Log(("PGMVerifyAccess: success (dirty)\n"));
2240	else
2241	# endif /* PGM_SYNC_DIRTY_BIT */
2242	{
2243	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2244	if (uErr & X86_TRAP_PF_US)
2245	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2246	else /* supervisor */
2247	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2248
2249	rc = PGM_BTH_NAME(SyncPage)(pVM, pPDSrc->a[iPDSrc], GCPtrPage, 1, 0);
2250	if (VBOX_SUCCESS(rc))
2251	{
2252	/* Page was successfully synced */
2253	Log(("PGMVerifyAccess: success (sync)\n"));
2254	rc = VINF_SUCCESS;
2255	}
2256	else
2257	{
2258	Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2259	return VINF_EM_RAW_GUEST_TRAP;
2260	}
2261	}
2262	return rc;
2263
2264	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2265
2266	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2267	return VERR_INTERNAL_ERROR;
2268	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2269	}
2270
2271
2272	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2273	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2274	/**
2275	* Figures out which kind of shadow page this guest PDE warrants.
2276	*
2277	* @returns Shadow page kind.
2278	* @param pPdeSrc The guest PDE in question.
2279	* @param cr4 The current guest cr4 value.
2280	*/
2281	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const VBOXPDE *pPdeSrc, uint32_t cr4)
2282	{
2283	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
2284	return BTH_PGMPOOLKIND_PT_FOR_PT;
2285	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
2286	//{
2287	// case 0:
2288	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2289	// case X86_PDE4M_RW:
2290	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2291	// case X86_PDE4M_US:
2292	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2293	// case X86_PDE4M_RW \| X86_PDE4M_US:
2294	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2295	# if 0
2296	// case X86_PDE4M_PAE_NX:
2297	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2298	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
2299	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2300	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2301	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2302	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2303	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2304	# endif
2305	return BTH_PGMPOOLKIND_PT_FOR_BIG;
2306	//}
2307	}
2308	# endif
2309	#endif
2310
2311	#undef MY_STAM_COUNTER_INC
2312	#define MY_STAM_COUNTER_INC(a) do { } while (0)
2313
2314
2315	/**
2316	* Syncs the paging hierarchy starting at CR3.
2317	*
2318	* @returns VBox status code, no specials.
2319	* @param pVM The virtual machine.
2320	* @param cr0 Guest context CR0 register
2321	* @param cr3 Guest context CR3 register
2322	* @param cr4 Guest context CR4 register
2323	* @param fGlobal Including global page directories or not
2324	*/
2325	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal)
2326	{
2327	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2328	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2329	/*
2330	* Inform the PGM PD Cache Manager about the pending sync.
2331	*/
2332	if (fGlobal \|\| VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2333	{
2334	# if 0 /** @todo what the heck is this about? */
2335	/* Don't cause an additional global CR3 reload the next time (the flag is cleared in PGMSyncCR3). */
2336	VM_FF_CLEAR(pVM, VM_FF_PGM_SYNC_CR3);
2337	# endif
2338
2339	/* Change this CR3 reload to be a global one. */
2340	fGlobal = true;
2341	}
2342	# endif
2343	#endif
2344
2345	/*
2346	* Update page access handlers.
2347	* The virtual are always flushed, while the physical are only on demand.
2348	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2349	* have to look into that later because it will have a bad influence on the performance.
2350	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
2351	* bird: Yes, but that won't work for aliases.
2352	*/
2353	/** @todo this MUST go away. See #1557. */
2354	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2355	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2356	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2357
2358	#ifdef PGMPOOL_WITH_MONITORING
2359	/*
2360	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2361	* Occationally we will have to clear all the shadow page tables because we wanted
2362	* to monitor a page which was mapped by too many shadowed page tables. This operation
2363	* sometimes refered to as a 'lightweight flush'.
2364	*/
2365	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2366	pgmPoolMonitorModifiedClearAll(pVM);
2367	else
2368	{
2369	# ifdef IN_RING3
2370	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2371	pgmPoolClearAll(pVM);
2372	# else
2373	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2374	return VINF_PGM_SYNC_CR3;
2375	# endif
2376	}
2377	#endif
2378
2379	Assert(fGlobal \|\| (cr4 & X86_CR4_PGE));
2380	MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2381
2382	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2383	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2384	/*
2385	* Get page directory addresses.
2386	*/
2387	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2388	PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2389	# else
2390	PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2391	# endif
2392	PVBOXPD pPDSrc = pVM->pgm.s.CTXSUFF(pGuestPD);
2393
2394	Assert(pPDSrc);
2395	#ifndef IN_GC
2396	Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2397	#endif
2398
2399	/*
2400	* Iterate the page directory.
2401	*/
2402	PPGMMAPPING pMapping;
2403	unsigned iPdNoMapping;
2404	const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2405	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2406
2407	/* Only check mappings if they are supposed to be put into the shadow page table. */
2408	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2409	{
2410	pMapping = pVM->pgm.s.CTXSUFF(pMappings);
2411	iPdNoMapping = (pMapping) ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2412	}
2413	else
2414	{
2415	pMapping = 0;
2416	iPdNoMapping = ~0U;
2417	}
2418
2419	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2420	{
2421	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2422	Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2423	# else
2424	Assert(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst);
2425	# endif
2426	register VBOXPDE PdeSrc = pPDSrc->a[iPD];
2427	if ( PdeSrc.n.u1Present
2428	&& (PdeSrc.n.u1User \|\| fRawR0Enabled))
2429	{
2430	/*
2431	* Check for conflicts with GC mappings.
2432	*/
2433	if (iPD == iPdNoMapping)
2434	{
2435	if (pVM->pgm.s.fMappingsFixed)
2436	{
2437	/* It's fixed, just skip the mapping. */
2438	const unsigned cPTs = pMapping->cPTs;
2439	iPD += cPTs - 1;
2440	pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2441	pMapping = pMapping->CTXSUFF(pNext);
2442	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2443	continue;
2444	}
2445
2446	#ifdef IN_RING3
2447	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2448	if (VBOX_FAILURE(rc))
2449	return rc;
2450
2451	/*
2452	* Update iPdNoMapping and pMapping.
2453	*/
2454	pMapping = pVM->pgm.s.pMappingsHC;
2455	while (pMapping && pMapping->GCPtr < (iPD << PGDIR_SHIFT))
2456	pMapping = pMapping->pNextHC;
2457	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2458	#else
2459	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2460	return VINF_PGM_SYNC_CR3;
2461	#endif
2462	}
2463
2464	/*
2465	* Sync page directory entry.
2466	*
2467	* The current approach is to allocated the page table but to set
2468	* the entry to not-present and postpone the page table synching till
2469	* it's actually used.
2470	*/
2471	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2472	const unsigned iPdShw = iPD; NOREF(iPdShw);
2473	# else
2474	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2475	# endif
2476	{
2477	SHWPDE PdeDst = *pPDEDst;
2478	if (PdeDst.n.u1Present)
2479	{
2480	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2481	RTGCPHYS GCPhys;
2482	if ( !PdeSrc.b.u1Size
2483	\|\| !(cr4 & X86_CR4_PSE))
2484	{
2485	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2486	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2487	GCPhys \|= i * (PAGE_SIZE / 2);
2488	# endif
2489	}
2490	else
2491	{
2492	GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
2493	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2494	GCPhys \|= i * X86_PAGE_2M_SIZE;
2495	# endif
2496	}
2497
2498	if ( pShwPage->GCPhys == GCPhys
2499	&& pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2500	&& ( pShwPage->fCached
2501	\|\| ( !fGlobal
2502	&& ( false
2503	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2504	\|\| ( (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2505	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE)) /* global 2/4MB page. */
2506	\|\| ( !pShwPage->fSeenNonGlobal
2507	&& (cr4 & X86_CR4_PGE))
2508	# endif
2509	)
2510	)
2511	)
2512	&& ( (PdeSrc.u & (X86_PDE_US \| X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US \| X86_PDE_RW))
2513	\|\| ( (cr4 & X86_CR4_PSE)
2514	&& ((PdeSrc.u & (X86_PDE_US \| X86_PDE4M_PS \| X86_PDE4M_D)) \| PGM_PDFLAGS_TRACK_DIRTY)
2515	== ((PdeDst.u & (X86_PDE_US \| X86_PDE_RW \| PGM_PDFLAGS_TRACK_DIRTY)) \| X86_PDE4M_PS))
2516	)
2517	)
2518	{
2519	# ifdef VBOX_WITH_STATISTICS
2520	if ( !fGlobal
2521	&& (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2522	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE))
2523	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2524	else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2525	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2526	else
2527	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2528	# endif /* VBOX_WITH_STATISTICS */
2529	/** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2530	* The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2531	//# ifdef PGMPOOL_WITH_CACHE
2532	// pgmPoolCacheUsed(pPool, pShwPage);
2533	//# endif
2534	}
2535	else
2536	{
2537	pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
2538	pPDEDst->u = 0;
2539	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
2540	}
2541	}
2542	else
2543	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
2544	pPDEDst++;
2545	}
2546	}
2547	else if (iPD != iPdNoMapping)
2548	{
2549	/*
2550	* Check if there is any page directory to mark not present here.
2551	*/
2552	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2553	const unsigned iPdShw = iPD; NOREF(iPdShw);
2554	# else
2555	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2556	# endif
2557	{
2558	if (pPDEDst->n.u1Present)
2559	{
2560	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
2561	pPDEDst->u = 0;
2562	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
2563	}
2564	pPDEDst++;
2565	}
2566	}
2567	else
2568	{
2569	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2570	const unsigned cPTs = pMapping->cPTs;
2571	if (pVM->pgm.s.fMappingsFixed)
2572	{
2573	/* It's fixed, just skip the mapping. */
2574	pMapping = pMapping->CTXSUFF(pNext);
2575	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2576	}
2577	else
2578	{
2579	/*
2580	* Check for conflicts for subsequent pagetables
2581	* and advance to the next mapping.
2582	*/
2583	iPdNoMapping = ~0U;
2584	unsigned iPT = cPTs;
2585	while (iPT-- > 1)
2586	{
2587	if ( pPDSrc->a[iPD + iPT].n.u1Present
2588	&& (pPDSrc->a[iPD + iPT].n.u1User \|\| fRawR0Enabled))
2589	{
2590	# ifdef IN_RING3
2591	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2592	if (VBOX_FAILURE(rc))
2593	return rc;
2594
2595	/*
2596	* Update iPdNoMapping and pMapping.
2597	*/
2598	pMapping = pVM->pgm.s.CTXSUFF(pMappings);
2599	while (pMapping && pMapping->GCPtr < (iPD << PGDIR_SHIFT))
2600	pMapping = pMapping->CTXSUFF(pNext);
2601	iPdNoMapping = pMapping ? pMapping->GCPtr >> PGDIR_SHIFT : ~0U;
2602	break;
2603	# else
2604	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2605	return VINF_PGM_SYNC_CR3;
2606	# endif
2607	}
2608	}
2609	if (iPdNoMapping == ~0U && pMapping)
2610	{
2611	pMapping = pMapping->CTXSUFF(pNext);
2612	if (pMapping)
2613	iPdNoMapping = pMapping->GCPtr >> PGDIR_SHIFT;
2614	}
2615	}
2616	/* advance. */
2617	iPD += cPTs - 1;
2618	pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2619	}
2620
2621	} /* for iPD */
2622	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2623	# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
2624	# endif
2625
2626	return VINF_SUCCESS;
2627
2628	#elif PGM_GST_TYPE == PGM_TYPE_PAE
2629	# if PGM_SHW_TYPE == PGM_TYPE_PAE
2630	//# error not implemented
2631	return VERR_INTERNAL_ERROR;
2632
2633	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2634	# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
2635	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2636
2637	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
2638	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
2639	//# error not implemented
2640	return VERR_INTERNAL_ERROR;
2641
2642	# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2643	# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
2644	# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2645
2646	#else /* guest real and protected mode */
2647
2648	return VINF_SUCCESS;
2649	#endif
2650	}
2651
2652
2653
2654
2655	#ifdef VBOX_STRICT
2656	#ifdef IN_GC
2657	# undef AssertMsgFailed
2658	# define AssertMsgFailed Log
2659	#endif
2660	#ifdef IN_RING3
2661	# include <VBox/dbgf.h>
2662
2663	/**
2664	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
2665	*
2666	* @returns VBox status code (VINF_SUCCESS).
2667	* @param pVM The VM handle.
2668	* @param cr3 The root of the hierarchy.
2669	* @param crr The cr4, only PAE and PSE is currently used.
2670	* @param fLongMode Set if long mode, false if not long mode.
2671	* @param cMaxDepth Number of levels to dump.
2672	* @param pHlp Pointer to the output functions.
2673	*/
2674	__BEGIN_DECLS
2675	PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
2676	__END_DECLS
2677
2678	#endif
2679
2680	/**
2681	* Checks that the shadow page table is in sync with the guest one.
2682	*
2683	* @returns The number of errors.
2684	* @param pVM The virtual machine.
2685	* @param cr3 Guest context CR3 register
2686	* @param cr4 Guest context CR4 register
2687	* @param GCPtr Where to start. Defaults to 0.
2688	* @param cb How much to check. Defaults to everything.
2689	*/
2690	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
2691	{
2692	unsigned cErrors = 0;
2693
2694	#if PGM_GST_TYPE == PGM_TYPE_32BIT
2695
2696	# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2697	# error "Invalid shadow mode for 32-bit guest paging."
2698	# endif
2699
2700	PPGM pPGM = &pVM->pgm.s;
2701	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
2702	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
2703	RTHCPHYS HCPhys; /* general usage. */
2704	int rc;
2705
2706	/*
2707	* Check that the Guest CR3 and all it's mappings are correct.
2708	*/
2709	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & X86_CR3_PAGE_MASK),
2710	("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
2711	false);
2712	rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
2713	AssertRCReturn(rc, 1);
2714	rc = PGMRamGCPhys2HCPhys(pPGM, cr3 & X86_CR3_PAGE_MASK, &HCPhys);
2715	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
2716	# ifndef IN_GC
2717	RTGCPHYS GCPhys;
2718	rc = PGMPhysHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
2719	AssertRCReturn(rc, 1);
2720	AssertMsgReturn((cr3 & X86_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
2721	# endif
2722	const X86PD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
2723
2724	/*
2725	* Get and check the Shadow CR3.
2726	*/
2727	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2728	const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
2729	unsigned cPDEs = ELEMENTS(pPDDst->a);
2730	# else
2731	const X86PDPAE pPDDst = pPGM->CTXMID(ap,PaePDs[0]); / use it as a 2048 entry PD */
2732	unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
2733	# endif
2734	if (cb != ~(RTGCUINTPTR)0)
2735	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
2736
2737	/** @todo call the other two PGMAssert() functions. /
2738
2739	/*
2740	* Iterate the shadow page directory.
2741	*/
2742	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
2743	unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
2744	cPDEs += iPDDst;
2745	for (;
2746	iPDDst < cPDEs;
2747	iPDDst++, GCPtr += _4G / cPDEs)
2748	{
2749	const SHWPDE PdeDst = pPDDst->a[iPDDst];
2750	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2751	{
2752	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2753	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
2754	{
2755	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
2756	cErrors++;
2757	continue;
2758	}
2759	}
2760	else if ( (PdeDst.u & X86_PDE_P)
2761	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
2762	)
2763	{
2764	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
2765	PPGMPOOLPAGE pPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
2766	if (!pPage)
2767	{
2768	AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
2769	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
2770	cErrors++;
2771	continue;
2772	}
2773	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPage);
2774
2775	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
2776	{
2777	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
2778	GCPtr, (uint64_t)PdeDst.u));
2779	cErrors++;
2780	}
2781
2782	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
2783	{
2784	AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
2785	GCPtr, (uint64_t)PdeDst.u));
2786	cErrors++;
2787	}
2788
2789	const X86PDE PdeSrc = pPDSrc->a[iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)];
2790	if (!PdeSrc.n.u1Present)
2791	{
2792	AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
2793	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
2794	cErrors++;
2795	continue;
2796	}
2797
2798	if ( !PdeSrc.b.u1Size
2799	\|\| !(cr4 & X86_CR4_PSE))
2800	{
2801	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
2802	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2803	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2804	# endif
2805	}
2806	else
2807	{
2808	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
2809	{
2810	AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
2811	GCPtr, (uint64_t)PdeSrc.u));
2812	cErrors++;
2813	continue;
2814	}
2815	GCPhysGst = PdeSrc.u & GST_PDE4M_PG_MASK;
2816	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2817	GCPhysGst \|= GCPtr & BIT(X86_PAGE_2M_SHIFT);
2818	# endif
2819	}
2820
2821	if ( pPage->enmKind
2822	!= (!PdeSrc.b.u1Size \|\| !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
2823	{
2824	AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
2825	pPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
2826	cErrors++;
2827	}
2828
2829	rc = PGMRamGCPhys2HCPhysWithFlags(pPGM, GCPhysGst, &HCPhys);
2830	if (VBOX_FAILURE(rc))
2831	{
2832	AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
2833	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
2834	cErrors++;
2835	continue;
2836	}
2837
2838	if (GCPhysGst != pPage->GCPhys)
2839	{
2840	AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
2841	GCPhysGst, pPage->GCPhys, GCPtr));
2842	cErrors++;
2843	continue;
2844	}
2845
2846	if ( !PdeSrc.b.u1Size
2847	\|\| !(cr4 & X86_CR4_PSE))
2848	{
2849	/*
2850	* Page Table.
2851	*/
2852	const GSTPT *pPTSrc;
2853	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
2854	if (VBOX_FAILURE(rc))
2855	{
2856	AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
2857	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
2858	cErrors++;
2859	continue;
2860	}
2861	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
2862	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
2863	{
2864	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
2865	// (This problem will go away when/if we shadow multiple CR3s.)
2866	AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
2867	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
2868	cErrors++;
2869	continue;
2870	}
2871	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
2872	{
2873	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
2874	GCPtr, (uint64_t)PdeDst.u));
2875	cErrors++;
2876	continue;
2877	}
2878
2879	/* iterate the page table. */
2880	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2881	const unsigned offPTSrc = 0;
2882	# else
2883	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
2884	# endif
2885	for (unsigned iPT = 0, off = 0;
2886	iPT < ELEMENTS(pPTDst->a);
2887	iPT++, off += PAGE_SIZE)
2888	{
2889	const SHWPTE PteDst = pPTDst->a[iPT];
2890
2891	/* skip not-present entries. */
2892	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
2893	continue;
2894	Assert(PteDst.n.u1Present);
2895
2896	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
2897	if (!PteSrc.n.u1Present)
2898	{
2899	#ifdef IN_RING3
2900	PGMAssertHandlerAndFlagsInSync(pVM);
2901	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
2902	#endif
2903	AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
2904	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
2905	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
2906	cErrors++;
2907	continue;
2908	}
2909
2910	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
2911	# if 1 /** @todo sync accessed bit properly... */
2912	fIgnoreFlags \|= X86_PTE_A;
2913	# endif
2914
2915	/* match the physical addresses */
2916	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
2917	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
2918
2919	# ifdef IN_RING3
2920	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
2921	if (VBOX_FAILURE(rc))
2922	{
2923	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
2924	{
2925	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
2926	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2927	cErrors++;
2928	continue;
2929	}
2930	}
2931	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
2932	{
2933	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
2934	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2935	cErrors++;
2936	continue;
2937	}
2938	# endif
2939
2940	rc = PGMRamGCPhys2HCPhysWithFlags(pPGM, GCPhysGst, &HCPhys);
2941	if (VBOX_FAILURE(rc))
2942	{
2943	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
2944	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
2945	{
2946	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
2947	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2948	cErrors++;
2949	continue;
2950	}
2951	# endif
2952	if (PteDst.n.u1Write)
2953	{
2954	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
2955	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2956	cErrors++;
2957	}
2958	fIgnoreFlags \|= X86_PTE_RW;
2959	}
2960	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
2961	{
2962	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
2963	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2964	cErrors++;
2965	continue;
2966	}
2967
2968	/* flags */
2969	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
2970	{
2971	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
2972	{
2973	if (PteDst.n.u1Write)
2974	{
2975	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
2976	GCPtr + off, HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2977	cErrors++;
2978	continue;
2979	}
2980	fIgnoreFlags \|= X86_PTE_RW;
2981	}
2982	else
2983	{
2984	if (PteDst.n.u1Present)
2985	{
2986	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
2987	GCPtr + off, HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
2988	cErrors++;
2989	continue;
2990	}
2991	fIgnoreFlags \|= X86_PTE_P;
2992	}
2993	}
2994	else
2995	{
2996	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
2997	{
2998	if (PteDst.n.u1Write)
2999	{
3000	AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3001	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3002	cErrors++;
3003	continue;
3004	}
3005	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3006	{
3007	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3008	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3009	cErrors++;
3010	continue;
3011	}
3012	if (PteDst.n.u1Dirty)
3013	{
3014	AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3015	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3016	cErrors++;
3017	}
3018	# if 0 /** @todo sync access bit properly... */
3019	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3020	{
3021	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3022	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3023	cErrors++;
3024	}
3025	fIgnoreFlags \|= X86_PTE_RW;
3026	# else
3027	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3028	# endif
3029	}
3030	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3031	{
3032	/* access bit emulation (not implemented). */
3033	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3034	{
3035	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3036	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3037	cErrors++;
3038	continue;
3039	}
3040	if (!PteDst.n.u1Accessed)
3041	{
3042	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3043	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3044	cErrors++;
3045	}
3046	fIgnoreFlags \|= X86_PTE_P;
3047	}
3048	# ifdef DEBUG_sandervl
3049	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3050	# endif
3051	}
3052
3053	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3054	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3055	)
3056	{
3057	AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3058	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3059	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3060	cErrors++;
3061	continue;
3062	}
3063	} /* foreach PTE */
3064	}
3065	else
3066	{
3067	/*
3068	* Big Page.
3069	*/
3070	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| X86_PDE_PAE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3071	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3072	{
3073	if (PdeDst.n.u1Write)
3074	{
3075	AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3076	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3077	cErrors++;
3078	continue;
3079	}
3080	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3081	{
3082	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3083	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3084	cErrors++;
3085	continue;
3086	}
3087	# if 0 /** @todo sync access bit properly... */
3088	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3089	{
3090	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3091	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3092	cErrors++;
3093	}
3094	fIgnoreFlags \|= X86_PTE_RW;
3095	# else
3096	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3097	# endif
3098	}
3099	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3100	{
3101	/* access bit emulation (not implemented). */
3102	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3103	{
3104	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3105	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3106	cErrors++;
3107	continue;
3108	}
3109	if (!PdeDst.n.u1Accessed)
3110	{
3111	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3112	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3113	cErrors++;
3114	}
3115	fIgnoreFlags \|= X86_PTE_P;
3116	}
3117
3118	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3119	{
3120	AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3121	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3122	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3123	cErrors++;
3124	}
3125
3126	/* iterate the page table. */
3127	for (unsigned iPT = 0, off = 0;
3128	iPT < ELEMENTS(pPTDst->a);
3129	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3130	{
3131	const SHWPTE PteDst = pPTDst->a[iPT];
3132
3133	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3134	{
3135	AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3136	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3137	cErrors++;
3138	}
3139
3140	/* skip not-present entries. */
3141	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3142	continue;
3143
3144	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3145
3146	/* match the physical addresses */
3147	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3148
3149	# ifdef IN_RING3
3150	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3151	if (VBOX_FAILURE(rc))
3152	{
3153	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3154	{
3155	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3156	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3157	cErrors++;
3158	}
3159	}
3160	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3161	{
3162	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3163	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3164	cErrors++;
3165	continue;
3166	}
3167	# endif
3168
3169	rc = PGMRamGCPhys2HCPhysWithFlags(pPGM, GCPhysGst, &HCPhys);
3170	if (VBOX_FAILURE(rc))
3171	{
3172	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3173	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3174	{
3175	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3176	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3177	cErrors++;
3178	continue;
3179	}
3180	# endif
3181	if (PteDst.n.u1Write)
3182	{
3183	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3184	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3185	cErrors++;
3186	}
3187	fIgnoreFlags \|= X86_PTE_RW;
3188	}
3189	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3190	{
3191	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3192	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3193	cErrors++;
3194	continue;
3195	}
3196
3197	/* flags */
3198	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL \| MM_RAM_FLAGS_VIRTUAL_ALL \| MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
3199	{
3200	if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_WRITE \| MM_RAM_FLAGS_VIRTUAL_WRITE))
3201	{
3202	if (!(HCPhys & MM_RAM_FLAGS_PHYSICAL_TEMP_OFF))
3203	{
3204	if (PteDst.n.u1Write)
3205	{
3206	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3207	GCPtr + off, HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3208	cErrors++;
3209	continue;
3210	}
3211	fIgnoreFlags \|= X86_PTE_RW;
3212	}
3213	}
3214	else
3215	{
3216	if (PteDst.n.u1Present)
3217	{
3218	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3219	GCPtr + off, HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3220	cErrors++;
3221	continue;
3222	}
3223	fIgnoreFlags \|= X86_PTE_P;
3224	}
3225	}
3226
3227	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3228	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3229	)
3230	{
3231	AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3232	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3233	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3234	cErrors++;
3235	continue;
3236	}
3237	} /* foreach PTE */
3238	}
3239	}
3240	/* not present */
3241
3242	} /* forearch PDE */
3243
3244	# ifdef DEBUG
3245	if (cErrors)
3246	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3247	# endif
3248
3249	#elif PGM_GST_TYPE == PGM_TYPE_PAE
3250	//# error not implemented
3251
3252
3253	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3254	//# error not implemented
3255
3256	/#else: guest real and protected mode /
3257	#endif
3258	return cErrors;
3259	}
3260	#endif /* VBOX_STRICT */
3261

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 2096

Download in other formats: