PGMAllBth.h@ 107200

Last change on this file since 107200 was 107171, checked in by vboxsync, 2 months ago
VMM/PGM: Introducing VBOX_WITH_ONLY_PGM_NEM_MODE to disable lots unused code on *.arm64 and darwin. jiraref:VBP-1466
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 231.4 KB

Line
1	/* $Id: PGMAllBth.h 107171 2024-11-28 10:38:10Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7	* bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8	*
9	* @remarks This file is one big \#ifdef-orgy!
10	*
11	*/
12
13	/*
14	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
15	*
16	* This file is part of VirtualBox base platform packages, as
17	* available from https://www.virtualbox.org.
18	*
19	* This program is free software; you can redistribute it and/or
20	* modify it under the terms of the GNU General Public License
21	* as published by the Free Software Foundation, in version 3 of the
22	* License.
23	*
24	* This program is distributed in the hope that it will be useful, but
25	* WITHOUT ANY WARRANTY; without even the implied warranty of
26	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
27	* General Public License for more details.
28	*
29	* You should have received a copy of the GNU General Public License
30	* along with this program; if not, see <https://www.gnu.org/licenses>.
31	*
32	* SPDX-License-Identifier: GPL-3.0-only
33	*/
34
35	#ifdef _MSC_VER
36	/** @todo we're generating unnecessary code in nested/ept shadow mode and for
37	* real/prot-guest+RC mode. */
38	# pragma warning(disable: 4505)
39	#endif
40
41
42	/*********************************************************************************************************************************
43	* Internal Functions *
44	*********************************************************************************************************************************/
45	RT_C_DECLS_BEGIN
46	PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
47	#ifndef IN_RING3
48	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken);
49	PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
50	bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken);
51	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT && !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
52	static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
53	unsigned iPte, SLATPTE GstSlatPte);
54	static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
55	uint32_t uErr, PPGMPTWALKGST pGstWalkAll);
56	static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll);
57	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
58	#endif
59	PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
60	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
61	static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
62	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
63	static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
64	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
65	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
66	# else
67	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
68	# endif
69	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
70	PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
71	PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
72	#ifdef VBOX_STRICT
73	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
74	#endif
75	PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
76	PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
77
78	#ifdef IN_RING3
79	PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
80	#endif
81	RT_C_DECLS_END
82
83
84
85
86	/*
87	* Filter out some illegal combinations of guest and shadow paging, so we can
88	* remove redundant checks inside functions.
89	*/
90	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
91	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
92	# error "Invalid combination; PAE guest implies PAE shadow"
93	#endif
94
95	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
96	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \
97	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
98	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
99	#endif
100
101	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
102	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
103	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
104	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
105	#endif
106
107	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
108	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
109	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
110	#endif
111
112
113	/**
114	* Enters the shadow+guest mode.
115	*
116	* @returns VBox status code.
117	* @param pVCpu The cross context virtual CPU structure.
118	* @param GCPhysCR3 The physical address from the CR3 register.
119	*/
120	PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
121	{
122	/* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
123	* Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
124	*/
125	#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
126	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
127	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
128	&& ( PGM_GST_TYPE == PGM_TYPE_REAL \
129	\|\| PGM_GST_TYPE == PGM_TYPE_PROT)) \
130	&& !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
131
132	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
133
134	Assert(!pVM->pgm.s.fNestedPaging);
135
136	PGM_LOCK_VOID(pVM);
137	/* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
138	* but any calls to GC need a proper shadow page setup as well.
139	*/
140	/* Free the previous root mapping if still active. */
141	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
142	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
143	if (pOldShwPageCR3)
144	{
145	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
146
147	/* Mark the page as unlocked; allow flushing again. */
148	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
149
150	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
151	pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
152	pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
153	}
154
155	/* construct a fake address. */
156	GCPhysCR3 = RT_BIT_64(63);
157	PPGMPOOLPAGE pNewShwPageCR3;
158	int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
159	NIL_PGMPOOL_IDX, UINT32_MAX, false /fLockPage/,
160	&pNewShwPageCR3);
161	AssertRCReturn(rc, rc);
162
163	pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
164	pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
165
166	/* Mark the page as locked; disallow flushing. */
167	pgmPoolLockPage(pPool, pNewShwPageCR3);
168
169	/* Set the current hypervisor CR3. */
170	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
171
172	PGM_UNLOCK(pVM);
173	return rc;
174	#else
175	NOREF(pVCpu); NOREF(GCPhysCR3);
176	return VINF_SUCCESS;
177	#endif
178	}
179
180
181	#ifndef IN_RING3
182
183	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
184	/**
185	* Deal with a guest page fault.
186	*
187	* @returns Strict VBox status code.
188	* @retval VINF_EM_RAW_GUEST_TRAP
189	* @retval VINF_EM_RAW_EMULATE_INSTR
190	*
191	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
192	* @param pWalk The guest page table walk result.
193	* @param uErr The error code.
194	*/
195	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr)
196	{
197	/*
198	* Calc the error code for the guest trap.
199	*/
200	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
201	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
202	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
203	if ( pWalk->fRsvdError
204	\|\| pWalk->fBadPhysAddr)
205	{
206	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
207	Assert(!pWalk->fNotPresent);
208	}
209	else if (!pWalk->fNotPresent)
210	uNewErr \|= X86_TRAP_PF_P;
211	TRPMSetErrorCode(pVCpu, uNewErr);
212
213	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel));
214	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
215	return VINF_EM_RAW_GUEST_TRAP;
216	}
217	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
218
219
220	#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE && !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
221	/**
222	* Deal with a guest page fault.
223	*
224	* The caller has taken the PGM lock.
225	*
226	* @returns Strict VBox status code.
227	*
228	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
229	* @param uErr The error code.
230	* @param pCtx Pointer to the register context for the CPU.
231	* @param pvFault The fault address.
232	* @param pPage The guest page at @a pvFault.
233	* @param pWalk The guest page table walk result.
234	* @param pGstWalk The guest paging-mode specific walk information.
235	* @param pfLockTaken PGM lock taken here or not (out). This is true
236	* when we're called.
237	*/
238	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
239	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
240	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
241	, PPGMPTWALK pWalk
242	, PGSTPTWALK pGstWalk
243	# endif
244	)
245	{
246	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
247	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
248	# endif
249	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
250	VBOXSTRICTRC rcStrict;
251
252	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
253	{
254	/*
255	* Physical page access handler.
256	*/
257	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
258	const RTGCPHYS GCPhysFault = pWalk->GCPhys;
259	# else
260	const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
261	# endif
262	PPGMPHYSHANDLER pCur;
263	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysFault, &pCur);
264	if (RT_SUCCESS(rcStrict))
265	{
266	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
267
268	# ifdef PGM_SYNC_N_PAGES
269	/*
270	* If the region is write protected and we got a page not present fault, then sync
271	* the pages. If the fault was caused by a read, then restart the instruction.
272	* In case of write access continue to the GC write handler.
273	*
274	* ASSUMES that there is only one handler per page or that they have similar write properties.
275	*/
276	if ( !(uErr & X86_TRAP_PF_P)
277	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
278	{
279	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
280	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
281	# else
282	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
283	# endif
284	if ( RT_FAILURE(rcStrict)
285	\|\| !(uErr & X86_TRAP_PF_RW)
286	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
287	{
288	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
289	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
290	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
291	return rcStrict;
292	}
293	}
294	# endif
295	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
296	/*
297	* If the access was not thru a #PF(RSVD\|...) resync the page.
298	*/
299	if ( !(uErr & X86_TRAP_PF_RSVD)
300	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
301	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
302	&& (pWalk->fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK))
303	== PGM_PTATTRS_W_MASK /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
304	# endif
305	)
306	{
307	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
308	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
309	# else
310	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
311	# endif
312	if ( RT_FAILURE(rcStrict)
313	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
314	{
315	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
316	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
317	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
318	return rcStrict;
319	}
320	}
321	# endif
322
323	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
324	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
325	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
326	pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
327	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
328	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
329	else
330	{
331	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
332	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
333	}
334
335	if (pCurType->pfnPfHandler)
336	{
337	STAM_PROFILE_START(&pCur->Stat, h);
338
339	if (pCurType->fKeepPgmLock)
340	{
341	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault,
342	!pCurType->fRing0DevInsIdx ? pCur->uUser
343	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser));
344
345	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
346	}
347	else
348	{
349	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
350	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
351	PGM_UNLOCK(pVM);
352	*pfLockTaken = false;
353
354	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault, uUser);
355
356	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
357	}
358	}
359	else
360	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
361
362	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
363	return rcStrict;
364	}
365	AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
366	}
367
368	/*
369	* There is a handled area of the page, but this fault doesn't belong to it.
370	* We must emulate the instruction.
371	*
372	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
373	* we first check if this was a page-not-present fault for a page with only
374	* write access handlers. Restart the instruction if it wasn't a write access.
375	*/
376	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled);
377
378	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
379	&& !(uErr & X86_TRAP_PF_P))
380	{
381	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
382	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
383	# else
384	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
385	# endif
386	if ( RT_FAILURE(rcStrict)
387	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
388	\|\| !(uErr & X86_TRAP_PF_RW))
389	{
390	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
391	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
392	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
393	return rcStrict;
394	}
395	}
396
397	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
398	* It's writing to an unhandled part of the LDT page several million times.
399	*/
400	rcStrict = PGMInterpretInstruction(pVCpu, pvFault);
401	LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
402	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; });
403	return rcStrict;
404	} /* if any kind of handler */
405	# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
406
407
408	/**
409	* \#PF Handler for raw-mode guest execution.
410	*
411	* @returns VBox status code (appropriate for trap handling and GC return).
412	*
413	* @param pVCpu The cross context virtual CPU structure.
414	* @param uErr The trap error code.
415	* @param pCtx Pointer to the register context for the CPU.
416	* @param pvFault The fault address.
417	* @param pfLockTaken PGM lock taken here or not (out)
418	*/
419	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken)
420	{
421	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
422
423	*pfLockTaken = false;
424
425	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
426	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
427	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
428	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
429	&& PGM_SHW_TYPE != PGM_TYPE_NONE \
430	&& !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
431	int rc;
432
433	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
434	/*
435	* Walk the guest page translation tables and check if it's a guest fault.
436	*/
437	PGMPTWALK Walk;
438	GSTPTWALK GstWalk;
439	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk);
440	if (RT_FAILURE_NP(rc))
441	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
442
443	/* assert some GstWalk sanity. */
444	# if PGM_GST_TYPE == PGM_TYPE_AMD64
445	/AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. /
446	# endif
447	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
448	/AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto /
449	# endif
450	/AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto /
451	/AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto /
452	Assert(Walk.fSucceeded);
453	Assert(Walk.fEffective & PGM_PTATTRS_R_MASK);
454
455	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
456	{
457	if ( ( (uErr & X86_TRAP_PF_RW)
458	&& !(Walk.fEffective & PGM_PTATTRS_W_MASK)
459	&& ( (uErr & X86_TRAP_PF_US)
460	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
461	\|\| ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK))
462	\|\| ((uErr & X86_TRAP_PF_ID) && (Walk.fEffective & PGM_PTATTRS_NX_MASK))
463	)
464	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
465	}
466
467	/* Take the big lock now before we update flags. */
468	*pfLockTaken = true;
469	PGM_LOCK_VOID(pVM);
470
471	/*
472	* Set the accessed and dirty flags.
473	*/
474	/** @todo Should probably use cmpxchg logic here as we're potentially racing
475	* other CPUs in SMP configs. (the lock isn't enough, since we take it
476	* after walking and the page tables could be stale already) */
477	# if PGM_GST_TYPE == PGM_TYPE_AMD64
478	if (!(GstWalk.Pml4e.u & X86_PML4E_A))
479	{
480	GstWalk.Pml4e.u \|= X86_PML4E_A;
481	GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A);
482	}
483	if (!(GstWalk.Pdpe.u & X86_PDPE_A))
484	{
485	GstWalk.Pdpe.u \|= X86_PDPE_A;
486	GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A);
487	}
488	# endif
489	if (Walk.fBigPage)
490	{
491	Assert(GstWalk.Pde.u & X86_PDE_PS);
492	if (uErr & X86_TRAP_PF_RW)
493	{
494	if ((GstWalk.Pde.u & (X86_PDE4M_A \| X86_PDE4M_D)) != (X86_PDE4M_A \| X86_PDE4M_D))
495	{
496	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
497	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A \| X86_PDE4M_D);
498	}
499	}
500	else
501	{
502	if (!(GstWalk.Pde.u & X86_PDE4M_A))
503	{
504	GstWalk.Pde.u \|= X86_PDE4M_A;
505	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A);
506	}
507	}
508	}
509	else
510	{
511	Assert(!(GstWalk.Pde.u & X86_PDE_PS));
512	if (!(GstWalk.Pde.u & X86_PDE_A))
513	{
514	GstWalk.Pde.u \|= X86_PDE_A;
515	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A);
516	}
517
518	if (uErr & X86_TRAP_PF_RW)
519	{
520	# ifdef VBOX_WITH_STATISTICS
521	if (GstWalk.Pte.u & X86_PTE_D)
522	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty));
523	else
524	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage));
525	# endif
526	if ((GstWalk.Pte.u & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
527	{
528	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
529	GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A \| X86_PTE_D);
530	}
531	}
532	else
533	{
534	if (!(GstWalk.Pte.u & X86_PTE_A))
535	{
536	GstWalk.Pte.u \|= X86_PTE_A;
537	GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A);
538	}
539	}
540	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
541	}
542	#if 0
543	/* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */
544	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
545	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
546	#endif
547
548	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
549	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
550
551	/* Take the big lock now. */
552	*pfLockTaken = true;
553	PGM_LOCK_VOID(pVM);
554	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
555
556	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
557	/*
558	* If it is a reserved bit fault we know that it is an MMIO (access
559	* handler) related fault and can skip some 200 lines of code.
560	*/
561	if (uErr & X86_TRAP_PF_RSVD)
562	{
563	Assert(uErr & X86_TRAP_PF_P);
564	PPGMPAGE pPage;
565	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
566	rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage);
567	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
568	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage,
569	pfLockTaken, &Walk, &GstWalk));
570	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
571	# else
572	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
573	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
574	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
575	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
576	# endif
577	AssertRC(rc);
578	PGM_INVL_PG(pVCpu, pvFault);
579	return rc; /* Restart with the corrected entry. */
580	}
581	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
582
583	/*
584	* Fetch the guest PDE, PDPE and PML4E.
585	*/
586	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
587	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
588	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
589
590	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
591	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
592	PX86PDPAE pPDDst;
593	# if PGM_GST_TYPE == PGM_TYPE_PAE
594	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
595	# else
596	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
597	# endif
598	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
599
600	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
601	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
602	PX86PDPAE pPDDst;
603	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
604	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
605	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
606	# else
607	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
608	# endif
609	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
610
611	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
612	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
613	PEPTPD pPDDst;
614	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
615	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
616	# endif
617	Assert(pPDDst);
618
619	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
620	/*
621	* Dirty page handling.
622	*
623	* If we successfully correct the write protection fault due to dirty bit
624	* tracking, then return immediately.
625	*/
626	if (uErr & X86_TRAP_PF_RW) /* write fault? */
627	{
628	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
629	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
630	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
631	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
632	{
633	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0
634	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
635	? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed
636	: &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
637	Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
638	return VINF_SUCCESS;
639	}
640	#ifdef DEBUG_bird
641	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
642	AssertMsg(Walk.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
643	#endif
644	}
645
646	# if 0 /* rarely useful; leave for debugging. */
647	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
648	# endif
649	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
650
651	/*
652	* A common case is the not-present error caused by lazy page table syncing.
653	*
654	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
655	* here so we can safely assume that the shadow PT is present when calling
656	* SyncPage later.
657	*
658	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
659	* of mapping conflict and defer to SyncCR3 in R3.
660	* (Again, we do NOT support access handlers for non-present guest pages.)
661	*
662	*/
663	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
664	Assert(GstWalk.Pde.u & X86_PDE_P);
665	# endif
666	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
667	&& !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
668	{
669	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
670	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
671	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
672	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
673	# else
674	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
675	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
676	# endif
677	if (RT_SUCCESS(rc))
678	return rc;
679	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
680	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
681	return VINF_PGM_SYNC_CR3;
682	}
683
684	/*
685	* Check if this fault address is flagged for special treatment,
686	* which means we'll have to figure out the physical address and
687	* check flags associated with it.
688	*
689	* ASSUME that we can limit any special access handling to pages
690	* in page tables which the guest believes to be present.
691	*/
692	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
693	RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
694	# else
695	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
696	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
697	PPGMPAGE pPage;
698	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
699	if (RT_FAILURE(rc))
700	{
701	/*
702	* When the guest accesses invalid physical memory (e.g. probing
703	* of RAM or accessing a remapped MMIO range), then we'll fall
704	* back to the recompiler to emulate the instruction.
705	*/
706	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
707	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
708	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
709	return VINF_EM_RAW_EMULATE_INSTR;
710	}
711
712	/*
713	* Any handlers for this page?
714	*/
715	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
716	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
717	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken,
718	&Walk, &GstWalk));
719	# else
720	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
721	# endif
722
723	/*
724	* We are here only if page is present in Guest page tables and
725	* trap is not handled by our handlers.
726	*
727	* Check it for page out-of-sync situation.
728	*/
729	if (!(uErr & X86_TRAP_PF_P))
730	{
731	/*
732	* Page is not present in our page tables. Try to sync it!
733	*/
734	if (uErr & X86_TRAP_PF_US)
735	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
736	else /* supervisor */
737	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
738
739	if (PGM_PAGE_IS_BALLOONED(pPage))
740	{
741	/* Emulate reads from ballooned pages as they are not present in
742	our shadow page tables. (Required for e.g. Solaris guests; soft
743	ecc, random nr generator.) */
744	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
745	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
746	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
747	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
748	return rc;
749	}
750
751	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
752	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
753	# else
754	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
755	# endif
756	if (RT_SUCCESS(rc))
757	{
758	/* The page was successfully synced, return to the guest. */
759	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
760	return VINF_SUCCESS;
761	}
762	}
763	else /* uErr & X86_TRAP_PF_P: */
764	{
765	/*
766	* Write protected pages are made writable when the guest makes the
767	* first write to it. This happens for pages that are shared, write
768	* monitored or not yet allocated.
769	*
770	* We may also end up here when CR0.WP=0 in the guest.
771	*
772	* Also, a side effect of not flushing global PDEs are out of sync
773	* pages due to physical monitored regions, that are no longer valid.
774	* Assume for now it only applies to the read/write flag.
775	*/
776	if (uErr & X86_TRAP_PF_RW)
777	{
778	/*
779	* Check if it is a read-only page.
780	*/
781	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
782	{
783	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
784	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
785	Assert(!PGM_PAGE_IS_ZERO(pPage));
786	# endif
787	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
788	# ifdef PGM_WITH_PAGE_ZEROING_DETECTION
789	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ZERO
790	&& (pvFault & X86_PAGE_OFFSET_MASK) == 0
791	&& pgmHandlePageZeroingCode(pVCpu, pCtx))
792	{
793	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2PageZeroing; });
794	return VINF_SUCCESS;
795	}
796	# endif
797	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
798
799	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
800	if (rc != VINF_SUCCESS)
801	{
802	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
803	return rc;
804	}
805	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
806	return VINF_EM_NO_MEMORY;
807	}
808
809	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
810	/*
811	* Check to see if we need to emulate the instruction if CR0.WP=0.
812	*/
813	if ( !(Walk.fEffective & PGM_PTATTRS_W_MASK)
814	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
815	&& CPUMGetGuestCPL(pVCpu) < 3)
816	{
817	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
818
819	/*
820	* The Netware WP0+RO+US hack.
821	*
822	* Netware sometimes(/always?) runs with WP0. It has been observed doing
823	* excessive write accesses to pages which are mapped with US=1 and RW=0
824	* while WP=0. This causes a lot of exits and extremely slow execution.
825	* To avoid trapping and emulating every write here, we change the shadow
826	* page table entry to map it as US=0 and RW=1 until user mode tries to
827	* access it again (see further below). We count these shadow page table
828	* changes so we can avoid having to clear the page pool every time the WP
829	* bit changes to 1 (see PGMCr0WpEnabled()).
830	*/
831	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && 1
832	if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
833	&& (Walk.fBigPage \|\| (GstWalk.Pde.u & X86_PDE_RW))
834	&& pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
835	{
836	Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage));
837	rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
838	if (rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3)
839	{
840	PGM_INVL_PG(pVCpu, pvFault);
841	pVCpu->pgm.s.cNetwareWp0Hacks++;
842	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; });
843	return rc;
844	}
845	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
846	Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
847	}
848	# endif
849
850	/* Interpret the access. */
851	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
852	Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK)));
853	if (RT_SUCCESS(rc))
854	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ);
855	else
856	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3);
857	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; });
858	return rc;
859	}
860	# endif
861	/// @todo count the above case; else
862	if (uErr & X86_TRAP_PF_US)
863	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
864	else /* supervisor */
865	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
866
867	/*
868	* Sync the page.
869	*
870	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
871	* page is not present, which is not true in this case.
872	*/
873	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
874	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
875	# else
876	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
877	# endif
878	if (RT_SUCCESS(rc))
879	{
880	/*
881	* Page was successfully synced, return to guest but invalidate
882	* the TLB first as the page is very likely to be in it.
883	*/
884	# if PGM_SHW_TYPE == PGM_TYPE_EPT
885	HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
886	# else
887	PGM_INVL_PG(pVCpu, pvFault);
888	# endif
889	# ifdef VBOX_STRICT
890	PGMPTWALK GstPageWalk;
891	GstPageWalk.GCPhys = RTGCPHYS_MAX;
892	if (!pVM->pgm.s.fNestedPaging)
893	{
894	rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
895	AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) \|\| ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective));
896	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective));
897	}
898	# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
899	uint64_t fPageShw = 0;
900	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
901	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
902	("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault));
903	# endif
904	# endif /* VBOX_STRICT */
905	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
906	return VINF_SUCCESS;
907	}
908	}
909	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
910	/*
911	* Check for Netware WP0+RO+US hack from above and undo it when user
912	* mode accesses the page again.
913	*/
914	else if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
915	&& (Walk.fBigPage \|\| (GstWalk.Pde.u & X86_PDE_RW))
916	&& pVCpu->pgm.s.cNetwareWp0Hacks > 0
917	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
918	&& CPUMGetGuestCPL(pVCpu) == 3
919	&& pVM->cCpus == 1
920	)
921	{
922	Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
923	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
924	if (RT_SUCCESS(rc))
925	{
926	PGM_INVL_PG(pVCpu, pvFault);
927	pVCpu->pgm.s.cNetwareWp0Hacks--;
928	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; });
929	return VINF_SUCCESS;
930	}
931	}
932	# endif /* PGM_WITH_PAGING */
933
934	/** @todo else: why are we here? */
935
936	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
937	/*
938	* Check for VMM page flags vs. Guest page flags consistency.
939	* Currently only for debug purposes.
940	*/
941	if (RT_SUCCESS(rc))
942	{
943	/* Get guest page flags. */
944	PGMPTWALK GstPageWalk;
945	int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
946	if (RT_SUCCESS(rc2))
947	{
948	uint64_t fPageShw = 0;
949	rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
950
951	#if 0
952	/*
953	* Compare page flags.
954	* Note: we have AVL, A, D bits desynced.
955	*/
956	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
957	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
958	\|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
959	&& (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
960	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
961	&& (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW
962	&& (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US),
963	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
964	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
965	01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
966	01:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
967
968	01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
969	01:01:15.625516 00:08:43.268051 Location :
970	e:\vbox\svn\trunk\srcPage flags mismatch!
971	pvFault=fffff801b0d7b000
972	uErr=11 X86_TRAP_PF_ID \| X86_TRAP_PF_P
973	GCPhys=0000000019b52000
974	fPageShw=0
975	fPageGst=77b0000000000121
976	rc=0
977	#endif
978
979	}
980	else
981	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
982	}
983	else
984	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
985	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
986	}
987
988
989	/*
990	* If we get here it is because something failed above, i.e. most like guru
991	* meditiation time.
992	*/
993	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
994	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pCtx->cs.Sel, pCtx->rip));
995	return rc;
996
997	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
998	NOREF(uErr); NOREF(pCtx); NOREF(pvFault);
999	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1000	return VERR_PGM_NOT_USED_IN_MODE;
1001	# endif
1002	}
1003
1004
1005	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
1006	/**
1007	* Deals with a nested-guest \#PF fault for a guest-physical page with a handler.
1008	*
1009	* @returns Strict VBox status code.
1010	* @param pVCpu The cross context virtual CPU structure.
1011	* @param uErr The error code.
1012	* @param pCtx Pointer to the register context for the CPU.
1013	* @param GCPhysNestedFault The nested-guest physical address of the fault.
1014	* @param pPage The guest page at @a GCPhysNestedFault.
1015	* @param GCPhysFault The guest-physical address of the fault.
1016	* @param pGstWalkAll The guest page walk result.
1017	* @param pfLockTaken Where to store whether the PGM is still held when
1018	* this function completes.
1019	*
1020	* @note The caller has taken the PGM lock.
1021	*/
1022	static VBOXSTRICTRC PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
1023	RTGCPHYS GCPhysNestedFault, PPGMPAGE pPage,
1024	RTGCPHYS GCPhysFault, PPGMPTWALKGST pGstWalkAll,
1025	bool *pfLockTaken)
1026	{
1027	# if PGM_GST_TYPE == PGM_TYPE_PROT \
1028	&& PGM_SHW_TYPE == PGM_TYPE_EPT
1029
1030	/** @todo Assert uErr isn't X86_TRAP_PF_RSVD and remove release checks. */
1031	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysFault);
1032	AssertMsgReturn(PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage), ("%RGp %RGp uErr=%u\n", GCPhysNestedFault, GCPhysFault, uErr),
1033	VERR_PGM_HANDLER_IPE_1);
1034
1035	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1036	RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1037	RTGCPHYS const GCPhysPage = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1038
1039	/*
1040	* Physical page access handler.
1041	*/
1042	PPGMPHYSHANDLER pCur;
1043	VBOXSTRICTRC rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pCur);
1044	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
1045
1046	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
1047	Assert(pCurType);
1048
1049	/*
1050	* If the region is write protected and we got a page not present fault, then sync
1051	* the pages. If the fault was caused by a read, then restart the instruction.
1052	* In case of write access continue to the GC write handler.
1053	*/
1054	if ( !(uErr & X86_TRAP_PF_P)
1055	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1056	{
1057	Log7Func(("Syncing Monitored: GCPhysNestedPage=%RGp GCPhysPage=%RGp uErr=%#x\n", GCPhysNestedPage, GCPhysPage, uErr));
1058	rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /cPages/, uErr, pGstWalkAll);
1059	Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1060	if ( RT_FAILURE(rcStrict)
1061	\|\| !(uErr & X86_TRAP_PF_RW))
1062	{
1063	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1064	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1065	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1066	return rcStrict;
1067	}
1068	}
1069	else if ( !(uErr & X86_TRAP_PF_RSVD)
1070	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE)
1071	{
1072	/*
1073	* If the access was NOT through an EPT misconfig (i.e. RSVD), sync the page.
1074	* This can happen for the VMX APIC-access page.
1075	*/
1076	Log7Func(("Syncing MMIO: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1077	rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /cPages/, uErr, pGstWalkAll);
1078	Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1079	if (RT_FAILURE(rcStrict))
1080	{
1081	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1082	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1083	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1084	return rcStrict;
1085	}
1086	}
1087
1088	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
1089	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
1090	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
1091	GCPhysNestedFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
1092	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1093	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
1094	else
1095	{
1096	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
1097	if (uErr & X86_TRAP_PF_RSVD)
1098	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
1099	}
1100
1101	if (pCurType->pfnPfHandler)
1102	{
1103	STAM_PROFILE_START(&pCur->Stat, h);
1104	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
1105	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
1106
1107	if (pCurType->fKeepPgmLock)
1108	{
1109	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1110	STAM_PROFILE_STOP(&pCur->Stat, h);
1111	}
1112	else
1113	{
1114	PGM_UNLOCK(pVM);
1115	*pfLockTaken = false;
1116	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1117	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
1118	}
1119	}
1120	else
1121	{
1122	AssertMsgFailed(("What's going on here!? Fault falls outside handler range!?\n"));
1123	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
1124	}
1125
1126	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
1127	return rcStrict;
1128
1129	# else
1130	RT_NOREF8(pVCpu, uErr, pCtx, GCPhysNestedFault, pPage, GCPhysFault, pGstWalkAll, pfLockTaken);
1131	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1132	return VERR_PGM_NOT_USED_IN_MODE;
1133	# endif
1134	}
1135	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1136
1137
1138	/**
1139	* Nested \#PF handler for nested-guest hardware-assisted execution using nested
1140	* paging.
1141	*
1142	* @returns VBox status code (appropriate for trap handling and GC return).
1143	* @param pVCpu The cross context virtual CPU structure.
1144	* @param uErr The fault error (X86_TRAP_PF_*).
1145	* @param pCtx Pointer to the register context for the CPU.
1146	* @param GCPhysNestedFault The nested-guest physical address of the fault.
1147	* @param fIsLinearAddrValid Whether translation of a nested-guest linear address
1148	* caused this fault. If @c false, GCPtrNestedFault
1149	* must be 0.
1150	* @param GCPtrNestedFault The nested-guest linear address of this fault.
1151	* @param pWalk The guest page table walk result.
1152	* @param pfLockTaken Where to store whether the PGM lock is still held
1153	* when this function completes.
1154	*/
1155	PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
1156	bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken)
1157	{
1158	*pfLockTaken = false;
1159	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \
1160	&& PGM_GST_TYPE == PGM_TYPE_PROT \
1161	&& PGM_SHW_TYPE == PGM_TYPE_EPT \
1162	&& !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
1163
1164	Assert(CPUMIsGuestVmxEptPagingEnabled(pVCpu));
1165	Assert(PGM_A20_IS_ENABLED(pVCpu));
1166
1167	/* We don't support mode-based execute control for EPT yet. */
1168	Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
1169	Assert(!(uErr & X86_TRAP_PF_US));
1170
1171	/* Take the big lock now. */
1172	*pfLockTaken = true;
1173	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1174	PGM_LOCK_VOID(pVM);
1175
1176	/*
1177	* Walk the guest EPT tables and check if it's an EPT violation or misconfiguration.
1178	*/
1179	if (fIsLinearAddrValid)
1180	Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x GCPtrNestedFault=%RGv\n",
1181	pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr, GCPtrNestedFault));
1182	else
1183	Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x\n",
1184	pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr));
1185	PGMPTWALKGST GstWalkAll;
1186	int rc = pgmGstSlatWalk(pVCpu, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk, &GstWalkAll);
1187	if (RT_FAILURE(rc))
1188	return rc;
1189
1190	Assert(GstWalkAll.enmType == PGMPTWALKGSTTYPE_EPT);
1191	Assert(pWalk->fSucceeded);
1192	Assert(pWalk->fEffective & (PGM_PTATTRS_EPT_R_MASK \| PGM_PTATTRS_EPT_W_MASK \| PGM_PTATTRS_EPT_X_SUPER_MASK));
1193	Assert(pWalk->fIsSlat);
1194
1195	# ifdef DEBUG_ramshankar
1196	/* Paranoia. */
1197	Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_R_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_R_MASK));
1198	Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_W_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_W_MASK));
1199	Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_NX_MASK) == !RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_X_SUPER_MASK));
1200	# endif
1201
1202	Log7Func(("SLAT: GCPhysNestedFault=%RGp -> GCPhys=%#RGp\n", GCPhysNestedFault, pWalk->GCPhys));
1203
1204	/*
1205	* Check page-access permissions.
1206	*/
1207	if ( ((uErr & X86_TRAP_PF_RW) && !(pWalk->fEffective & PGM_PTATTRS_W_MASK))
1208	\|\| ((uErr & X86_TRAP_PF_ID) && (pWalk->fEffective & PGM_PTATTRS_NX_MASK)))
1209	{
1210	Log7Func(("Permission failed! GCPtrNested=%RGv GCPhysNested=%RGp uErr=%#x fEffective=%#RX64\n", GCPtrNestedFault,
1211	GCPhysNestedFault, uErr, pWalk->fEffective));
1212	pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION;
1213	return VERR_ACCESS_DENIED;
1214	}
1215
1216	PGM_A20_ASSERT_MASKED(pVCpu, pWalk->GCPhys);
1217	RTGCPHYS const GCPhysPage = pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1218	RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1219
1220	/*
1221	* If we were called via an EPT misconfig, it should've already resulted in a nested-guest VM-exit.
1222	*/
1223	AssertMsgReturn(!(uErr & X86_TRAP_PF_RSVD),
1224	("Unexpected EPT misconfig VM-exit. GCPhysPage=%RGp GCPhysNestedPage=%RGp\n", GCPhysPage, GCPhysNestedPage),
1225	VERR_PGM_MAPPING_IPE);
1226
1227	/*
1228	* Fetch and sync the nested-guest EPT page directory pointer.
1229	*/
1230	PEPTPD pEptPd;
1231	rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL /ppPdpt/, &pEptPd, &GstWalkAll);
1232	AssertRCReturn(rc, rc);
1233	Assert(pEptPd);
1234
1235	/*
1236	* A common case is the not-present error caused by lazy page table syncing.
1237	*
1238	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
1239	* here so we can safely assume that the shadow PT is present when calling
1240	* NestedSyncPage later.
1241	*
1242	* NOTE: It's possible we will be syncing the VMX APIC-access page here.
1243	* In that case, we would sync the page but will NOT go ahead with emulating
1244	* the APIC-access VM-exit through IEM. However, once the page is mapped in
1245	* the shadow tables, subsequent APIC-access VM-exits for the nested-guest
1246	* will be triggered by hardware. Maybe calling the IEM #PF handler can be
1247	* considered as an optimization later.
1248	*/
1249	unsigned const iPde = (GCPhysNestedPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1250	if ( !(uErr & X86_TRAP_PF_P)
1251	&& !(pEptPd->a[iPde].u & EPT_PRESENT_MASK))
1252	{
1253	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
1254	Log7Func(("NestedSyncPT: Lazy. GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1255	rc = PGM_BTH_NAME(NestedSyncPT)(pVCpu, GCPhysNestedPage, GCPhysPage, &GstWalkAll);
1256	if (RT_SUCCESS(rc))
1257	return rc;
1258	AssertMsgFailedReturn(("NestedSyncPT: %RGv failed! rc=%Rrc\n", GCPhysNestedPage, rc), VERR_PGM_MAPPING_IPE);
1259	}
1260
1261	/*
1262	* Check if this fault address is flagged for special treatment.
1263	* This handles faults on an MMIO or write-monitored page.
1264	*
1265	* If this happens to be the VMX APIC-access page, we don't treat is as MMIO
1266	* but rather sync it further below (as a regular guest page) which lets
1267	* hardware-assisted execution trigger the APIC-access VM-exits of the
1268	* nested-guest directly.
1269	*/
1270	PPGMPAGE pPage;
1271	rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1272	if (RT_FAILURE(rc))
1273	{
1274	/*
1275	* We failed to get the physical page which means it's a reserved/invalid
1276	* page address (not MMIO even). This can typically be observed with
1277	* Microsoft Hyper-V enabled Windows guests. We must fall back to emulating
1278	* the instruction, see @bugref{10318#c7}.
1279	*/
1280	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
1281	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
1282	return VINF_EM_RAW_EMULATE_INSTR;
1283	}
1284	/* Check if this is an MMIO page and NOT the VMX APIC-access page. */
1285	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1286	{
1287	Log7Func(("MMIO: Calling NestedTrap0eHandlerDoAccessHandlers for GCPhys %RGp\n", GCPhysPage));
1288	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, GCPhysNestedFault,
1289	pPage, pWalk->GCPhys, &GstWalkAll,
1290	pfLockTaken));
1291	}
1292
1293	/*
1294	* We are here only if page is present in nested-guest page tables but the
1295	* trap is not handled by our handlers. Check for page out-of-sync situation.
1296	*/
1297	if (!(uErr & X86_TRAP_PF_P))
1298	{
1299	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
1300	Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1301	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
1302
1303	Log7Func(("SyncPage: Not-Present: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedFault, GCPhysPage));
1304	rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, PGM_SYNC_NR_PAGES, uErr, &GstWalkAll);
1305	if (RT_SUCCESS(rc))
1306	{
1307	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
1308	return VINF_SUCCESS;
1309	}
1310	}
1311	else if (uErr & X86_TRAP_PF_RW)
1312	{
1313	/*
1314	* Write protected pages are made writable when the guest makes the
1315	* first write to it. This happens for pages that are shared, write
1316	* monitored or not yet allocated.
1317	*
1318	* We may also end up here when CR0.WP=0 in the guest.
1319	*
1320	* Also, a side effect of not flushing global PDEs are out of sync
1321	* pages due to physical monitored regions, that are no longer valid.
1322	* Assume for now it only applies to the read/write flag.
1323	*/
1324	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1325	{
1326	/* This is a read-only page. */
1327	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysPage));
1328	#ifdef PGM_WITH_PAGE_ZEROING_DETECTION
1329	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ZERO
1330	&& (GCPhysNestedFault & X86_PAGE_OFFSET_MASK) == 0
1331	&& pgmHandlePageZeroingCode(pVCpu, pCtx))
1332	{
1333	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2PageZeroing; });
1334	return VINF_SUCCESS;
1335	}
1336	#endif
1337	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
1338
1339	Log7Func(("Calling pgmPhysPageMakeWritable for GCPhysPage=%RGp\n", GCPhysPage));
1340	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1341	if (rc != VINF_SUCCESS)
1342	{
1343	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
1344	return rc;
1345	}
1346	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
1347	return VINF_EM_NO_MEMORY;
1348	}
1349
1350	Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1351	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1352
1353	/*
1354	* Sync the write-protected page.
1355	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1356	* page is not present, which is not true in this case.
1357	*/
1358	Log7Func(("SyncPage: RW: cs:rip=%04x:%#RX64 GCPhysNestedPage=%RGp uErr=%#RX32 GCPhysPage=%RGp WalkGCPhys=%RGp\n",
1359	pCtx->cs.Sel, pCtx->rip, GCPhysNestedPage, (uint32_t)uErr, GCPhysPage, pWalk->GCPhys));
1360	rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /* cPages */, uErr, &GstWalkAll);
1361	if (RT_SUCCESS(rc))
1362	{
1363	HMInvalidatePhysPage(pVM, GCPhysPage);
1364	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
1365	return VINF_SUCCESS;
1366	}
1367	}
1368
1369	/*
1370	* If we get here it is because something failed above => guru meditation time?
1371	*/
1372	LogRelMaxFunc(32, ("rc=%Rrc GCPhysNestedFault=%#RGp (%#RGp) uErr=%#RX32 cs:rip=%04x:%08RX64\n",
1373	rc, GCPhysNestedFault, GCPhysPage, (uint32_t)uErr, pCtx->cs.Sel, pCtx->rip));
1374	return VERR_PGM_MAPPING_IPE;
1375
1376	# else /* !VBOX_WITH_NESTED_HWVIRT_VMX_EPT \|\| PGM_GST_TYPE != PGM_TYPE_PROT \|\| PGM_SHW_TYPE != PGM_TYPE_EPT */
1377	RT_NOREF7(pVCpu, uErr, pCtx, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk);
1378	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1379	return VERR_PGM_NOT_USED_IN_MODE;
1380	# endif
1381	}
1382
1383	#endif /* !IN_RING3 */
1384
1385
1386	/**
1387	* Emulation of the invlpg instruction.
1388	*
1389	*
1390	* @returns VBox status code.
1391	*
1392	* @param pVCpu The cross context virtual CPU structure.
1393	* @param GCPtrPage Page to invalidate.
1394	*
1395	* @remark ASSUMES that the guest is updating before invalidating. This order
1396	* isn't required by the CPU, so this is speculative and could cause
1397	* trouble.
1398	* @remark No TLB shootdown is done on any other VCPU as we assume that
1399	* invlpg emulation is the only reason for calling this function.
1400	* (The guest has to shoot down TLB entries on other CPUs itself)
1401	* Currently true, but keep in mind!
1402	*
1403	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1404	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1405	*/
1406	PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1407	{
1408	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1409	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1410	&& PGM_SHW_TYPE != PGM_TYPE_NONE \
1411	&& !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
1412	int rc;
1413	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1414	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1415
1416	PGM_LOCK_ASSERT_OWNER(pVM);
1417
1418	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1419
1420	/*
1421	* Get the shadow PD entry and skip out if this PD isn't present.
1422	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1423	*/
1424	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1425	const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1426	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1427	AssertReturn(pPdeDst, VERR_INTERNAL_ERROR_3);
1428
1429	/* Fetch the pgm pool shadow descriptor. */
1430	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1431	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1432	if (!pShwPde)
1433	{
1434	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1435	return VINF_SUCCESS;
1436	}
1437	# else
1438	Assert(pShwPde);
1439	# endif
1440
1441	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1442	const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1443	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1444
1445	/* If the shadow PDPE isn't present, then skip the invalidate. */
1446	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1447	if (!pPdptDst \|\| !(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1448	# else
1449	if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1450	# endif
1451	{
1452	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1453	PGM_INVL_PG(pVCpu, GCPtrPage);
1454	return VINF_SUCCESS;
1455	}
1456
1457	/* Fetch the pgm pool shadow descriptor. */
1458	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1459	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1460
1461	PX86PDPAE pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1462	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1463	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1464
1465	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1466	/* PML4 */
1467	/const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;/
1468	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1469	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1470	PX86PDPAE pPDDst;
1471	PX86PDPT pPdptDst;
1472	PX86PML4E pPml4eDst;
1473	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1474	if (rc != VINF_SUCCESS)
1475	{
1476	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1477	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1478	PGM_INVL_PG(pVCpu, GCPtrPage);
1479	return VINF_SUCCESS;
1480	}
1481	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1482	Assert(pPDDst);
1483	Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P);
1484
1485	/* Fetch the pgm pool shadow descriptor. */
1486	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1487	Assert(pShwPde);
1488
1489	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1490
1491	const SHWPDE PdeDst = *pPdeDst;
1492	if (!(PdeDst.u & X86_PDE_P))
1493	{
1494	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1495	PGM_INVL_PG(pVCpu, GCPtrPage);
1496	return VINF_SUCCESS;
1497	}
1498
1499	/*
1500	* Get the guest PD entry and calc big page.
1501	*/
1502	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1503	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1504	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1505	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1506	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1507	unsigned iPDSrc = 0;
1508	# if PGM_GST_TYPE == PGM_TYPE_PAE
1509	X86PDPE PdpeSrcIgn;
1510	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1511	# else /* AMD64 */
1512	PX86PML4E pPml4eSrcIgn;
1513	X86PDPE PdpeSrcIgn;
1514	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1515	# endif
1516	GSTPDE PdeSrc;
1517
1518	if (pPDSrc)
1519	PdeSrc = pPDSrc->a[iPDSrc];
1520	else
1521	PdeSrc.u = 0;
1522	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1523	const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1524	const bool fIsBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1525	if (fWasBigPage != fIsBigPage)
1526	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1527
1528	# ifdef IN_RING3
1529	/*
1530	* If a CR3 Sync is pending we may ignore the invalidate page operation
1531	* depending on the kind of sync and if it's a global page or not.
1532	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1533	*/
1534	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1535	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1536	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1537	&& fIsBigPage
1538	&& (PdeSrc.u & X86_PDE4M_G)
1539	)
1540	)
1541	# else
1542	if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1543	# endif
1544	{
1545	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1546	return VINF_SUCCESS;
1547	}
1548	# endif /* IN_RING3 */
1549
1550	/*
1551	* Deal with the Guest PDE.
1552	*/
1553	rc = VINF_SUCCESS;
1554	if (PdeSrc.u & X86_PDE_P)
1555	{
1556	Assert( (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1557	&& ((PdeSrc.u & X86_PDE_RW) \|\| !(PdeDst.u & X86_PDE_RW) \|\| pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1558	if (!fIsBigPage)
1559	{
1560	/*
1561	* 4KB - page.
1562	*/
1563	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1564	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1565
1566	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1567	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1568	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1569	# endif
1570	if (pShwPage->GCPhys == GCPhys)
1571	{
1572	/* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1573	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1574
1575	PGSTPT pPTSrc;
1576	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1577	if (RT_SUCCESS(rc))
1578	{
1579	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1580	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1581	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1582	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1583	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1584	GCPtrPage, PteSrc.u & X86_PTE_P,
1585	(PteSrc.u & PdeSrc.u & X86_PTE_RW),
1586	(PteSrc.u & PdeSrc.u & X86_PTE_US),
1587	(uint64_t)PteSrc.u,
1588	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1589	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1590	}
1591	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1592	PGM_INVL_PG(pVCpu, GCPtrPage);
1593	}
1594	else
1595	{
1596	/*
1597	* The page table address changed.
1598	*/
1599	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1600	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1601	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1602	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1603	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1604	PGM_INVL_VCPU_TLBS(pVCpu);
1605	}
1606	}
1607	else
1608	{
1609	/*
1610	* 2/4MB - page.
1611	*/
1612	/* Before freeing the page, check if anything really changed. */
1613	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1614	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1615	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1616	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1617	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1618	# endif
1619	if ( pShwPage->GCPhys == GCPhys
1620	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1621	{
1622	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1623	/** @todo This test is wrong as it cannot check the G bit!
1624	* FIXME */
1625	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1626	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1627	&& ( (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1628	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1629	{
1630	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1631	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1632	return VINF_SUCCESS;
1633	}
1634	}
1635
1636	/*
1637	* Ok, the page table is present and it's been changed in the guest.
1638	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1639	* We could do this for some flushes in GC too, but we need an algorithm for
1640	* deciding which 4MB pages containing code likely to be executed very soon.
1641	*/
1642	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1643	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1644	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1645	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1646	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1647	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1648	}
1649	}
1650	else
1651	{
1652	/*
1653	* Page directory is not present, mark shadow PDE not present.
1654	*/
1655	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1656	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1657	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1658	PGM_INVL_PG(pVCpu, GCPtrPage);
1659	}
1660	return rc;
1661
1662	#else /* guest real and protected mode, nested + ept, none. */
1663	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1664	NOREF(pVCpu); NOREF(GCPtrPage);
1665	return VINF_SUCCESS;
1666	#endif
1667	}
1668
1669	#if PGM_SHW_TYPE != PGM_TYPE_NONE && !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
1670
1671	/**
1672	* Update the tracking of shadowed pages.
1673	*
1674	* @param pVCpu The cross context virtual CPU structure.
1675	* @param pShwPage The shadow page.
1676	* @param HCPhys The physical page we is being dereferenced.
1677	* @param iPte Shadow PTE index
1678	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1679	*/
1680	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1681	RTGCPHYS GCPhysPage)
1682	{
1683	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1684
1685	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1686	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1687	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1688
1689	/* Use the hint we retrieved from the cached guest PT. */
1690	if (pShwPage->fDirty)
1691	{
1692	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1693
1694	Assert(pShwPage->cPresent);
1695	Assert(pPool->cPresent);
1696	pShwPage->cPresent--;
1697	pPool->cPresent--;
1698
1699	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1700	AssertRelease(pPhysPage);
1701	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1702	return;
1703	}
1704	# else
1705	NOREF(GCPhysPage);
1706	# endif
1707
1708	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1709	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1710	* 2. write protect all shadowed pages. I.e. implement caching.
1711	*
1712	* 2023-08-24 bird: If we allow the ZeroPg to enter the shadow page tables,
1713	* this becomes a common occurence and we screw up. A better to the above would
1714	* be to have a parallel table that records the guest physical addresses of the
1715	* pages mapped by the shadow page table... For nested page tables,
1716	* we can easily correleate a table entry to a page entry, so it won't be
1717	* needed for those.
1718	*/
1719	# if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1720	/*
1721	* For non-paged guest tables, EPT and nested tables we can figure out the
1722	* physical page corresponding to the entry and dereference it.
1723	* (This ASSUMES that shadow PTs won't be used ever be used out of place.)
1724	*/
1725	if ( pShwPage->enmKind == PGMPOOLKIND_EPT_PT_FOR_PHYS
1726	\|\| pShwPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PHYS
1727	\|\| pShwPage->enmKind == PGMPOOLKIND_32BIT_PT_FOR_PHYS)
1728	{
1729	RTGCPHYS GCPhysNestedEntry = pShwPage->GCPhys + ((uint32_t)iPte << X86_PAGE_SHIFT);
1730	if (!pShwPage->fA20Enabled)
1731	GCPhysNestedEntry &= ~(uint64_t)RT_BIT_64(20);
1732	PPGMPAGE const pPhysPage = pgmPhysGetPage(pVM, GCPhysNestedEntry);
1733	AssertRelease(pPhysPage);
1734	pgmTrackDerefGCPhys(pVM->pgm.s.CTX_SUFF(pPool), pShwPage, pPhysPage, iPte);
1735	}
1736	else
1737	AssertMsgFailed(("enmKind=%d GCPhys=%RGp\n", pShwPage->enmKind, pShwPage->GCPhys));
1738	# endif
1739
1740	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1741
1742	/*
1743	* Find the guest address.
1744	*/
1745	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a);
1746	LogFlow(("SyncPageWorkerTrackDeref(%d,%d): Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n",
1747	PGM_SHW_TYPE, PGM_GST_TYPE, HCPhys, pShwPage->idx));
1748	uint32_t const idRamRangeMax = RT_MIN(pVM->pgm.s.idRamRangeMax, RT_ELEMENTS(pVM->pgm.s.apRamRanges) - 1U);
1749	Assert(pVM->pgm.s.apRamRanges[0] == NULL);
1750	for (uint32_t idx = 1; idx <= idRamRangeMax; idx++)
1751	{
1752	PPGMRAMRANGE const pRam = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idx];
1753	AssertContinue(pRam);
1754	unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT;
1755	while (iPage-- > 0)
1756	{
1757	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1758	{
1759	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1760
1761	Assert(pShwPage->cPresent);
1762	Assert(pPool->cPresent);
1763	pShwPage->cPresent--;
1764	pPool->cPresent--;
1765
1766	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1767	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a);
1768	return;
1769	}
1770	}
1771	}
1772
1773	for (;;)
1774	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1775	}
1776
1777
1778	/**
1779	* Update the tracking of shadowed pages.
1780	*
1781	* @param pVCpu The cross context virtual CPU structure.
1782	* @param pShwPage The shadow page.
1783	* @param u16 The top 16-bit of the pPage->HCPhys.
1784	* @param pPage Pointer to the guest page. this will be modified.
1785	* @param iPTDst The index into the shadow table.
1786	*/
1787	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16,
1788	PPGMPAGE pPage, const unsigned iPTDst)
1789	{
1790	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1791
1792	/*
1793	* Just deal with the simple first time here.
1794	*/
1795	if (!u16)
1796	{
1797	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin);
1798	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1799	/* Save the page table index. */
1800	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1801	}
1802	else
1803	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1804
1805	/* write back */
1806	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x pPage=%p\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst, pPage));
1807	PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1808
1809	/* update statistics. */
1810	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1811	pShwPage->cPresent++;
1812	if (pShwPage->iFirstPresent > iPTDst)
1813	pShwPage->iFirstPresent = iPTDst;
1814	}
1815
1816
1817	/**
1818	* Modifies a shadow PTE to account for access handlers.
1819	*
1820	* @param pVM The cross context VM structure.
1821	* @param pVCpu The cross context virtual CPU structure.
1822	* @param pPage The page in question.
1823	* @param GCPhysPage The guest-physical address of the page.
1824	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1825	* A (accessed) bit so it can be emulated correctly.
1826	* @param pPteDst The shadow PTE (output). This is temporary storage and
1827	* does not need to be set atomically.
1828	*/
1829	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PVMCPUCC pVCpu, PCPGMPAGE pPage, RTGCPHYS GCPhysPage, uint64_t fPteSrc,
1830	PSHWPTE pPteDst)
1831	{
1832	RT_NOREF_PV(pVM); RT_NOREF_PV(fPteSrc); RT_NOREF_PV(pVCpu); RT_NOREF_PV(GCPhysPage);
1833
1834	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1835	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1836	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1837	{
1838	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1839	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1840	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) \| EPT_E_READ \| EPT_E_EXECUTE \| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
1841	# else
1842	if (fPteSrc & X86_PTE_A)
1843	{
1844	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1845	SHW_PTE_SET_RO(*pPteDst);
1846	}
1847	else
1848	SHW_PTE_SET(*pPteDst, 0);
1849	# endif
1850	}
1851	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1852	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1853	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1854	&& ( BTH_IS_NP_ACTIVE(pVM)
1855	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1856	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1857	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1858	# endif
1859	)
1860	{
1861	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1862	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1863	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1864	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg
1865	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1866	\| EPT_E_WRITE
1867	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1868	\| EPT_E_MEMTYPE_INVALID_3;
1869	# else
1870	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1871	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1872	# endif
1873	}
1874	# endif
1875	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1876	else
1877	{
1878	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1879	SHW_PTE_SET(*pPteDst, 0);
1880	}
1881	/** @todo count these kinds of entries. */
1882	}
1883
1884
1885	/**
1886	* Creates a 4K shadow page for a guest page.
1887	*
1888	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1889	* physical address. The PdeSrc argument only the flags are used. No page
1890	* structured will be mapped in this function.
1891	*
1892	* @param pVCpu The cross context virtual CPU structure.
1893	* @param pPteDst Destination page table entry.
1894	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1895	* Can safely assume that only the flags are being used.
1896	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1897	* @param pShwPage Pointer to the shadow page.
1898	* @param iPTDst The index into the shadow table.
1899	*
1900	* @remark Not used for 2/4MB pages!
1901	*/
1902	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
1903	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1904	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1905	# else
1906	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1907	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1908	# endif
1909	{
1910	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1911	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1912
1913	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1914	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1915	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1916
1917	if (pShwPage->fDirty)
1918	{
1919	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1920	PGSTPT pGstPT;
1921
1922	/* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1923	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1924	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1925	pGstPT->a[iPTDst].u = PteSrc.u;
1926	}
1927	# else
1928	Assert(!pShwPage->fDirty);
1929	# endif
1930
1931	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1932	if ( (PteSrc.u & X86_PTE_P)
1933	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1934	# endif
1935	{
1936	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1937	RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1938	# endif
1939	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1940
1941	/*
1942	* Find the ram range.
1943	*/
1944	PPGMPAGE pPage;
1945	int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1946	if (RT_SUCCESS(rc))
1947	{
1948	/* Ignore ballooned pages.
1949	Don't return errors or use a fatal assert here as part of a
1950	shadow sync range might included ballooned pages. */
1951	if (PGM_PAGE_IS_BALLOONED(pPage))
1952	{
1953	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1954	return;
1955	}
1956
1957	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1958	/* Make the page writable if necessary. */
1959	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1960	&& ( PGM_PAGE_IS_ZERO(pPage)
1961	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1962	\|\| ( (PteSrc.u & X86_PTE_RW)
1963	# else
1964	\|\| ( 1
1965	# endif
1966	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1967	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1968	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1969	# endif
1970	# ifdef VBOX_WITH_PAGE_SHARING
1971	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1972	# endif
1973	)
1974	)
1975	)
1976	{
1977	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1978	AssertRC(rc);
1979	}
1980	# endif
1981
1982	/*
1983	* Make page table entry.
1984	*/
1985	SHWPTE PteDst;
1986	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1987	uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1988	# else
1989	uint64_t fGstShwPteFlags = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
1990	# endif
1991	if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \|\| PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1992	{
1993	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1994	/*
1995	* If the page or page directory entry is not marked accessed,
1996	* we mark the page not present.
1997	*/
1998	if (!(PteSrc.u & X86_PTE_A) \|\| !(PdeSrc.u & X86_PDE_A))
1999	{
2000	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
2001	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage));
2002	SHW_PTE_SET(PteDst, 0);
2003	}
2004	/*
2005	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
2006	* when the page is modified.
2007	*/
2008	else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW))
2009	{
2010	AssertCompile(X86_PTE_RW == X86_PDE_RW);
2011	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage));
2012	SHW_PTE_SET(PteDst,
2013	fGstShwPteFlags
2014	\| PGM_PAGE_GET_HCPHYS(pPage)
2015	\| PGM_PTFLAGS_TRACK_DIRTY);
2016	SHW_PTE_SET_RO(PteDst);
2017	}
2018	else
2019	# endif
2020	{
2021	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped));
2022	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2023	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage)
2024	\| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE \| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
2025	# else
2026	SHW_PTE_SET(PteDst, fGstShwPteFlags \| PGM_PAGE_GET_HCPHYS(pPage));
2027	# endif
2028	}
2029
2030	/*
2031	* Make sure only allocated pages are mapped writable.
2032	*/
2033	if ( SHW_PTE_IS_P_RW(PteDst)
2034	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2035	{
2036	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2037	/* Still applies to shared pages. */
2038	Assert(!PGM_PAGE_IS_ZERO(pPage));
2039	# endif
2040	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
2041	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
2042	}
2043	}
2044	else
2045	PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhysPage, fGstShwPteFlags, &PteDst);
2046
2047	/*
2048	* Keep user track up to date.
2049	*/
2050	if (SHW_PTE_IS_P(PteDst))
2051	{
2052	if (!SHW_PTE_IS_P(*pPteDst))
2053	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2054	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
2055	{
2056	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
2057	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2058	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2059	}
2060	}
2061	else if (SHW_PTE_IS_P(*pPteDst))
2062	{
2063	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
2064	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2065	}
2066
2067	/*
2068	* Update statistics and commit the entry.
2069	*/
2070	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2071	if (!(PteSrc.u & X86_PTE_G))
2072	pShwPage->fSeenNonGlobal = true;
2073	# endif
2074	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2075	return;
2076	}
2077
2078	/** @todo count these three different kinds. */
2079	Log2(("SyncPageWorker: invalid address in Pte\n"));
2080	}
2081	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2082	else if (!(PteSrc.u & X86_PTE_P))
2083	Log2(("SyncPageWorker: page not present in Pte\n"));
2084	else
2085	Log2(("SyncPageWorker: invalid Pte\n"));
2086	# endif
2087
2088	/*
2089	* The page is not present or the PTE is bad. Replace the shadow PTE by
2090	* an empty entry, making sure to keep the user tracking up to date.
2091	*/
2092	if (SHW_PTE_IS_P(*pPteDst))
2093	{
2094	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
2095	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2096	}
2097	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
2098	}
2099
2100
2101	/**
2102	* Syncs a guest OS page.
2103	*
2104	* There are no conflicts at this point, neither is there any need for
2105	* page table allocations.
2106	*
2107	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
2108	* When called in AMD64 guest mode, the guest PML4E shall be valid.
2109	*
2110	* @returns VBox status code.
2111	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
2112	* @param pVCpu The cross context virtual CPU structure.
2113	* @param PdeSrc Page directory entry of the guest.
2114	* @param GCPtrPage Guest context page address.
2115	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2116	* @param uErr Fault error (X86_TRAP_PF_*).
2117	*/
2118	static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
2119	{
2120	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2121	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2122	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
2123	RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
2124
2125	PGM_LOCK_ASSERT_OWNER(pVM);
2126
2127	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2128	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2129	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2130	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
2131
2132	/*
2133	* Assert preconditions.
2134	*/
2135	Assert(PdeSrc.u & X86_PDE_P);
2136	Assert(cPages);
2137	# if 0 /* rarely useful; leave for debugging. */
2138	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
2139	# endif
2140
2141	/*
2142	* Get the shadow PDE, find the shadow page table in the pool.
2143	*/
2144	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2145	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2146	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2147	AssertReturn(pPdeDst, VERR_INTERNAL_ERROR_3);
2148
2149	/* Fetch the pgm pool shadow descriptor. */
2150	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2151	Assert(pShwPde);
2152
2153	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2154	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2155	PPGMPOOLPAGE pShwPde = NULL;
2156	PX86PDPAE pPDDst;
2157
2158	/* Fetch the pgm pool shadow descriptor. */
2159	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2160	AssertRCSuccessReturn(rc2, rc2);
2161	Assert(pShwPde);
2162
2163	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2164	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2165
2166	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2167	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2168	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2169	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2170	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2171
2172	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2173	AssertRCSuccessReturn(rc2, rc2);
2174	Assert(pPDDst && pPdptDst);
2175	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2176	# endif
2177	SHWPDE PdeDst = *pPdeDst;
2178
2179	/*
2180	* - In the guest SMP case we could have blocked while another VCPU reused
2181	* this page table.
2182	* - With W7-64 we may also take this path when the A bit is cleared on
2183	* higher level tables (PDPE/PML4E). The guest does not invalidate the
2184	* relevant TLB entries. If we're write monitoring any page mapped by
2185	* the modified entry, we may end up here with a "stale" TLB entry.
2186	*/
2187	if (!(PdeDst.u & X86_PDE_P))
2188	{
2189	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2190	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
2191	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2192	if (uErr & X86_TRAP_PF_P)
2193	PGM_INVL_PG(pVCpu, GCPtrPage);
2194	return VINF_SUCCESS; /* force the instruction to be executed again. */
2195	}
2196
2197	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2198	Assert(pShwPage);
2199
2200	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2201	/* Fetch the pgm pool shadow descriptor. */
2202	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2203	Assert(pShwPde);
2204	# endif
2205
2206	/*
2207	* Check that the page is present and that the shadow PDE isn't out of sync.
2208	*/
2209	const bool fBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
2210	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
2211	RTGCPHYS GCPhys;
2212	if (!fBigPage)
2213	{
2214	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2215	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2216	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2217	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
2218	# endif
2219	}
2220	else
2221	{
2222	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2223	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2224	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2225	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2226	# endif
2227	}
2228	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2229	if ( fPdeValid
2230	&& pShwPage->GCPhys == GCPhys
2231	&& (PdeSrc.u & X86_PDE_P)
2232	&& (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
2233	&& ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) \|\| !(PdeDst.u & X86_PDE_RW))
2234	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2235	&& ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) \|\| !GST_IS_NX_ACTIVE(pVCpu))
2236	# endif
2237	)
2238	{
2239	/*
2240	* Check that the PDE is marked accessed already.
2241	* Since we set the accessed bit before getting here on a #PF, this
2242	* check is only meant for dealing with non-#PF'ing paths.
2243	*/
2244	if (PdeSrc.u & X86_PDE_A)
2245	{
2246	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2247	if (!fBigPage)
2248	{
2249	/*
2250	* 4KB Page - Map the guest page table.
2251	*/
2252	PGSTPT pPTSrc;
2253	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2254	if (RT_SUCCESS(rc))
2255	{
2256	# ifdef PGM_SYNC_N_PAGES
2257	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2258	if ( cPages > 1
2259	&& !(uErr & X86_TRAP_PF_P)
2260	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2261	{
2262	/*
2263	* This code path is currently only taken when the caller is PGMTrap0eHandler
2264	* for non-present pages!
2265	*
2266	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2267	* deal with locality.
2268	*/
2269	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2270	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2271	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2272	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2273	# else
2274	const unsigned offPTSrc = 0;
2275	# endif
2276	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2277	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2278	iPTDst = 0;
2279	else
2280	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2281
2282	for (; iPTDst < iPTDstEnd; iPTDst++)
2283	{
2284	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2285
2286	if ( (pPteSrc->u & X86_PTE_P)
2287	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2288	{
2289	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT))
2290	\| ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT);
2291	NOREF(GCPtrCurPage);
2292	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2293	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2294	GCPtrCurPage, pPteSrc->u & X86_PTE_P,
2295	!!(pPteSrc->u & PdeSrc.u & X86_PTE_RW),
2296	!!(pPteSrc->u & PdeSrc.u & X86_PTE_US),
2297	(uint64_t)pPteSrc->u,
2298	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2299	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2300	}
2301	}
2302	}
2303	else
2304	# endif /* PGM_SYNC_N_PAGES */
2305	{
2306	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2307	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2308	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2309	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2310	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2311	GCPtrPage, PteSrc.u & X86_PTE_P,
2312	!!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
2313	!!(PteSrc.u & PdeSrc.u & X86_PTE_US),
2314	(uint64_t)PteSrc.u,
2315	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2316	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2317	}
2318	}
2319	else /* MMIO or invalid page: emulated in #PF handler. */
2320	{
2321	LogFlow(("PGM_GCPHYS_2_PTR_V2 %RGp failed with %Rrc\n", GCPhys, rc));
2322	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2323	}
2324	}
2325	else
2326	{
2327	/*
2328	* 4/2MB page - lazy syncing shadow 4K pages.
2329	* (There are many causes of getting here, it's no longer only CSAM.)
2330	*/
2331	/* Calculate the GC physical address of this 4KB shadow page. */
2332	GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2333	/* Find ram range. */
2334	PPGMPAGE pPage;
2335	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2336	if (RT_SUCCESS(rc))
2337	{
2338	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2339
2340	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2341	/* Try to make the page writable if necessary. */
2342	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2343	&& ( PGM_PAGE_IS_ZERO(pPage)
2344	\|\| ( (PdeSrc.u & X86_PDE_RW)
2345	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2346	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2347	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2348	# endif
2349	# ifdef VBOX_WITH_PAGE_SHARING
2350	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2351	# endif
2352	)
2353	)
2354	)
2355	{
2356	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2357	AssertRC(rc);
2358	}
2359	# endif
2360
2361	/*
2362	* Make shadow PTE entry.
2363	*/
2364	SHWPTE PteDst;
2365	if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \|\| PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2366	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
2367	else
2368	PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2369
2370	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2371	if ( SHW_PTE_IS_P(PteDst)
2372	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2373	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2374
2375	/* Make sure only allocated pages are mapped writable. */
2376	if ( SHW_PTE_IS_P_RW(PteDst)
2377	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2378	{
2379	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2380	/* Still applies to shared pages. */
2381	Assert(!PGM_PAGE_IS_ZERO(pPage));
2382	# endif
2383	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2384	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2385	}
2386
2387	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2388
2389	/*
2390	* If the page is not flagged as dirty and is writable, then make it read-only
2391	* at PD level, so we can set the dirty bit when the page is modified.
2392	*
2393	* ASSUMES that page access handlers are implemented on page table entry level.
2394	* Thus we will first catch the dirty access and set PDE.D and restart. If
2395	* there is an access handler, we'll trap again and let it work on the problem.
2396	*/
2397	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2398	* As for invlpg, it simply frees the whole shadow PT.
2399	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2400	if ((PdeSrc.u & (X86_PDE4M_D \| X86_PDE_RW)) == X86_PDE_RW)
2401	{
2402	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2403	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2404	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2405	}
2406	else
2407	{
2408	PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY \| X86_PDE_RW);
2409	PdeDst.u \|= PdeSrc.u & X86_PDE_RW;
2410	}
2411	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2412	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2413	GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US),
2414	(uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2415	}
2416	else
2417	{
2418	LogFlow(("pgmPhysGetPageEx %RGp (big) failed with %Rrc\n", GCPhys, rc));
2419	/** @todo must wipe the shadow page table entry in this
2420	* case. */
2421	}
2422	}
2423	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2424	return VINF_SUCCESS;
2425	}
2426
2427	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs));
2428	}
2429	else if (fPdeValid)
2430	{
2431	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2432	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2433	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2434	}
2435	else
2436	{
2437	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2438	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2439	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2440	}
2441
2442	/*
2443	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2444	* Yea, I'm lazy.
2445	*/
2446	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2447	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
2448
2449	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2450	PGM_INVL_VCPU_TLBS(pVCpu);
2451	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2452
2453
2454	# elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2455	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2456	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
2457	NOREF(PdeSrc);
2458
2459	# ifdef PGM_SYNC_N_PAGES
2460	/*
2461	* Get the shadow PDE, find the shadow page table in the pool.
2462	*/
2463	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2464	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2465
2466	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2467	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2468
2469	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2470	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2471	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2472	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2473	X86PDEPAE PdeDst;
2474	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2475
2476	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2477	AssertRCSuccessReturn(rc, rc);
2478	Assert(pPDDst && pPdptDst);
2479	PdeDst = pPDDst->a[iPDDst];
2480
2481	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2482	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2483	PEPTPD pPDDst;
2484	EPTPDE PdeDst;
2485
2486	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2487	if (rc != VINF_SUCCESS)
2488	{
2489	AssertRC(rc);
2490	return rc;
2491	}
2492	Assert(pPDDst);
2493	PdeDst = pPDDst->a[iPDDst];
2494	# endif
2495	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2496	if (!SHW_PDE_IS_P(PdeDst))
2497	{
2498	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2499	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2500	return VINF_SUCCESS; /* force the instruction to be executed again. */
2501	}
2502
2503	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2504	if (SHW_PDE_IS_BIG(PdeDst))
2505	{
2506	Assert(pVM->pgm.s.fNestedPaging);
2507	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2508	return VINF_SUCCESS;
2509	}
2510
2511	/* Mask away the page offset. */
2512	GCPtrPage &= ~((RTGCPTR)0xfff);
2513
2514	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2515	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2516
2517	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2518	if ( cPages > 1
2519	&& !(uErr & X86_TRAP_PF_P)
2520	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2521	{
2522	/*
2523	* This code path is currently only taken when the caller is PGMTrap0eHandler
2524	* for non-present pages!
2525	*
2526	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2527	* deal with locality.
2528	*/
2529	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2530	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2531	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2532	iPTDst = 0;
2533	else
2534	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2535	for (; iPTDst < iPTDstEnd; iPTDst++)
2536	{
2537	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2538	{
2539	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2540	\| (iPTDst << GUEST_PAGE_SHIFT));
2541
2542	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2543	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2544	GCPtrCurPage,
2545	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2546	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2547
2548	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2549	break;
2550	}
2551	else
2552	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n",
2553	(GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2554	}
2555	}
2556	else
2557	# endif /* PGM_SYNC_N_PAGES */
2558	{
2559	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2560	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2561	\| (iPTDst << GUEST_PAGE_SHIFT));
2562
2563	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2564
2565	Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2566	GCPtrPage,
2567	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2568	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2569	}
2570	return VINF_SUCCESS;
2571
2572	# else
2573	NOREF(PdeSrc);
2574	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2575	return VERR_PGM_NOT_USED_IN_MODE;
2576	# endif
2577	}
2578
2579	#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE && !VBOX_WITH_ONLY_PGM_NEM_MODE */
2580
2581	#if !defined(IN_RING3) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT && !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
2582
2583	/**
2584	* Sync a shadow page for a nested-guest page.
2585	*
2586	* @param pVCpu The cross context virtual CPU structure.
2587	* @param pPte The shadow page table entry.
2588	* @param GCPhysPage The guest-physical address of the page.
2589	* @param pShwPage The shadow page of the page table.
2590	* @param iPte The index of the page table entry.
2591	* @param pGstSlatPte The guest SLAT page table entry.
2592	*
2593	* @note Not to be used for 2/4MB pages!
2594	*/
2595	static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
2596	unsigned iPte, SLATPTE GstSlatPte)
2597	{
2598	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2599	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
2600	Assert(!pShwPage->fDirty);
2601	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
2602	AssertMsg(!(GstSlatPte.u & EPT_E_LEAF), ("Large page unexpected: %RX64\n", GstSlatPte.u));
2603	AssertMsg((GstSlatPte.u & EPT_PTE_PG_MASK) == GCPhysPage,
2604	("PTE address mismatch. GCPhysPage=%RGp Pte=%RX64\n", GCPhysPage, GstSlatPte.u & EPT_PTE_PG_MASK));
2605
2606	/*
2607	* Find the ram range.
2608	*/
2609	PPGMPAGE pPage;
2610	int rc = pgmPhysGetPageEx(pVCpu->CTX_SUFF(pVM), GCPhysPage, &pPage);
2611	if (RT_SUCCESS(rc))
2612	{ /* likely */ }
2613	else
2614	{
2615	/*
2616	* This is a RAM hole/invalid/reserved address (not MMIO).
2617	* Nested Microsoft Hyper-V maps addresses like 0xf0220000 as RW WB memory.
2618	* Shadow a not-present page similar to MMIO, see @bugref{10318#c7}.
2619	*/
2620	Assert(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS);
2621	if (SHW_PTE_IS_P(*pPte))
2622	{
2623	Log2(("NestedSyncPageWorker: deref! pPte=%RX64\n", SHW_PTE_LOG64(pPte)));
2624	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2625	}
2626	Log7Func(("RAM hole/reserved %RGp -> ShwPte=0\n", GCPhysPage));
2627	SHW_PTE_ATOMIC_SET(*pPte, 0);
2628	return;
2629	}
2630
2631	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2632
2633	/*
2634	* Make page table entry.
2635	*/
2636	SHWPTE Pte;
2637	uint64_t const fGstShwPteFlags = (GstSlatPte.u & pVCpu->pgm.s.fGstEptShadowedPteMask)
2638	\| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
2639	if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \|\| PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2640	{
2641	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2642	/* If it's the zero page or write to an unallocated page, allocate it to make it writable. */
2643	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2644	&& ( PGM_PAGE_IS_ZERO(pPage)
2645	\|\| ( (GstSlatPte.u & EPT_E_WRITE)
2646	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2647	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2648	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2649	# endif
2650	# ifdef VBOX_WITH_PAGE_SHARING
2651	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2652	# endif
2653	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_BALLOONED
2654	)
2655	)
2656	)
2657	{
2658	rc = pgmPhysPageMakeWritable(pVCpu->CTX_SUFF(pVM), pPage, GCPhysPage);
2659	AssertRC(rc);
2660	Log7Func(("made writable (%R[pgmpage]) at %RGp\n", pPage, GCPhysPage));
2661	}
2662	# endif
2663	/** @todo access bit. */
2664	Pte.u = PGM_PAGE_GET_HCPHYS(pPage) \| fGstShwPteFlags;
2665	Log7Func(("regular page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2666
2667	/* Make sure only allocated pages are mapped writable. */
2668	if ( (fGstShwPteFlags & EPT_E_WRITE)
2669	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2670	{
2671	Pte.u &= ~EPT_E_WRITE;
2672	Log7Func(("write-protecting page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2673	}
2674	}
2675	else if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2676	{
2677	/** @todo access bit. */
2678	Pte.u = PGM_PAGE_GET_HCPHYS(pPage) \| (fGstShwPteFlags & ~EPT_E_WRITE);
2679	Log7Func(("monitored page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2680	}
2681	else
2682	{
2683	/** @todo Do MMIO optimizations here too? */
2684	Log7Func(("mmio/all page (%R[pgmpage]) at %RGp -> 0\n", pPage, GCPhysPage));
2685	Pte.u = 0;
2686	}
2687
2688	/* Make sure only allocated pages are mapped writable. */
2689	Assert(!SHW_PTE_IS_P_RW(Pte) \|\| PGM_PAGE_IS_ALLOCATED(pPage));
2690
2691	/*
2692	* Keep user track up to date.
2693	*/
2694	if (SHW_PTE_IS_P(Pte))
2695	{
2696	if (!SHW_PTE_IS_P(*pPte))
2697	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2698	else if (SHW_PTE_GET_HCPHYS(*pPte) != SHW_PTE_GET_HCPHYS(Pte))
2699	{
2700	Log2(("NestedSyncPageWorker: deref! pPte=%RX64 Pte=%RX64\n", SHW_PTE_LOG64(pPte), SHW_PTE_LOG64(Pte)));
2701	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2702	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2703	}
2704	}
2705	else if (SHW_PTE_IS_P(*pPte))
2706	{
2707	Log2(("NestedSyncPageWorker: deref! pPte=%RX64\n", SHW_PTE_LOG64(pPte)));
2708	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2709	}
2710
2711	/*
2712	* Commit the entry.
2713	*/
2714	SHW_PTE_ATOMIC_SET2(*pPte, Pte);
2715	return;
2716	}
2717
2718
2719	/**
2720	* Syncs a nested-guest page.
2721	*
2722	* There are no conflicts at this point, neither is there any need for
2723	* page table allocations.
2724	*
2725	* @returns VBox status code.
2726	* @param pVCpu The cross context virtual CPU structure.
2727	* @param GCPhysNestedPage The nested-guest physical address of the page being
2728	* synced.
2729	* @param GCPhysPage The guest-physical address of the page being synced.
2730	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2731	* @param uErr The page fault error (X86_TRAP_PF_XXX).
2732	* @param pGstWalkAll The guest page table walk result.
2733	*/
2734	static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
2735	uint32_t uErr, PPGMPTWALKGST pGstWalkAll)
2736	{
2737	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2738	Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2739	Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2740
2741	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2742	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2743	Log7Func(("GCPhysNestedPage=%RGv GCPhysPage=%RGp cPages=%u uErr=%#x\n", GCPhysNestedPage, GCPhysPage, cPages, uErr));
2744	RT_NOREF_PV(uErr); RT_NOREF_PV(cPages);
2745
2746	PGM_LOCK_ASSERT_OWNER(pVM);
2747
2748	/*
2749	* Get the shadow PDE, find the shadow page table in the pool.
2750	*/
2751	unsigned const iPde = ((GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK);
2752	PEPTPD pPd;
2753	int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL, &pPd, pGstWalkAll);
2754	if (RT_SUCCESS(rc))
2755	{ /* likely */ }
2756	else
2757	{
2758	Log(("Failed to fetch EPT PD for %RGp (%RGp) rc=%Rrc\n", GCPhysNestedPage, GCPhysPage, rc));
2759	return rc;
2760	}
2761	Assert(pPd);
2762	EPTPDE Pde = pPd->a[iPde];
2763
2764	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2765	if (!SHW_PDE_IS_P(Pde))
2766	{
2767	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)Pde.u));
2768	Log7Func(("CPU%d: SyncPage: Pde at %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2769	return VINF_SUCCESS; /* force the instruction to be executed again. */
2770	}
2771
2772	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2773	if (SHW_PDE_IS_BIG(Pde))
2774	{
2775	Log7Func(("CPU%d: SyncPage: %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2776	return VINF_SUCCESS;
2777	}
2778
2779	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, Pde.u & EPT_PDE_PG_MASK);
2780	PEPTPT pPt = (PEPTPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2781
2782	/*
2783	* If we've shadowed a guest EPT PDE that maps a 2M page using a 4K table,
2784	* then sync the 4K sub-page in the 2M range.
2785	*/
2786	if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2787	{
2788	Assert(!SHW_PDE_IS_BIG(Pde));
2789
2790	Assert(pGstWalkAll->u.Ept.Pte.u == 0);
2791	Assert((Pde.u & EPT_PRESENT_MASK) == (pGstWalkAll->u.Ept.Pde.u & EPT_PRESENT_MASK));
2792	Assert(pShwPage->GCPhys == (pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK));
2793
2794	#if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
2795	PPGMPAGE pPage;
2796	rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); AssertRC(rc);
2797	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
2798	Assert(pShwPage->enmKind == PGMPOOLKIND_EPT_PT_FOR_EPT_2MB);
2799	#endif
2800	uint64_t const fGstShwPteFlags = (pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask & ~EPT_E_LEAF)
2801	\| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
2802	SLATPTE GstSlatPte;
2803	GstSlatPte.u = GCPhysPage \| fGstShwPteFlags;
2804
2805	unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2806	PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, GstSlatPte);
2807	Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2808	return VINF_SUCCESS;
2809	}
2810
2811	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2812	# ifdef PGM_SYNC_N_PAGES
2813	if ( cPages > 1
2814	&& !(uErr & X86_TRAP_PF_P)
2815	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2816	{
2817	/*
2818	* This code path is currently only taken for non-present pages!
2819	*
2820	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2821	* deal with locality.
2822	*/
2823	unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2824	unsigned const iPteEnd = RT_MIN(iPte + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPt->a));
2825	if (iPte < PGM_SYNC_NR_PAGES / 2)
2826	iPte = 0;
2827	else
2828	iPte -= PGM_SYNC_NR_PAGES / 2;
2829	for (; iPte < iPteEnd; iPte++)
2830	{
2831	if (!SHW_PTE_IS_P(pPt->a[iPte]))
2832	{
2833	PGMPTWALKGST GstWalkPt;
2834	PGMPTWALK WalkPt;
2835	GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2836	GCPhysNestedPage \|= (iPte << GUEST_PAGE_SHIFT);
2837	rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /fIsLinearAddrValid/, 0 /GCPtrNested/, &WalkPt,
2838	&GstWalkPt);
2839	if (RT_SUCCESS(rc))
2840	PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], WalkPt.GCPhys, pShwPage, iPte, GstWalkPt.u.Ept.Pte);
2841	else
2842	{
2843	/*
2844	* This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
2845	* Ensure the shadow tables entry is not-present.
2846	*/
2847	/** @todo Potential room for optimization (explained in NestedSyncPT). */
2848	AssertMsg(!pPt->a[iPte].u, ("%RX64\n", pPt->a[iPte].u));
2849	}
2850	Log7Func(("Many: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2851	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2852	break;
2853	}
2854	else
2855	{
2856	# ifdef VBOX_STRICT
2857	/* Paranoia - Verify address of the page is what it should be. */
2858	PGMPTWALKGST GstWalkPt;
2859	PGMPTWALK WalkPt;
2860	GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2861	GCPhysNestedPage \|= (iPte << GUEST_PAGE_SHIFT);
2862	rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /fIsLinearAddrValid/, 0 /GCPtrNested/, &WalkPt, &GstWalkPt);
2863	AssertRC(rc);
2864	PPGMPAGE pPage;
2865	rc = pgmPhysGetPageEx(pVM, WalkPt.GCPhys, &pPage);
2866	AssertRC(rc);
2867	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]),
2868	("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp HCPhys=%RHp Shw=%RHp\n",
2869	GCPhysNestedPage, WalkPt.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
2870	# endif
2871	Log7Func(("Many3: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2872	}
2873	}
2874	}
2875	else
2876	# endif /* PGM_SYNC_N_PAGES */
2877	{
2878	unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2879	PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll->u.Ept.Pte);
2880	Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2881	}
2882
2883	return VINF_SUCCESS;
2884	}
2885
2886
2887	/**
2888	* Sync a shadow page table for a nested-guest page table.
2889	*
2890	* The shadow page table is not present in the shadow PDE.
2891	*
2892	* Handles mapping conflicts.
2893	*
2894	* A precondition for this method is that the shadow PDE is not present. The
2895	* caller must take the PGM lock before checking this and continue to hold it
2896	* when calling this method.
2897	*
2898	* @returns VBox status code.
2899	* @param pVCpu The cross context virtual CPU structure.
2900	* @param GCPhysNestedPage The nested-guest physical page address of the page
2901	* being synced.
2902	* @param GCPhysPage The guest-physical address of the page being synced.
2903	* @param pGstWalkAll The guest page table walk result.
2904	*/
2905	static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll)
2906	{
2907	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2908	Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2909	Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2910
2911	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2912	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2913
2914	Log7Func(("GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
2915
2916	PGM_LOCK_ASSERT_OWNER(pVM);
2917	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2918
2919	PEPTPD pPd;
2920	PEPTPDPT pPdpt;
2921	unsigned const iPde = (GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK;
2922	int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, &pPdpt, &pPd, pGstWalkAll);
2923	if (RT_SUCCESS(rc))
2924	{ /* likely */ }
2925	else
2926	{
2927	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2928	AssertRC(rc);
2929	return rc;
2930	}
2931	Assert(pPd);
2932	PSHWPDE pPde = &pPd->a[iPde];
2933
2934	unsigned const iPdpt = (GCPhysNestedPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2935	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdpt->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2936	Assert(pShwPde->enmKind == PGMPOOLKIND_EPT_PD_FOR_EPT_PD);
2937
2938	SHWPDE Pde = *pPde;
2939	Assert(!SHW_PDE_IS_P(Pde)); /* We're only supposed to call SyncPT on PDE!P and conflicts. */
2940
2941	# ifdef PGM_WITH_LARGE_PAGES
2942	Assert(BTH_IS_NP_ACTIVE(pVM));
2943
2944	/*
2945	* Check if the guest is mapping a 2M page.
2946	*/
2947	if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2948	{
2949	PPGMPAGE pPage;
2950	rc = pgmPhysGetPageEx(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK, &pPage);
2951	AssertRCReturn(rc, rc);
2952
2953	/* A20 is always enabled in VMX root and non-root operation. */
2954	Assert(PGM_A20_IS_ENABLED(pVCpu));
2955
2956	/*
2957	* Check if we have or can get a 2M backing page here.
2958	*/
2959	RTHCPHYS HCPhys = NIL_RTHCPHYS;
2960	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
2961	{
2962	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
2963	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2964	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2965	}
2966	else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2967	{
2968	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
2969	rc = pgmPhysRecheckLargePage(pVM, GCPhysPage, pPage);
2970	if (RT_SUCCESS(rc))
2971	{
2972	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2973	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2974	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2975	}
2976	}
2977	else if (PGMIsUsingLargePages(pVM))
2978	{
2979	rc = pgmPhysAllocLargePage(pVM, GCPhysPage);
2980	if (RT_SUCCESS(rc))
2981	{
2982	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2983	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2984	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2985	}
2986	}
2987
2988	/*
2989	* If we have a 2M backing page, we can map the guest's 2M page right away.
2990	*/
2991	uint64_t const fGstShwBigPdeFlags = (pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask)
2992	\| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
2993	if (HCPhys != NIL_RTHCPHYS)
2994	{
2995	Pde.u = HCPhys \| fGstShwBigPdeFlags;
2996	Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzBigPdeMask));
2997	Assert(Pde.u & EPT_E_LEAF);
2998	SHW_PDE_ATOMIC_SET2(*pPde, Pde);
2999
3000	/* Add a reference to the first page only. */
3001	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPde);
3002
3003	Assert(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED);
3004
3005	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3006	Log7Func(("GstPde=%RGp ShwPde=%RX64 [2M]\n", pGstWalkAll->u.Ept.Pde.u, Pde.u));
3007	return VINF_SUCCESS;
3008	}
3009
3010	/*
3011	* We didn't get a perfect 2M fit. Split the 2M page into 4K pages.
3012	* The page ought not to be marked as a big (2M) page at this point.
3013	*/
3014	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
3015
3016	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
3017	PGMPOOLACCESS enmAccess;
3018	{
3019	/*
3020	* Mode-based execute control for EPT not supported.
3021	*
3022	* However, Windows 10 with Hyper-V enabled sets the EPT_E_USER_EXECUTE bit but does
3023	* not enable "mode-based execute control for EPT" in the VT-x secondary VM-execution
3024	* controls. The CPU ignores this bit when the control isn't set. Hence, the assertion
3025	* below is commented out.
3026	*/
3027	/* Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_USER_EXECUTE)); */
3028	Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
3029	bool const fNoExecute = !(pGstWalkAll->u.Ept.Pde.u & EPT_E_EXECUTE);
3030	if (pGstWalkAll->u.Ept.Pde.u & EPT_E_WRITE)
3031	enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
3032	else
3033	enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
3034	}
3035
3036	/*
3037	* Allocate & map a 4K shadow table to cover the 2M guest page.
3038	*/
3039	PPGMPOOLPAGE pShwPage;
3040	RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK;
3041	rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_2MB, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
3042	pShwPde->idx, iPde, false /fLockPage/, &pShwPage);
3043	if ( rc == VINF_SUCCESS
3044	\|\| rc == VINF_PGM_CACHED_PAGE)
3045	{ /* likely */ }
3046	else
3047	{
3048	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3049	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3050	}
3051
3052	PSHWPT pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3053	Assert(pPt);
3054	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
3055	if (rc == VINF_SUCCESS)
3056	{
3057	/* The 4K PTEs shall inherit the flags of the 2M PDE page sans the leaf bit. */
3058	uint64_t const fGstShwPteFlags = fGstShwBigPdeFlags & ~EPT_E_LEAF;
3059
3060	/* Sync each 4K pages in the 2M range. */
3061	for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++)
3062	{
3063	RTGCPHYS const GCPhysSubPage = GCPhysPt \| (iPte << GUEST_PAGE_SHIFT);
3064	SLATPTE GstSlatPte;
3065	GstSlatPte.u = GCPhysSubPage \| fGstShwPteFlags;
3066	Assert(!(GstSlatPte.u & pVCpu->pgm.s.fGstEptMbzPteMask));
3067	PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysSubPage, pShwPage, iPte, GstSlatPte);
3068	Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [2M->4K]\n", pGstWalkAll->u.Ept.Pte, pPt->a[iPte].u, iPte));
3069	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3070	break;
3071	}
3072	}
3073	else
3074	{
3075	Assert(rc == VINF_PGM_CACHED_PAGE);
3076	# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3077	/* Paranoia - Verify address of each of the subpages are what they should be. */
3078	RTGCPHYS GCPhysSubPage = GCPhysPt;
3079	for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++, GCPhysSubPage += GUEST_PAGE_SIZE)
3080	{
3081	PPGMPAGE pSubPage;
3082	rc = pgmPhysGetPageEx(pVM, GCPhysSubPage, &pSubPage);
3083	AssertRC(rc);
3084	AssertMsg( PGM_PAGE_GET_HCPHYS(pSubPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte])
3085	\|\| !SHW_PTE_IS_P(pPt->a[iPte]),
3086	("PGM 2M page and shadow PTE conflict. GCPhysSubPage=%RGp Page=%RHp Shw=%RHp\n",
3087	GCPhysSubPage, PGM_PAGE_GET_HCPHYS(pSubPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3088	}
3089	# endif
3090	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3091	}
3092
3093	/* Save the new PDE. */
3094	uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3095	Pde.u = pShwPage->Core.Key \| fShwPdeFlags;
3096	Assert(!(Pde.u & EPT_E_LEAF));
3097	Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3098	SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3099	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3100	Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3101	return rc;
3102	}
3103	# endif /* PGM_WITH_LARGE_PAGES */
3104
3105	/*
3106	* Allocate & map the shadow page table.
3107	*/
3108	PSHWPT pPt;
3109	PPGMPOOLPAGE pShwPage;
3110
3111	RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE_PG_MASK;
3112	rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_PT, PGMPOOLACCESS_DONTCARE,
3113	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPde, false /fLockPage/, &pShwPage);
3114	if ( rc == VINF_SUCCESS
3115	\|\| rc == VINF_PGM_CACHED_PAGE)
3116	{ /* likely */ }
3117	else
3118	{
3119	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3120	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3121	}
3122
3123	pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3124	Assert(pPt);
3125	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
3126
3127	if (rc == VINF_SUCCESS)
3128	{
3129	/* Sync the page we've already translated through SLAT. */
3130	const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3131	PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll->u.Ept.Pte);
3132	Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3133
3134	/* Sync the rest of page table (expensive but might be cheaper than nested-guest VM-exits in hardware). */
3135	for (unsigned iPteCur = 0; iPteCur < RT_ELEMENTS(pPt->a); iPteCur++)
3136	{
3137	if (iPteCur != iPte)
3138	{
3139	PGMPTWALKGST GstWalkPt;
3140	PGMPTWALK WalkPt;
3141	GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
3142	GCPhysNestedPage \|= (iPteCur << GUEST_PAGE_SHIFT);
3143	int const rc2 = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /fIsLinearAddrValid/, 0 /GCPtrNested/,
3144	&WalkPt, &GstWalkPt);
3145	if (RT_SUCCESS(rc2))
3146	{
3147	PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPteCur], WalkPt.GCPhys, pShwPage, iPteCur,
3148	GstWalkPt.u.Ept.Pte);
3149	Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", GstWalkPt.u.Ept.Pte.u, pPt->a[iPteCur].u, iPteCur));
3150	}
3151	else
3152	{
3153	/*
3154	* This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
3155	* Ensure the shadow tables entry is not-present.
3156	*/
3157	/** @todo We currently don't configure these to cause EPT misconfigs but rather trap
3158	* them using EPT violations and walk the guest EPT tables to determine
3159	* whether they are EPT misconfigs VM-exits for the nested-hypervisor. We
3160	* could optimize this by using a specific combination of reserved bits
3161	* which we could immediately identify as EPT misconfigs of the
3162	* nested-hypervisor without having to walk its EPT tables. However, tracking
3163	* non-present entries might be tricky...
3164	*/
3165	AssertMsg(!pPt->a[iPteCur].u, ("%RX64\n", pPt->a[iPteCur].u));
3166	}
3167	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3168	break;
3169	}
3170	}
3171	}
3172	else
3173	{
3174	Assert(rc == VINF_PGM_CACHED_PAGE);
3175	# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3176	/* Paranoia - Verify address of the page is what it should be. */
3177	PPGMPAGE pPage;
3178	rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
3179	AssertRC(rc);
3180	const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3181	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]) \|\| !SHW_PTE_IS_P(pPt->a[iPte]),
3182	("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp Page=%RHp Shw=%RHp\n",
3183	GCPhysNestedPage, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3184	Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [cache]\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3185	# endif
3186	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3187	}
3188
3189	/* Save the new PDE. */
3190	uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3191	Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF));
3192	Assert(!(pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3193	Pde.u = pShwPage->Core.Key \| fShwPdeFlags;
3194	SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3195	Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3196
3197	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3198	return rc;
3199	}
3200
3201	#endif /* !IN_RING3 && VBOX_WITH_NESTED_HWVIRT_VMX_EPT && PGM_SHW_TYPE == PGM_TYPE_EPT && !VBOX_WITH_ONLY_PGM_NEM_MODE */
3202	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE && !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
3203
3204	/**
3205	* Handle dirty bit tracking faults.
3206	*
3207	* @returns VBox status code.
3208	* @param pVCpu The cross context virtual CPU structure.
3209	* @param uErr Page fault error code.
3210	* @param pPdeSrc Guest page directory entry.
3211	* @param pPdeDst Shadow page directory entry.
3212	* @param GCPtrPage Guest context page address.
3213	*/
3214	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
3215	RTGCPTR GCPtrPage)
3216	{
3217	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3218	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3219	NOREF(uErr);
3220
3221	PGM_LOCK_ASSERT_OWNER(pVM);
3222
3223	/*
3224	* Handle big page.
3225	*/
3226	if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
3227	{
3228	if ((pPdeDst->u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3229	{
3230	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3231	Assert(pPdeSrc->u & X86_PDE_RW);
3232
3233	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
3234	* fault again and take this path to only invalidate the entry (see below). */
3235	SHWPDE PdeDst = *pPdeDst;
3236	PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY;
3237	PdeDst.u \|= X86_PDE_RW \| X86_PDE_A;
3238	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3239	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
3240	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3241	}
3242
3243	# ifdef IN_RING0
3244	/* Check for stale TLB entry; only applies to the SMP guest case. */
3245	if ( pVM->cCpus > 1
3246	&& (pPdeDst->u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_A)) == (X86_PDE_P \| X86_PDE_RW \| X86_PDE_A))
3247	{
3248	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3249	if (pShwPage)
3250	{
3251	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3252	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3253	if (SHW_PTE_IS_P_RW(*pPteDst))
3254	{
3255	/* Stale TLB entry. */
3256	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3257	PGM_INVL_PG(pVCpu, GCPtrPage);
3258	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3259	}
3260	}
3261	}
3262	# endif /* IN_RING0 */
3263	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3264	}
3265
3266	/*
3267	* Map the guest page table.
3268	*/
3269	PGSTPT pPTSrc;
3270	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
3271	AssertRCReturn(rc, rc);
3272
3273	if (SHW_PDE_IS_P(*pPdeDst))
3274	{
3275	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
3276	const GSTPTE PteSrc = *pPteSrc;
3277
3278	/*
3279	* Map shadow page table.
3280	*/
3281	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3282	if (pShwPage)
3283	{
3284	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3285	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3286	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
3287	{
3288	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
3289	{
3290	PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
3291	SHWPTE PteDst = *pPteDst;
3292
3293	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
3294	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3295
3296	Assert(PteSrc.u & X86_PTE_RW);
3297
3298	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
3299	* entry will not harm; write access will simply fault again and
3300	* take this path to only invalidate the entry.
3301	*/
3302	if (RT_LIKELY(pPage))
3303	{
3304	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3305	{
3306	//AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
3307	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
3308	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
3309	SHW_PTE_SET_RO(PteDst);
3310	}
3311	else
3312	{
3313	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
3314	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
3315	{
3316	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
3317	AssertRC(rc);
3318	}
3319	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
3320	SHW_PTE_SET_RW(PteDst);
3321	else
3322	{
3323	/* Still applies to shared pages. */
3324	Assert(!PGM_PAGE_IS_ZERO(pPage));
3325	SHW_PTE_SET_RO(PteDst);
3326	}
3327	}
3328	}
3329	else
3330	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
3331
3332	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
3333	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
3334	PGM_INVL_PG(pVCpu, GCPtrPage);
3335	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3336	}
3337
3338	# ifdef IN_RING0
3339	/* Check for stale TLB entry; only applies to the SMP guest case. */
3340	if ( pVM->cCpus > 1
3341	&& SHW_PTE_IS_RW(*pPteDst)
3342	&& SHW_PTE_IS_A(*pPteDst))
3343	{
3344	/* Stale TLB entry. */
3345	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3346	PGM_INVL_PG(pVCpu, GCPtrPage);
3347	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3348	}
3349	# endif
3350	}
3351	}
3352	else
3353	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
3354	}
3355
3356	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3357	}
3358
3359	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE && !VBOX_WITH_ONLY_PGM_NEM_MODE */
3360
3361	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3362	/**
3363	* Sync a shadow page table.
3364	*
3365	* The shadow page table is not present in the shadow PDE.
3366	*
3367	* Handles mapping conflicts.
3368	*
3369	* This is called by PrefetchPage, InvalidatePage (on conflict), and
3370	* Trap0eHandler.
3371	*
3372	* A precondition for this method is that the shadow PDE is not present. The
3373	* caller must take the PGM lock before checking this and continue to hold it
3374	* when calling this method.
3375	*
3376	* @returns VBox status code.
3377	* @param pVCpu The cross context virtual CPU structure.
3378	* @param iPDSrc Page directory index.
3379	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
3380	* Assume this is a temporary mapping.
3381	* @param GCPtrPage GC Pointer of the page that caused the fault
3382	*/
3383	static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
3384	{
3385	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3386	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3387
3388	# if 0 /* rarely useful; leave for debugging. */
3389	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
3390	# endif
3391	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
3392
3393	PGM_LOCK_ASSERT_OWNER(pVM);
3394
3395	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3396	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3397	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
3398	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3399	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3400	int rc = VINF_SUCCESS;
3401
3402	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3403
3404	/*
3405	* Some input validation first.
3406	*/
3407	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
3408
3409	/*
3410	* Get the relevant shadow PDE entry.
3411	*/
3412	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3413	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
3414	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3415	AssertReturn(pPdeDst, VERR_INTERNAL_ERROR_3);
3416
3417	/* Fetch the pgm pool shadow descriptor. */
3418	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3419	Assert(pShwPde);
3420
3421	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3422	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3423	PPGMPOOLPAGE pShwPde = NULL;
3424	PX86PDPAE pPDDst;
3425	PSHWPDE pPdeDst;
3426
3427	/* Fetch the pgm pool shadow descriptor. */
3428	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3429	AssertRCSuccessReturn(rc, rc);
3430	Assert(pShwPde);
3431
3432	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3433	pPdeDst = &pPDDst->a[iPDDst];
3434
3435	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3436	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3437	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3438	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3439	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
3440	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3441	AssertRCSuccessReturn(rc, rc);
3442	Assert(pPDDst);
3443	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3444
3445	# endif
3446	SHWPDE PdeDst = *pPdeDst;
3447
3448	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3449	/* Fetch the pgm pool shadow descriptor. */
3450	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3451	Assert(pShwPde);
3452	# endif
3453
3454	Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/
3455
3456	/*
3457	* Sync the page directory entry.
3458	*/
3459	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3460	const bool fPageTable = !(PdeSrc.u & X86_PDE_PS) \|\| !GST_IS_PSE_ACTIVE(pVCpu);
3461	if ( (PdeSrc.u & X86_PDE_P)
3462	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
3463	{
3464	/*
3465	* Allocate & map the page table.
3466	*/
3467	PSHWPT pPTDst;
3468	PPGMPOOLPAGE pShwPage;
3469	RTGCPHYS GCPhys;
3470	if (fPageTable)
3471	{
3472	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
3473	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3474	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3475	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
3476	# endif
3477	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
3478	pShwPde->idx, iPDDst, false /fLockPage/,
3479	&pShwPage);
3480	}
3481	else
3482	{
3483	PGMPOOLACCESS enmAccess;
3484	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
3485	const bool fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu);
3486	# else
3487	const bool fNoExecute = false;
3488	# endif
3489
3490	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3491	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3492	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
3493	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
3494	# endif
3495	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
3496	if (PdeSrc.u & X86_PDE_US)
3497	{
3498	if (PdeSrc.u & X86_PDE_RW)
3499	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
3500	else
3501	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
3502	}
3503	else
3504	{
3505	if (PdeSrc.u & X86_PDE_RW)
3506	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
3507	else
3508	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
3509	}
3510	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
3511	pShwPde->idx, iPDDst, false /fLockPage/,
3512	&pShwPage);
3513	}
3514	if (rc == VINF_SUCCESS)
3515	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3516	else if (rc == VINF_PGM_CACHED_PAGE)
3517	{
3518	/*
3519	* The PT was cached, just hook it up.
3520	*/
3521	if (fPageTable)
3522	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3523	else
3524	{
3525	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3526	/* (see explanation and assumptions further down.) */
3527	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
3528	{
3529	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3530	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
3531	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3532	}
3533	}
3534	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3535	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3536	return VINF_SUCCESS;
3537	}
3538	else
3539	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3540	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
3541	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
3542	* irrelevant at this point. */
3543	PdeDst.u &= X86_PDE_AVL_MASK;
3544	PdeDst.u \|= pShwPage->Core.Key;
3545
3546	/*
3547	* Page directory has been accessed (this is a fault situation, remember).
3548	*/
3549	/** @todo
3550	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
3551	* fault situation. What's more, the Trap0eHandler has already set the
3552	* accessed bit. So, it's actually just VerifyAccessSyncPage which
3553	* might need setting the accessed flag.
3554	*
3555	* The best idea is to leave this change to the caller and add an
3556	* assertion that it's set already. */
3557	pPDSrc->a[iPDSrc].u \|= X86_PDE_A;
3558	if (fPageTable)
3559	{
3560	/*
3561	* Page table - 4KB.
3562	*
3563	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
3564	*/
3565	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
3566	GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u));
3567	PGSTPT pPTSrc;
3568	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
3569	if (RT_SUCCESS(rc))
3570	{
3571	/*
3572	* Start by syncing the page directory entry so CSAM's TLB trick works.
3573	*/
3574	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
3575	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3576	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3577	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3578
3579	/*
3580	* Directory/page user or supervisor privilege: (same goes for read/write)
3581	*
3582	* Directory Page Combined
3583	* U/S U/S U/S
3584	* 0 0 0
3585	* 0 1 0
3586	* 1 0 0
3587	* 1 1 1
3588	*
3589	* Simple AND operation. Table listed for completeness.
3590	*
3591	*/
3592	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K));
3593	# ifdef PGM_SYNC_N_PAGES
3594	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3595	unsigned iPTDst = iPTBase;
3596	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
3597	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
3598	iPTDst = 0;
3599	else
3600	iPTDst -= PGM_SYNC_NR_PAGES / 2;
3601	# else /* !PGM_SYNC_N_PAGES */
3602	unsigned iPTDst = 0;
3603	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
3604	# endif /* !PGM_SYNC_N_PAGES */
3605	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
3606	\| ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT);
3607	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3608	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3609	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
3610	# else
3611	const unsigned offPTSrc = 0;
3612	# endif
3613	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE)
3614	{
3615	const unsigned iPTSrc = iPTDst + offPTSrc;
3616	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
3617	if (PteSrc.u & X86_PTE_P)
3618	{
3619	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
3620	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
3621	GCPtrCur,
3622	PteSrc.u & X86_PTE_P,
3623	!!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
3624	!!(PteSrc.u & PdeSrc.u & X86_PTE_US),
3625	(uint64_t)PteSrc.u,
3626	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
3627	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
3628	}
3629	/* else: the page table was cleared by the pool */
3630	} /* for PTEs */
3631	}
3632	}
3633	else
3634	{
3635	/*
3636	* Big page - 2/4MB.
3637	*
3638	* We'll walk the ram range list in parallel and optimize lookups.
3639	* We will only sync one shadow page table at a time.
3640	*/
3641	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M));
3642
3643	/**
3644	* @todo It might be more efficient to sync only a part of the 4MB
3645	* page (similar to what we do for 4KB PDs).
3646	*/
3647
3648	/*
3649	* Start by syncing the page directory entry.
3650	*/
3651	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
3652	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3653
3654	/*
3655	* If the page is not flagged as dirty and is writable, then make it read-only
3656	* at PD level, so we can set the dirty bit when the page is modified.
3657	*
3658	* ASSUMES that page access handlers are implemented on page table entry level.
3659	* Thus we will first catch the dirty access and set PDE.D and restart. If
3660	* there is an access handler, we'll trap again and let it work on the problem.
3661	*/
3662	/** @todo move the above stuff to a section in the PGM documentation. */
3663	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
3664	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
3665	{
3666	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3667	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
3668	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3669	}
3670	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3671	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3672
3673	/*
3674	* Fill the shadow page table.
3675	*/
3676	/* Get address and flags from the source PDE. */
3677	SHWPTE PteDstBase;
3678	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
3679
3680	/* Loop thru the entries in the shadow PT. */
3681	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
3682	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
3683	GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr,
3684	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
3685	unsigned iPTDst = 0;
3686	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3687	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3688	{
3689	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3690	if (pRam && GCPhys >= pRam->GCPhys)
3691	{
3692	# ifndef PGM_WITH_A20
3693	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
3694	# endif
3695	do
3696	{
3697	/* Make shadow PTE. */
3698	# ifdef PGM_WITH_A20
3699	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
3700	# else
3701	PPGMPAGE pPage = &pRam->aPages[iHCPage];
3702	# endif
3703	SHWPTE PteDst;
3704
3705	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3706	/* Try to make the page writable if necessary. */
3707	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3708	&& ( PGM_PAGE_IS_ZERO(pPage)
3709	\|\| ( SHW_PTE_IS_RW(PteDstBase)
3710	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3711	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3712	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3713	# endif
3714	# ifdef VBOX_WITH_PAGE_SHARING
3715	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3716	# endif
3717	&& !PGM_PAGE_IS_BALLOONED(pPage))
3718	)
3719	)
3720	{
3721	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3722	AssertRCReturn(rc, rc);
3723	if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3724	break;
3725	}
3726	# endif
3727
3728	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3729	PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, SHW_PTE_GET_U(PteDstBase), &PteDst);
3730	else if (PGM_PAGE_IS_BALLOONED(pPage))
3731	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3732	else
3733	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
3734
3735	/* Only map writable pages writable. */
3736	if ( SHW_PTE_IS_P_RW(PteDst)
3737	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3738	{
3739	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3740	/* Still applies to shared pages. */
3741	Assert(!PGM_PAGE_IS_ZERO(pPage));
3742	# endif
3743	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3744	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
3745	}
3746
3747	if (SHW_PTE_IS_P(PteDst))
3748	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3749
3750	/* commit it (not atomic, new table) */
3751	pPTDst->a[iPTDst] = PteDst;
3752	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3753	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3754	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3755
3756	/* advance */
3757	GCPhys += GUEST_PAGE_SIZE;
3758	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3759	# ifndef PGM_WITH_A20
3760	iHCPage++;
3761	# endif
3762	iPTDst++;
3763	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3764	&& GCPhys <= pRam->GCPhysLast);
3765	}
3766	else if (pRam)
3767	{
3768	Log(("Invalid pages at %RGp\n", GCPhys));
3769	do
3770	{
3771	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3772	GCPhys += GUEST_PAGE_SIZE;
3773	iPTDst++;
3774	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3775	&& GCPhys < pRam->GCPhys);
3776	PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3777	}
3778	else
3779	{
3780	Log(("Invalid pages at %RGp (2)\n", GCPhys));
3781	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3782	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3783	}
3784	} /* while more PTEs */
3785	} /* 4KB / 4MB */
3786	}
3787	else
3788	AssertRelease(!SHW_PDE_IS_P(PdeDst));
3789
3790	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3791	if (RT_FAILURE(rc))
3792	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3793	return rc;
3794
3795	# elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3796	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
3797	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3798	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3799	NOREF(iPDSrc); NOREF(pPDSrc);
3800
3801	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3802
3803	/*
3804	* Validate input a little bit.
3805	*/
3806	int rc = VINF_SUCCESS;
3807	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3808	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3809	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3810	AssertReturn(pPdeDst, VERR_INTERNAL_ERROR_3);
3811
3812	/* Fetch the pgm pool shadow descriptor. */
3813	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3814	Assert(pShwPde);
3815
3816	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3817	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3818	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3819	PX86PDPAE pPDDst;
3820	PSHWPDE pPdeDst;
3821
3822	/* Fetch the pgm pool shadow descriptor. */
3823	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3824	AssertRCSuccessReturn(rc, rc);
3825	Assert(pShwPde);
3826
3827	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3828	pPdeDst = &pPDDst->a[iPDDst];
3829
3830	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3831	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3832	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3833	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3834	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3835	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3836	AssertRCSuccessReturn(rc, rc);
3837	Assert(pPDDst);
3838	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3839
3840	/* Fetch the pgm pool shadow descriptor. */
3841	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3842	Assert(pShwPde);
3843
3844	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3845	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3846	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3847	PEPTPD pPDDst;
3848	PEPTPDPT pPdptDst;
3849
3850	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3851	if (rc != VINF_SUCCESS)
3852	{
3853	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3854	AssertRC(rc);
3855	return rc;
3856	}
3857	Assert(pPDDst);
3858	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3859
3860	/* Fetch the pgm pool shadow descriptor. */
3861	/** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
3862	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3863	Assert(pShwPde);
3864	# endif
3865	SHWPDE PdeDst = *pPdeDst;
3866
3867	Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3868
3869	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3870	if (BTH_IS_NP_ACTIVE(pVM))
3871	{
3872	Assert(!VM_IS_NEM_ENABLED(pVM));
3873
3874	/* Check if we allocated a big page before for this 2 MB range. */
3875	PPGMPAGE pPage;
3876	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3877	if (RT_SUCCESS(rc))
3878	{
3879	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3880	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3881	{
3882	if (PGM_A20_IS_ENABLED(pVCpu))
3883	{
3884	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3885	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3886	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3887	}
3888	else
3889	{
3890	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3891	pVM->pgm.s.cLargePagesDisabled++;
3892	}
3893	}
3894	else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3895	&& PGM_A20_IS_ENABLED(pVCpu))
3896	{
3897	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3898	rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3899	if (RT_SUCCESS(rc))
3900	{
3901	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3902	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3903	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3904	}
3905	}
3906	# if !defined(VBOX_WITH_NEW_LAZY_PAGE_ALLOC) && !defined(PGM_WITH_PAGE_ZEROING_DETECTION) /* This code is too aggresive! */
3907	else if ( PGMIsUsingLargePages(pVM)
3908	&& PGM_A20_IS_ENABLED(pVCpu))
3909	{
3910	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3911	if (RT_SUCCESS(rc))
3912	{
3913	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3914	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3915	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3916	}
3917	else
3918	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3919	}
3920	# endif
3921
3922	if (HCPhys != NIL_RTHCPHYS)
3923	{
3924	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3925	PdeDst.u = HCPhys \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE \| EPT_E_LEAF \| EPT_E_IGNORE_PAT \| EPT_E_MEMTYPE_WB
3926	\| (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
3927	# else
3928	PdeDst.u = HCPhys \| X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PS
3929	\| (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */;
3930	# endif
3931	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3932
3933	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3934	/* Add a reference to the first page only. */
3935	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3936
3937	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3938	return VINF_SUCCESS;
3939	}
3940	}
3941	}
3942	# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
3943
3944	/*
3945	* Allocate & map the page table.
3946	*/
3947	PSHWPT pPTDst;
3948	PPGMPOOLPAGE pShwPage;
3949	RTGCPHYS GCPhys;
3950
3951	/* Virtual address = physical address */
3952	GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3953	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3954	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /fLockPage/,
3955	&pShwPage);
3956	if ( rc == VINF_SUCCESS
3957	\|\| rc == VINF_PGM_CACHED_PAGE)
3958	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3959	else
3960	{
3961	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3962	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3963	}
3964
3965	if (rc == VINF_SUCCESS)
3966	{
3967	/* New page table; fully set it up. */
3968	Assert(pPTDst);
3969
3970	/* Mask away the page offset. */
3971	GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
3972
3973	for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3974	{
3975	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3976	\| (iPTDst << GUEST_PAGE_SHIFT));
3977
3978	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3979	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3980	GCPtrCurPage,
3981	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3982	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3983
3984	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3985	break;
3986	}
3987	}
3988	else
3989	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3990
3991	/* Save the new PDE. */
3992	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3993	PdeDst.u = pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE
3994	\| (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
3995	# else
3996	PdeDst.u = pShwPage->Core.Key \| X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A
3997	\| (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */);
3998	# endif
3999	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
4000
4001	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
4002	if (RT_FAILURE(rc))
4003	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
4004	return rc;
4005
4006	# else
4007	NOREF(iPDSrc); NOREF(pPDSrc);
4008	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
4009	return VERR_PGM_NOT_USED_IN_MODE;
4010	# endif
4011	}
4012	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
4013
4014
4015
4016	/**
4017	* Prefetch a page/set of pages.
4018	*
4019	* Typically used to sync commonly used pages before entering raw mode
4020	* after a CR3 reload.
4021	*
4022	* @returns VBox status code.
4023	* @param pVCpu The cross context virtual CPU structure.
4024	* @param GCPtrPage Page to invalidate.
4025	*/
4026	PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
4027	{
4028	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
4029	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
4030	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
4031	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4032	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
4033	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
4034	&& PGM_SHW_TYPE != PGM_TYPE_NONE \
4035	&& !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
4036	/*
4037	* Check that all Guest levels thru the PDE are present, getting the
4038	* PD and PDE in the processes.
4039	*/
4040	int rc = VINF_SUCCESS;
4041	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4042	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4043	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
4044	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4045	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4046	unsigned iPDSrc;
4047	X86PDPE PdpeSrc;
4048	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
4049	if (!pPDSrc)
4050	return VINF_SUCCESS; /* not present */
4051	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4052	unsigned iPDSrc;
4053	PX86PML4E pPml4eSrc;
4054	X86PDPE PdpeSrc;
4055	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
4056	if (!pPDSrc)
4057	return VINF_SUCCESS; /* not present */
4058	# endif
4059	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
4060	# else
4061	PGSTPD pPDSrc = NULL;
4062	const unsigned iPDSrc = 0;
4063	GSTPDE const PdeSrc = { X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A }; /* faked so we don't have to #ifdef everything */
4064	# endif
4065
4066	if ((PdeSrc.u & (X86_PDE_P \| X86_PDE_A)) == (X86_PDE_P \| X86_PDE_A))
4067	{
4068	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4069	PGM_LOCK_VOID(pVM);
4070
4071	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4072	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
4073	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4074	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4075	PX86PDPAE pPDDst;
4076	X86PDEPAE PdeDst;
4077	# if PGM_GST_TYPE != PGM_TYPE_PAE
4078	X86PDPE PdpeSrc;
4079
4080	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
4081	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
4082	# endif
4083	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
4084	if (rc != VINF_SUCCESS)
4085	{
4086	PGM_UNLOCK(pVM);
4087	AssertRC(rc);
4088	return rc;
4089	}
4090	Assert(pPDDst);
4091	PdeDst = pPDDst->a[iPDDst];
4092
4093	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4094	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4095	PX86PDPAE pPDDst;
4096	X86PDEPAE PdeDst;
4097
4098	# if PGM_GST_TYPE == PGM_TYPE_PROT
4099	/* AMD-V nested paging */
4100	X86PML4E Pml4eSrc;
4101	X86PDPE PdpeSrc;
4102	PX86PML4E pPml4eSrc = &Pml4eSrc;
4103
4104	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
4105	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
4106	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
4107	# endif
4108
4109	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
4110	if (rc != VINF_SUCCESS)
4111	{
4112	PGM_UNLOCK(pVM);
4113	AssertRC(rc);
4114	return rc;
4115	}
4116	Assert(pPDDst);
4117	PdeDst = pPDDst->a[iPDDst];
4118	# endif
4119	if (!(PdeDst.u & X86_PDE_P))
4120	{
4121	/** @todo r=bird: This guy will set the A bit on the PDE,
4122	* probably harmless. */
4123	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
4124	}
4125	else
4126	{
4127	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
4128	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
4129	* makes no sense to prefetch more than one page.
4130	*/
4131	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
4132	if (RT_SUCCESS(rc))
4133	rc = VINF_SUCCESS;
4134	}
4135	PGM_UNLOCK(pVM);
4136	}
4137	return rc;
4138
4139	#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE \|\| defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
4140	NOREF(pVCpu); NOREF(GCPtrPage);
4141	return VINF_SUCCESS; /* ignore */
4142	#else
4143	AssertCompile(0);
4144	#endif
4145	}
4146
4147
4148	/**
4149	* Syncs the paging hierarchy starting at CR3.
4150	*
4151	* @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
4152	* informational status codes.
4153	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
4154	* the VMM into guest context.
4155	* @param pVCpu The cross context virtual CPU structure.
4156	* @param cr0 Guest context CR0 register.
4157	* @param cr3 Guest context CR3 register. Not subjected to the A20
4158	* mask.
4159	* @param cr4 Guest context CR4 register.
4160	* @param fGlobal Including global page directories or not
4161	*/
4162	PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
4163	{
4164	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4165	NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
4166
4167	LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
4168
4169	#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE && !defined(VBOX_WITH_ONLY_PGM_NEM_MODE)
4170	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4171	PGM_LOCK_VOID(pVM);
4172	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4173	if (pPool->cDirtyPages)
4174	pgmPoolResetDirtyPages(pVM);
4175	PGM_UNLOCK(pVM);
4176	# endif
4177	#endif /* !NESTED && !EPT && !VBOX_WITH_ONLY_PGM_NEM_MODE */
4178
4179	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
4180	/*
4181	* Nested / EPT / None - No work.
4182	*/
4183	return VINF_SUCCESS;
4184
4185	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4186	/*
4187	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
4188	* out the shadow parts when the guest modifies its tables.
4189	*/
4190	return VINF_SUCCESS;
4191
4192	#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4193
4194	return VINF_SUCCESS;
4195	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4196	}
4197
4198
4199
4200
4201	#ifdef VBOX_STRICT
4202
4203	/**
4204	* Checks that the shadow page table is in sync with the guest one.
4205	*
4206	* @returns The number of errors.
4207	* @param pVCpu The cross context virtual CPU structure.
4208	* @param cr3 Guest context CR3 register.
4209	* @param cr4 Guest context CR4 register.
4210	* @param GCPtr Where to start. Defaults to 0.
4211	* @param cb How much to check. Defaults to everything.
4212	*/
4213	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
4214	{
4215	NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
4216	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
4217	return 0;
4218	#else
4219	unsigned cErrors = 0;
4220	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4221	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
4222
4223	# if PGM_GST_TYPE == PGM_TYPE_PAE
4224	/** @todo currently broken; crashes below somewhere */
4225	AssertFailed();
4226	# endif
4227
4228	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
4229	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4230	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4231
4232	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
4233	bool const fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
4234	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
4235	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
4236	# ifndef IN_RING0
4237	RTHCPHYS HCPhys; /* general usage. */
4238	# endif
4239	# endif
4240	PPGMCPU const pPGM = &pVCpu->pgm.s;
4241	int rc;
4242	RT_NOREF(rc);
4243
4244	/*
4245	* Check that the Guest CR3 and all its mappings are correct.
4246	*/
4247	AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
4248	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
4249	false);
4250	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
4251	# if 0
4252	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4253	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
4254	# else
4255	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
4256	# endif
4257	AssertRCReturn(rc, 1);
4258	HCPhys = NIL_RTHCPHYS;
4259	rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
4260	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
4261	# endif
4262	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
4263	pgmGstGet32bitPDPtr(pVCpu);
4264	RTGCPHYS GCPhys;
4265	rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
4266	AssertRCReturn(rc, 1);
4267	AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
4268	# endif
4269	# endif /* !IN_RING0 */
4270
4271	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
4272	/*
4273	* Get and check the Shadow CR3.
4274	*/
4275	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4276	unsigned cPDEs = X86_PG_ENTRIES;
4277	unsigned cIncrement = X86_PG_ENTRIES * GUEST_PAGE_SIZE;
4278	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4279	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4280	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
4281	# else
4282	unsigned cPDEs = X86_PG_PAE_ENTRIES;
4283	# endif
4284	unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4285	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4286	unsigned cPDEs = X86_PG_PAE_ENTRIES;
4287	unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4288	# endif
4289	if (cb != ~(RTGCPTR)0)
4290	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
4291
4292	/** @todo call the other two PGMAssert() functions. /
4293
4294	# if PGM_GST_TYPE == PGM_TYPE_AMD64
4295	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4296
4297	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
4298	{
4299	PX86PML4E const pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
4300	AssertContinueStmt(pPml4eSrc, cErrors++);
4301
4302	PX86PML4E const pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
4303	AssertContinueStmt(pPml4eDst, cErrors++);
4304
4305	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
4306	if (!(pPml4eDst->u & X86_PML4E_P))
4307	{
4308	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4309	continue;
4310	}
4311
4312	PPGMPOOLPAGE pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
4313	RTGCPHYS GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
4314
4315	if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P))
4316	{
4317	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4318	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4319	cErrors++;
4320	continue;
4321	}
4322
4323	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
4324	{
4325	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
4326	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4327	cErrors++;
4328	continue;
4329	}
4330
4331	if ( (pPml4eDst->u & (X86_PML4E_US \| X86_PML4E_RW \| X86_PML4E_NX))
4332	!= (pPml4eSrc->u & (X86_PML4E_US \| X86_PML4E_RW \| X86_PML4E_NX)))
4333	{
4334	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4335	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4336	cErrors++;
4337	continue;
4338	}
4339	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4340	{
4341	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4342
4343	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
4344	/*
4345	* Check the PDPTEs too.
4346	*/
4347	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
4348
4349	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
4350	{
4351	unsigned iPDSrc = 0; /* initialized to shut up gcc */
4352	PPGMPOOLPAGE pShwPde = NULL;
4353	PX86PDPE pPdpeDst;
4354	RTGCPHYS GCPhysPdeSrc;
4355	X86PDPE PdpeSrc;
4356	PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
4357	# if PGM_GST_TYPE == PGM_TYPE_PAE
4358	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
4359	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
4360	# else
4361	PX86PML4E pPml4eSrcIgn;
4362	PX86PDPT pPdptDst;
4363	PX86PDPAE pPDDst;
4364	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
4365
4366	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
4367	if (rc != VINF_SUCCESS)
4368	{
4369	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
4370	GCPtr += 512 * _2M;
4371	continue; /* next PDPTE */
4372	}
4373	Assert(pPDDst);
4374	# endif
4375	Assert(iPDSrc == 0);
4376
4377	pPdpeDst = &pPdptDst->a[iPdpt];
4378
4379	if (!(pPdpeDst->u & X86_PDPE_P))
4380	{
4381	GCPtr += 512 * _2M;
4382	continue; /* next PDPTE */
4383	}
4384
4385	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
4386	GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
4387
4388	if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P))
4389	{
4390	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4391	GCPtr += 512 * _2M;
4392	cErrors++;
4393	continue;
4394	}
4395
4396	if (GCPhysPdeSrc != pShwPde->GCPhys)
4397	{
4398	# if PGM_GST_TYPE == PGM_TYPE_AMD64
4399	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4400	# else
4401	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4402	# endif
4403	GCPtr += 512 * _2M;
4404	cErrors++;
4405	continue;
4406	}
4407
4408	# if PGM_GST_TYPE == PGM_TYPE_AMD64
4409	if ( (pPdpeDst->u & (X86_PDPE_US \| X86_PDPE_RW \| X86_PDPE_LM_NX))
4410	!= (PdpeSrc.u & (X86_PDPE_US \| X86_PDPE_RW \| X86_PDPE_LM_NX)))
4411	{
4412	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4413	GCPtr += 512 * _2M;
4414	cErrors++;
4415	continue;
4416	}
4417	# endif
4418
4419	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4420	{
4421	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4422	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4423	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4424	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4425	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
4426	# endif
4427	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
4428	/*
4429	* Iterate the shadow page directory.
4430	*/
4431	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
4432	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
4433
4434	for (;
4435	iPDDst < cPDEs;
4436	iPDDst++, GCPtr += cIncrement)
4437	{
4438	# if PGM_SHW_TYPE == PGM_TYPE_PAE
4439	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4440	# else
4441	const SHWPDE PdeDst = pPDDst->a[iPDDst];
4442	# endif
4443	if ( (PdeDst.u & X86_PDE_P)
4444	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) )
4445	{
4446	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4447	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4448	if (!pPoolPage)
4449	{
4450	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4451	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4452	cErrors++;
4453	continue;
4454	}
4455	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4456
4457	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
4458	{
4459	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4460	GCPtr, (uint64_t)PdeDst.u));
4461	cErrors++;
4462	}
4463
4464	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
4465	{
4466	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4467	GCPtr, (uint64_t)PdeDst.u));
4468	cErrors++;
4469	}
4470
4471	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4472	if (!(PdeSrc.u & X86_PDE_P))
4473	{
4474	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4475	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4476	cErrors++;
4477	continue;
4478	}
4479
4480	if ( !(PdeSrc.u & X86_PDE_PS)
4481	\|\| !fBigPagesSupported)
4482	{
4483	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4484	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4485	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
4486	# endif
4487	}
4488	else
4489	{
4490	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4491	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4492	{
4493	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4494	GCPtr, (uint64_t)PdeSrc.u));
4495	cErrors++;
4496	continue;
4497	}
4498	# endif
4499	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4500	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4501	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4502	# endif
4503	}
4504
4505	if ( pPoolPage->enmKind
4506	!= (!(PdeSrc.u & X86_PDE_PS) \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4507	{
4508	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4509	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4510	cErrors++;
4511	}
4512
4513	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4514	if (!pPhysPage)
4515	{
4516	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4517	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4518	cErrors++;
4519	continue;
4520	}
4521
4522	if (GCPhysGst != pPoolPage->GCPhys)
4523	{
4524	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4525	GCPhysGst, pPoolPage->GCPhys, GCPtr));
4526	cErrors++;
4527	continue;
4528	}
4529
4530	if ( !(PdeSrc.u & X86_PDE_PS)
4531	\|\| !fBigPagesSupported)
4532	{
4533	/*
4534	* Page Table.
4535	*/
4536	const GSTPT *pPTSrc;
4537	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)),
4538	&pPTSrc);
4539	if (RT_FAILURE(rc))
4540	{
4541	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4542	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4543	cErrors++;
4544	continue;
4545	}
4546	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
4547	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
4548	{
4549	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4550	// (This problem will go away when/if we shadow multiple CR3s.)
4551	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4552	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4553	cErrors++;
4554	continue;
4555	}
4556	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4557	{
4558	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4559	GCPtr, (uint64_t)PdeDst.u));
4560	cErrors++;
4561	continue;
4562	}
4563
4564	/* iterate the page table. */
4565	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4566	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4567	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4568	# else
4569	const unsigned offPTSrc = 0;
4570	# endif
4571	for (unsigned iPT = 0, off = 0;
4572	iPT < RT_ELEMENTS(pPTDst->a);
4573	iPT++, off += GUEST_PAGE_SIZE)
4574	{
4575	const SHWPTE PteDst = pPTDst->a[iPT];
4576
4577	/* skip not-present and dirty tracked entries. */
4578	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4579	continue;
4580	Assert(SHW_PTE_IS_P(PteDst));
4581
4582	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4583	if (!(PteSrc.u & X86_PTE_P))
4584	{
4585	# ifdef IN_RING3
4586	PGMAssertHandlerAndFlagsInSync(pVM);
4587	DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
4588	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
4589	0, 0, UINT64_MAX, 99, NULL);
4590	# endif
4591	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4592	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4593	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4594	cErrors++;
4595	continue;
4596	}
4597
4598	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
4599	# if 1 /** @todo sync accessed bit properly... */
4600	fIgnoreFlags \|= X86_PTE_A;
4601	# endif
4602
4603	/* match the physical addresses */
4604	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4605	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4606
4607	# ifdef IN_RING3
4608	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4609	if (RT_FAILURE(rc))
4610	{
4611	# if 0
4612	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4613	{
4614	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4615	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4616	cErrors++;
4617	continue;
4618	}
4619	# endif
4620	}
4621	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4622	{
4623	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4624	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4625	cErrors++;
4626	continue;
4627	}
4628	# endif
4629
4630	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4631	if (!pPhysPage)
4632	{
4633	# if 0
4634	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4635	{
4636	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4637	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4638	cErrors++;
4639	continue;
4640	}
4641	# endif
4642	if (SHW_PTE_IS_RW(PteDst))
4643	{
4644	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4645	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4646	cErrors++;
4647	}
4648	fIgnoreFlags \|= X86_PTE_RW;
4649	}
4650	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4651	{
4652	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4653	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4654	cErrors++;
4655	continue;
4656	}
4657
4658	/* flags */
4659	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
4660	{
4661	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4662	{
4663	if (SHW_PTE_IS_RW(PteDst))
4664	{
4665	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4666	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4667	cErrors++;
4668	continue;
4669	}
4670	fIgnoreFlags \|= X86_PTE_RW;
4671	}
4672	else
4673	{
4674	if ( SHW_PTE_IS_P(PteDst)
4675	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4676	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4677	# endif
4678	)
4679	{
4680	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4681	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4682	cErrors++;
4683	continue;
4684	}
4685	fIgnoreFlags \|= X86_PTE_P;
4686	}
4687	}
4688	else
4689	{
4690	if ((PteSrc.u & (X86_PTE_RW \| X86_PTE_D)) == X86_PTE_RW)
4691	{
4692	if (SHW_PTE_IS_RW(PteDst))
4693	{
4694	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4695	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4696	cErrors++;
4697	continue;
4698	}
4699	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4700	{
4701	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4702	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4703	cErrors++;
4704	continue;
4705	}
4706	if (SHW_PTE_IS_D(PteDst))
4707	{
4708	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4709	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4710	cErrors++;
4711	}
4712	# if 0 /** @todo sync access bit properly... */
4713	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4714	{
4715	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4716	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4717	cErrors++;
4718	}
4719	fIgnoreFlags \|= X86_PTE_RW;
4720	# else
4721	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4722	# endif
4723	}
4724	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4725	{
4726	/* access bit emulation (not implemented). */
4727	if ((PteSrc.u & X86_PTE_A) \|\| SHW_PTE_IS_P(PteDst))
4728	{
4729	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4730	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4731	cErrors++;
4732	continue;
4733	}
4734	if (!SHW_PTE_IS_A(PteDst))
4735	{
4736	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4737	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4738	cErrors++;
4739	}
4740	fIgnoreFlags \|= X86_PTE_P;
4741	}
4742	# ifdef DEBUG_sandervl
4743	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4744	# endif
4745	}
4746
4747	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4748	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4749	)
4750	{
4751	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4752	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4753	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4754	cErrors++;
4755	continue;
4756	}
4757	} /* foreach PTE */
4758	}
4759	else
4760	{
4761	/*
4762	* Big Page.
4763	*/
4764	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4765	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
4766	{
4767	if (PdeDst.u & X86_PDE_RW)
4768	{
4769	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4770	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4771	cErrors++;
4772	continue;
4773	}
4774	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4775	{
4776	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4777	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4778	cErrors++;
4779	continue;
4780	}
4781	# if 0 /** @todo sync access bit properly... */
4782	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4783	{
4784	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4785	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4786	cErrors++;
4787	}
4788	fIgnoreFlags \|= X86_PTE_RW;
4789	# else
4790	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4791	# endif
4792	}
4793	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4794	{
4795	/* access bit emulation (not implemented). */
4796	if ((PdeSrc.u & X86_PDE_A) \|\| SHW_PDE_IS_P(PdeDst))
4797	{
4798	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4799	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4800	cErrors++;
4801	continue;
4802	}
4803	if (!SHW_PDE_IS_A(PdeDst))
4804	{
4805	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4806	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4807	cErrors++;
4808	}
4809	fIgnoreFlags \|= X86_PTE_P;
4810	}
4811
4812	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4813	{
4814	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4815	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4816	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4817	cErrors++;
4818	}
4819
4820	/* iterate the page table. */
4821	for (unsigned iPT = 0, off = 0;
4822	iPT < RT_ELEMENTS(pPTDst->a);
4823	iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE))
4824	{
4825	const SHWPTE PteDst = pPTDst->a[iPT];
4826
4827	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4828	{
4829	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4830	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4831	cErrors++;
4832	}
4833
4834	/* skip not-present entries. */
4835	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4836	continue;
4837
4838	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4839
4840	/* match the physical addresses */
4841	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4842
4843	# ifdef IN_RING3
4844	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4845	if (RT_FAILURE(rc))
4846	{
4847	# if 0
4848	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4849	{
4850	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4851	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4852	cErrors++;
4853	}
4854	# endif
4855	}
4856	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4857	{
4858	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4859	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4860	cErrors++;
4861	continue;
4862	}
4863	# endif
4864	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4865	if (!pPhysPage)
4866	{
4867	# if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4868	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4869	{
4870	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4871	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4872	cErrors++;
4873	continue;
4874	}
4875	# endif
4876	if (SHW_PTE_IS_RW(PteDst))
4877	{
4878	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4879	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4880	cErrors++;
4881	}
4882	fIgnoreFlags \|= X86_PTE_RW;
4883	}
4884	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4885	{
4886	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4887	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4888	cErrors++;
4889	continue;
4890	}
4891
4892	/* flags */
4893	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4894	{
4895	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4896	{
4897	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4898	{
4899	if ( SHW_PTE_IS_RW(PteDst)
4900	&& !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
4901	{
4902	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4903	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4904	cErrors++;
4905	continue;
4906	}
4907	fIgnoreFlags \|= X86_PTE_RW;
4908	}
4909	}
4910	else
4911	{
4912	if ( SHW_PTE_IS_P(PteDst)
4913	&& !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage)
4914	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4915	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4916	# endif
4917	)
4918	{
4919	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4920	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4921	cErrors++;
4922	continue;
4923	}
4924	fIgnoreFlags \|= X86_PTE_P;
4925	}
4926	}
4927
4928	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4929	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4930	)
4931	{
4932	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4933	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4934	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4935	cErrors++;
4936	continue;
4937	}
4938	} /* for each PTE */
4939	}
4940	}
4941	/* not present */
4942
4943	} /* for each PDE */
4944
4945	} /* for each PDPTE */
4946
4947	} /* for each PML4E */
4948
4949	# endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
4950	# ifdef DEBUG
4951	if (cErrors)
4952	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4953	# endif
4954	# endif /* GST is in {32BIT, PAE, AMD64} */
4955	return cErrors;
4956	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
4957	}
4958	#endif /* VBOX_STRICT */
4959
4960
4961	/**
4962	* Sets up the CR3 for shadow paging
4963	*
4964	* @returns Strict VBox status code.
4965	* @retval VINF_SUCCESS.
4966	*
4967	* @param pVCpu The cross context virtual CPU structure.
4968	* @param GCPhysCR3 The physical address in the CR3 register. (A20 mask
4969	* already applied.)
4970	*/
4971	PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
4972	{
4973	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4974	int rc = VINF_SUCCESS;
4975
4976	/* Update guest paging info. */
4977	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4978	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4979	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4980
4981	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4982	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4983
4984	# if PGM_GST_TYPE == PGM_TYPE_PAE
4985	if ( !pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped)
4986	\|\| pVCpu->pgm.s.GCPhysPaeCR3 != GCPhysCR3)
4987	# endif
4988	{
4989	/*
4990	* Map the page CR3 points at.
4991	*/
4992	RTHCPTR HCPtrGuestCR3;
4993	rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3);
4994	if (RT_SUCCESS(rc))
4995	{
4996	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4997	# ifdef IN_RING3
4998	pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
4999	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
5000	# else
5001	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
5002	pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
5003	# endif
5004
5005	# elif PGM_GST_TYPE == PGM_TYPE_PAE
5006	# ifdef IN_RING3
5007	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
5008	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
5009	# else
5010	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
5011	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
5012	# endif
5013
5014	X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
5015	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5016	/*
5017	* When EPT is enabled by the nested-hypervisor and the nested-guest is in PAE mode,
5018	* the guest-CPU context would've already been updated with the 4 PAE PDPEs specified
5019	* in the virtual VMCS. The PDPEs can differ from those in guest memory referenced by
5020	* the translated nested-guest CR3. We -MUST- use the PDPEs provided in the virtual VMCS
5021	* rather than those in guest memory.
5022	*
5023	* See Intel spec. 26.3.2.4 "Loading Page-Directory-Pointer-Table Entries".
5024	*/
5025	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
5026	CPUMGetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5027	else
5028	#endif
5029	{
5030	/* Update CPUM with the PAE PDPEs referenced by CR3. */
5031	memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
5032	CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5033	}
5034
5035	/*
5036	* Map the 4 PAE PDPEs.
5037	*/
5038	rc = PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
5039	if (RT_SUCCESS(rc))
5040	{
5041	# ifdef IN_RING3
5042	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
5043	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5044	# else
5045	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5046	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
5047	# endif
5048	pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
5049	}
5050
5051	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
5052	# ifdef IN_RING3
5053	pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
5054	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
5055	# else
5056	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
5057	pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
5058	# endif
5059	# endif
5060	}
5061	else
5062	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
5063	}
5064	#endif
5065
5066	/*
5067	* Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
5068	*/
5069	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5070	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
5071	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
5072	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
5073	&& PGM_GST_TYPE != PGM_TYPE_PROT) \
5074	&& !defined(VBOX_WITH_ONLY_PGM_NEM_MODE) )
5075
5076	Assert(!pVM->pgm.s.fNestedPaging);
5077	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
5078
5079	/*
5080	* Update the shadow root page as well since that's not fixed.
5081	*/
5082	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5083	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
5084	PPGMPOOLPAGE pNewShwPageCR3;
5085
5086	PGM_LOCK_VOID(pVM);
5087
5088	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5089	if (pPool->cDirtyPages)
5090	pgmPoolResetDirtyPages(pVM);
5091	# endif
5092
5093	Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */
5094	int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE,
5095	PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /fLockPage/, &pNewShwPageCR3);
5096	AssertFatalRC(rc2);
5097
5098	pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
5099	pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
5100
5101	/* Set the current hypervisor CR3. */
5102	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
5103
5104	/* Clean up the old CR3 root. */
5105	if ( pOldShwPageCR3
5106	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
5107	{
5108	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
5109
5110	/* Mark the page as unlocked; allow flushing again. */
5111	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
5112
5113	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
5114	}
5115	PGM_UNLOCK(pVM);
5116	# else
5117	NOREF(GCPhysCR3);
5118	# endif
5119
5120	return rc;
5121	}
5122
5123	/**
5124	* Unmaps the shadow CR3.
5125	*
5126	* @returns VBox status, no specials.
5127	* @param pVCpu The cross context virtual CPU structure.
5128	*/
5129	PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
5130	{
5131	LogFlow(("UnmapCR3\n"));
5132
5133	int rc = VINF_SUCCESS;
5134	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
5135
5136	/*
5137	* Update guest paging info.
5138	*/
5139	#if PGM_GST_TYPE == PGM_TYPE_32BIT
5140	pVCpu->pgm.s.pGst32BitPdR3 = 0;
5141	pVCpu->pgm.s.pGst32BitPdR0 = 0;
5142
5143	#elif PGM_GST_TYPE == PGM_TYPE_PAE
5144	pVCpu->pgm.s.pGstPaePdptR3 = 0;
5145	pVCpu->pgm.s.pGstPaePdptR0 = 0;
5146	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
5147	{
5148	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
5149	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
5150	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
5151	}
5152
5153	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
5154	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
5155	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
5156
5157	#else /* prot/real mode stub */
5158	/* nothing to do */
5159	#endif
5160
5161	/*
5162	* PAE PDPEs (and CR3) might have been mapped via PGMGstMapPaePdpesAtCr3()
5163	* prior to switching to PAE in pfnMapCr3(), so we need to clear them here.
5164	*/
5165	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5166	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5167	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
5168
5169	/*
5170	* Update shadow paging info.
5171	*/
5172	#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5173	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
5174	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
5175	&& !defined(VBOX_WITH_ONLY_PGM_NEM_MODE) )
5176	# if PGM_GST_TYPE != PGM_TYPE_REAL
5177	Assert(!pVM->pgm.s.fNestedPaging);
5178	# endif
5179	PGM_LOCK_VOID(pVM);
5180
5181	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
5182	{
5183	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5184
5185	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5186	if (pPool->cDirtyPages)
5187	pgmPoolResetDirtyPages(pVM);
5188	# endif
5189
5190	/* Mark the page as unlocked; allow flushing again. */
5191	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
5192
5193	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
5194	pVCpu->pgm.s.pShwPageCR3R3 = 0;
5195	pVCpu->pgm.s.pShwPageCR3R0 = 0;
5196	}
5197
5198	PGM_UNLOCK(pVM);
5199	#endif
5200
5201	return rc;
5202	}
5203

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 107200

Download in other formats: