VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 99654

Last change on this file since 99654 was 99133, checked in by vboxsync, 21 months ago

VMM: Nested VMX: bugref:10318 PGM fixes for supporting Hyper-V in a VM using hardware-assisted execution [Stats].

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 233.3 KB
Line 
1/* $Id: PGMAllBth.h 99133 2023-03-23 09:18:30Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6 * PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7 * bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8 *
9 * @remarks This file is one big \#ifdef-orgy!
10 *
11 */
12
13/*
14 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
15 *
16 * This file is part of VirtualBox base platform packages, as
17 * available from https://www.virtualbox.org.
18 *
19 * This program is free software; you can redistribute it and/or
20 * modify it under the terms of the GNU General Public License
21 * as published by the Free Software Foundation, in version 3 of the
22 * License.
23 *
24 * This program is distributed in the hope that it will be useful, but
25 * WITHOUT ANY WARRANTY; without even the implied warranty of
26 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
27 * General Public License for more details.
28 *
29 * You should have received a copy of the GNU General Public License
30 * along with this program; if not, see <https://www.gnu.org/licenses>.
31 *
32 * SPDX-License-Identifier: GPL-3.0-only
33 */
34
35#ifdef _MSC_VER
36/** @todo we're generating unnecessary code in nested/ept shadow mode and for
37 * real/prot-guest+RC mode. */
38# pragma warning(disable: 4505)
39#endif
40
41
42/*********************************************************************************************************************************
43* Internal Functions *
44*********************************************************************************************************************************/
45RT_C_DECLS_BEGIN
46PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
47#ifndef IN_RING3
48PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken);
49PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
50 bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken);
51# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
52static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
53 unsigned iPte, PCPGMPTWALKGST pGstWalkAll);
54static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
55 uint32_t uErr, PPGMPTWALKGST pGstWalkAll);
56static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll);
57# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
58#endif
59PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
60static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
61static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
62static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
63#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
64static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
65#else
66static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
67#endif
68PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
69PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
70PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
71#ifdef VBOX_STRICT
72PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
73#endif
74PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
75PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
76
77#ifdef IN_RING3
78PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
79#endif
80RT_C_DECLS_END
81
82
83
84
85/*
86 * Filter out some illegal combinations of guest and shadow paging, so we can
87 * remove redundant checks inside functions.
88 */
89#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
90 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
91# error "Invalid combination; PAE guest implies PAE shadow"
92#endif
93
94#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
95 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 \
96 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
97# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
98#endif
99
100#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
101 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE \
102 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
103# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
104#endif
105
106#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
107 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
108# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
109#endif
110
111
112/**
113 * Enters the shadow+guest mode.
114 *
115 * @returns VBox status code.
116 * @param pVCpu The cross context virtual CPU structure.
117 * @param GCPhysCR3 The physical address from the CR3 register.
118 */
119PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
120{
121 /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
122 * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
123 */
124#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
125 || PGM_SHW_TYPE == PGM_TYPE_PAE \
126 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
127 && ( PGM_GST_TYPE == PGM_TYPE_REAL \
128 || PGM_GST_TYPE == PGM_TYPE_PROT))
129
130 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
131
132 Assert(!pVM->pgm.s.fNestedPaging);
133
134 PGM_LOCK_VOID(pVM);
135 /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
136 * but any calls to GC need a proper shadow page setup as well.
137 */
138 /* Free the previous root mapping if still active. */
139 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
140 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
141 if (pOldShwPageCR3)
142 {
143 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
144
145 /* Mark the page as unlocked; allow flushing again. */
146 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
147
148 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
149 pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
150 pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
151 }
152
153 /* construct a fake address. */
154 GCPhysCR3 = RT_BIT_64(63);
155 PPGMPOOLPAGE pNewShwPageCR3;
156 int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
157 NIL_PGMPOOL_IDX, UINT32_MAX, false /*fLockPage*/,
158 &pNewShwPageCR3);
159 AssertRCReturn(rc, rc);
160
161 pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
162 pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
163
164 /* Mark the page as locked; disallow flushing. */
165 pgmPoolLockPage(pPool, pNewShwPageCR3);
166
167 /* Set the current hypervisor CR3. */
168 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
169
170 PGM_UNLOCK(pVM);
171 return rc;
172#else
173 NOREF(pVCpu); NOREF(GCPhysCR3);
174 return VINF_SUCCESS;
175#endif
176}
177
178
179#ifndef IN_RING3
180
181# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
182/**
183 * Deal with a guest page fault.
184 *
185 * @returns Strict VBox status code.
186 * @retval VINF_EM_RAW_GUEST_TRAP
187 * @retval VINF_EM_RAW_EMULATE_INSTR
188 *
189 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
190 * @param pWalk The guest page table walk result.
191 * @param uErr The error code.
192 */
193PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr)
194{
195 /*
196 * Calc the error code for the guest trap.
197 */
198 uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
199 ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)
200 : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
201 if ( pWalk->fRsvdError
202 || pWalk->fBadPhysAddr)
203 {
204 uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P;
205 Assert(!pWalk->fNotPresent);
206 }
207 else if (!pWalk->fNotPresent)
208 uNewErr |= X86_TRAP_PF_P;
209 TRPMSetErrorCode(pVCpu, uNewErr);
210
211 LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel));
212 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
213 return VINF_EM_RAW_GUEST_TRAP;
214}
215# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
216
217
218#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
219/**
220 * Deal with a guest page fault.
221 *
222 * The caller has taken the PGM lock.
223 *
224 * @returns Strict VBox status code.
225 *
226 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
227 * @param uErr The error code.
228 * @param pCtx Pointer to the register context for the CPU.
229 * @param pvFault The fault address.
230 * @param pPage The guest page at @a pvFault.
231 * @param pWalk The guest page table walk result.
232 * @param pGstWalk The guest paging-mode specific walk information.
233 * @param pfLockTaken PGM lock taken here or not (out). This is true
234 * when we're called.
235 */
236static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
237 RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
238# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
239 , PPGMPTWALK pWalk
240 , PGSTPTWALK pGstWalk
241# endif
242 )
243{
244# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
245 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
246# endif
247 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
248 VBOXSTRICTRC rcStrict;
249
250 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
251 {
252 /*
253 * Physical page access handler.
254 */
255# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
256 const RTGCPHYS GCPhysFault = pWalk->GCPhys;
257# else
258 const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
259# endif
260 PPGMPHYSHANDLER pCur;
261 rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysFault, &pCur);
262 if (RT_SUCCESS(rcStrict))
263 {
264 PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
265
266# ifdef PGM_SYNC_N_PAGES
267 /*
268 * If the region is write protected and we got a page not present fault, then sync
269 * the pages. If the fault was caused by a read, then restart the instruction.
270 * In case of write access continue to the GC write handler.
271 *
272 * ASSUMES that there is only one handler per page or that they have similar write properties.
273 */
274 if ( !(uErr & X86_TRAP_PF_P)
275 && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
276 {
277# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
278 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
279# else
280 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
281# endif
282 if ( RT_FAILURE(rcStrict)
283 || !(uErr & X86_TRAP_PF_RW)
284 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
285 {
286 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
287 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
288 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
289 return rcStrict;
290 }
291 }
292# endif
293# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
294 /*
295 * If the access was not thru a #PF(RSVD|...) resync the page.
296 */
297 if ( !(uErr & X86_TRAP_PF_RSVD)
298 && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
299# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
300 && (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK))
301 == PGM_PTATTRS_W_MASK /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
302# endif
303 )
304 {
305# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
306 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
307# else
308 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
309# endif
310 if ( RT_FAILURE(rcStrict)
311 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
312 {
313 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
314 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
315 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
316 return rcStrict;
317 }
318 }
319# endif
320
321 AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
322 || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
323 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
324 pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
325 if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
326 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
327 else
328 {
329 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
330 if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
331 }
332
333 if (pCurType->pfnPfHandler)
334 {
335 STAM_PROFILE_START(&pCur->Stat, h);
336
337 if (pCurType->fKeepPgmLock)
338 {
339 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault,
340 !pCurType->fRing0DevInsIdx ? pCur->uUser
341 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser));
342
343 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
344 }
345 else
346 {
347 uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
348 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
349 PGM_UNLOCK(pVM);
350 *pfLockTaken = false;
351
352 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault, uUser);
353
354 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
355 }
356 }
357 else
358 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
359
360 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
361 return rcStrict;
362 }
363 AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
364 }
365
366 /*
367 * There is a handled area of the page, but this fault doesn't belong to it.
368 * We must emulate the instruction.
369 *
370 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
371 * we first check if this was a page-not-present fault for a page with only
372 * write access handlers. Restart the instruction if it wasn't a write access.
373 */
374 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled);
375
376 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
377 && !(uErr & X86_TRAP_PF_P))
378 {
379# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
380 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
381# else
382 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
383# endif
384 if ( RT_FAILURE(rcStrict)
385 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
386 || !(uErr & X86_TRAP_PF_RW))
387 {
388 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
389 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
390 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
391 return rcStrict;
392 }
393 }
394
395 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
396 * It's writing to an unhandled part of the LDT page several million times.
397 */
398 rcStrict = PGMInterpretInstruction(pVCpu, pvFault);
399 LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
400 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; });
401 return rcStrict;
402} /* if any kind of handler */
403# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
404
405
406/**
407 * \#PF Handler for raw-mode guest execution.
408 *
409 * @returns VBox status code (appropriate for trap handling and GC return).
410 *
411 * @param pVCpu The cross context virtual CPU structure.
412 * @param uErr The trap error code.
413 * @param pCtx Pointer to the register context for the CPU.
414 * @param pvFault The fault address.
415 * @param pfLockTaken PGM lock taken here or not (out)
416 */
417PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken)
418{
419 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
420
421 *pfLockTaken = false;
422
423# if ( PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \
424 || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
425 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
426 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
427 && PGM_SHW_TYPE != PGM_TYPE_NONE
428 int rc;
429
430# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
431 /*
432 * Walk the guest page translation tables and check if it's a guest fault.
433 */
434 PGMPTWALK Walk;
435 GSTPTWALK GstWalk;
436 rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk);
437 if (RT_FAILURE_NP(rc))
438 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
439
440 /* assert some GstWalk sanity. */
441# if PGM_GST_TYPE == PGM_TYPE_AMD64
442 /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. */
443# endif
444# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
445 /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */
446# endif
447 /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */
448 /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */
449 Assert(Walk.fSucceeded);
450 Assert(Walk.fEffective & PGM_PTATTRS_R_MASK);
451
452 if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
453 {
454 if ( ( (uErr & X86_TRAP_PF_RW)
455 && !(Walk.fEffective & PGM_PTATTRS_W_MASK)
456 && ( (uErr & X86_TRAP_PF_US)
457 || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
458 || ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK))
459 || ((uErr & X86_TRAP_PF_ID) && (Walk.fEffective & PGM_PTATTRS_NX_MASK))
460 )
461 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
462 }
463
464 /* Take the big lock now before we update flags. */
465 *pfLockTaken = true;
466 PGM_LOCK_VOID(pVM);
467
468 /*
469 * Set the accessed and dirty flags.
470 */
471 /** @todo Should probably use cmpxchg logic here as we're potentially racing
472 * other CPUs in SMP configs. (the lock isn't enough, since we take it
473 * after walking and the page tables could be stale already) */
474# if PGM_GST_TYPE == PGM_TYPE_AMD64
475 if (!(GstWalk.Pml4e.u & X86_PML4E_A))
476 {
477 GstWalk.Pml4e.u |= X86_PML4E_A;
478 GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A);
479 }
480 if (!(GstWalk.Pdpe.u & X86_PDPE_A))
481 {
482 GstWalk.Pdpe.u |= X86_PDPE_A;
483 GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A);
484 }
485# endif
486 if (Walk.fBigPage)
487 {
488 Assert(GstWalk.Pde.u & X86_PDE_PS);
489 if (uErr & X86_TRAP_PF_RW)
490 {
491 if ((GstWalk.Pde.u & (X86_PDE4M_A | X86_PDE4M_D)) != (X86_PDE4M_A | X86_PDE4M_D))
492 {
493 GstWalk.Pde.u |= X86_PDE4M_A | X86_PDE4M_D;
494 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A | X86_PDE4M_D);
495 }
496 }
497 else
498 {
499 if (!(GstWalk.Pde.u & X86_PDE4M_A))
500 {
501 GstWalk.Pde.u |= X86_PDE4M_A;
502 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A);
503 }
504 }
505 }
506 else
507 {
508 Assert(!(GstWalk.Pde.u & X86_PDE_PS));
509 if (!(GstWalk.Pde.u & X86_PDE_A))
510 {
511 GstWalk.Pde.u |= X86_PDE_A;
512 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A);
513 }
514
515 if (uErr & X86_TRAP_PF_RW)
516 {
517# ifdef VBOX_WITH_STATISTICS
518 if (GstWalk.Pte.u & X86_PTE_D)
519 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty));
520 else
521 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage));
522# endif
523 if ((GstWalk.Pte.u & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
524 {
525 GstWalk.Pte.u |= X86_PTE_A | X86_PTE_D;
526 GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A | X86_PTE_D);
527 }
528 }
529 else
530 {
531 if (!(GstWalk.Pte.u & X86_PTE_A))
532 {
533 GstWalk.Pte.u |= X86_PTE_A;
534 GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A);
535 }
536 }
537 Assert(GstWalk.Pte.u == GstWalk.pPte->u);
538 }
539#if 0
540 /* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */
541 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u,
542 ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
543#endif
544
545# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
546 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */
547
548 /* Take the big lock now. */
549 *pfLockTaken = true;
550 PGM_LOCK_VOID(pVM);
551# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
552
553# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
554 /*
555 * If it is a reserved bit fault we know that it is an MMIO (access
556 * handler) related fault and can skip some 200 lines of code.
557 */
558 if (uErr & X86_TRAP_PF_RSVD)
559 {
560 Assert(uErr & X86_TRAP_PF_P);
561 PPGMPAGE pPage;
562# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
563 rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage);
564 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
565 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage,
566 pfLockTaken, &Walk, &GstWalk));
567 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
568# else
569 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
570 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
571 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
572 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
573# endif
574 AssertRC(rc);
575 PGM_INVL_PG(pVCpu, pvFault);
576 return rc; /* Restart with the corrected entry. */
577 }
578# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
579
580 /*
581 * Fetch the guest PDE, PDPE and PML4E.
582 */
583# if PGM_SHW_TYPE == PGM_TYPE_32BIT
584 const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
585 PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
586
587# elif PGM_SHW_TYPE == PGM_TYPE_PAE
588 const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
589 PX86PDPAE pPDDst;
590# if PGM_GST_TYPE == PGM_TYPE_PAE
591 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
592# else
593 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
594# endif
595 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
596
597# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
598 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
599 PX86PDPAE pPDDst;
600# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
601 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A,
602 X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst);
603# else
604 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
605# endif
606 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
607
608# elif PGM_SHW_TYPE == PGM_TYPE_EPT
609 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
610 PEPTPD pPDDst;
611 rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
612 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
613# endif
614 Assert(pPDDst);
615
616# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
617 /*
618 * Dirty page handling.
619 *
620 * If we successfully correct the write protection fault due to dirty bit
621 * tracking, then return immediately.
622 */
623 if (uErr & X86_TRAP_PF_RW) /* write fault? */
624 {
625 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
626 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
627 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
628 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
629 {
630 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0
631 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
632 ? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed
633 : &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
634 Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
635 return VINF_SUCCESS;
636 }
637#ifdef DEBUG_bird
638 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
639 AssertMsg(Walk.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
640#endif
641 }
642
643# if 0 /* rarely useful; leave for debugging. */
644 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
645# endif
646# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
647
648 /*
649 * A common case is the not-present error caused by lazy page table syncing.
650 *
651 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
652 * here so we can safely assume that the shadow PT is present when calling
653 * SyncPage later.
654 *
655 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
656 * of mapping conflict and defer to SyncCR3 in R3.
657 * (Again, we do NOT support access handlers for non-present guest pages.)
658 *
659 */
660# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
661 Assert(GstWalk.Pde.u & X86_PDE_P);
662# endif
663 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
664 && !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
665 {
666 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
667# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
668 LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
669 rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
670# else
671 LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
672 rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
673# endif
674 if (RT_SUCCESS(rc))
675 return rc;
676 Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
677 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
678 return VINF_PGM_SYNC_CR3;
679 }
680
681 /*
682 * Check if this fault address is flagged for special treatment,
683 * which means we'll have to figure out the physical address and
684 * check flags associated with it.
685 *
686 * ASSUME that we can limit any special access handling to pages
687 * in page tables which the guest believes to be present.
688 */
689# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
690 RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
691# else
692 RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
693# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
694 PPGMPAGE pPage;
695 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
696 if (RT_FAILURE(rc))
697 {
698 /*
699 * When the guest accesses invalid physical memory (e.g. probing
700 * of RAM or accessing a remapped MMIO range), then we'll fall
701 * back to the recompiler to emulate the instruction.
702 */
703 LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
704 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
705 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
706 return VINF_EM_RAW_EMULATE_INSTR;
707 }
708
709 /*
710 * Any handlers for this page?
711 */
712 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
713# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
714 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken,
715 &Walk, &GstWalk));
716# else
717 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
718# endif
719
720 /*
721 * We are here only if page is present in Guest page tables and
722 * trap is not handled by our handlers.
723 *
724 * Check it for page out-of-sync situation.
725 */
726 if (!(uErr & X86_TRAP_PF_P))
727 {
728 /*
729 * Page is not present in our page tables. Try to sync it!
730 */
731 if (uErr & X86_TRAP_PF_US)
732 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
733 else /* supervisor */
734 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
735
736 if (PGM_PAGE_IS_BALLOONED(pPage))
737 {
738 /* Emulate reads from ballooned pages as they are not present in
739 our shadow page tables. (Required for e.g. Solaris guests; soft
740 ecc, random nr generator.) */
741 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
742 LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
743 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
744 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
745 return rc;
746 }
747
748# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
749 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
750# else
751 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
752# endif
753 if (RT_SUCCESS(rc))
754 {
755 /* The page was successfully synced, return to the guest. */
756 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
757 return VINF_SUCCESS;
758 }
759 }
760 else /* uErr & X86_TRAP_PF_P: */
761 {
762 /*
763 * Write protected pages are made writable when the guest makes the
764 * first write to it. This happens for pages that are shared, write
765 * monitored or not yet allocated.
766 *
767 * We may also end up here when CR0.WP=0 in the guest.
768 *
769 * Also, a side effect of not flushing global PDEs are out of sync
770 * pages due to physical monitored regions, that are no longer valid.
771 * Assume for now it only applies to the read/write flag.
772 */
773 if (uErr & X86_TRAP_PF_RW)
774 {
775 /*
776 * Check if it is a read-only page.
777 */
778 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
779 {
780 Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
781 Assert(!PGM_PAGE_IS_ZERO(pPage));
782 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
783 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
784
785 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
786 if (rc != VINF_SUCCESS)
787 {
788 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
789 return rc;
790 }
791 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
792 return VINF_EM_NO_MEMORY;
793 }
794
795# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
796 /*
797 * Check to see if we need to emulate the instruction if CR0.WP=0.
798 */
799 if ( !(Walk.fEffective & PGM_PTATTRS_W_MASK)
800 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
801 && CPUMGetGuestCPL(pVCpu) < 3)
802 {
803 Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P));
804
805 /*
806 * The Netware WP0+RO+US hack.
807 *
808 * Netware sometimes(/always?) runs with WP0. It has been observed doing
809 * excessive write accesses to pages which are mapped with US=1 and RW=0
810 * while WP=0. This causes a lot of exits and extremely slow execution.
811 * To avoid trapping and emulating every write here, we change the shadow
812 * page table entry to map it as US=0 and RW=1 until user mode tries to
813 * access it again (see further below). We count these shadow page table
814 * changes so we can avoid having to clear the page pool every time the WP
815 * bit changes to 1 (see PGMCr0WpEnabled()).
816 */
817# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) && 1
818 if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
819 && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
820 && pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
821 {
822 Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage));
823 rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
824 if (rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3)
825 {
826 PGM_INVL_PG(pVCpu, pvFault);
827 pVCpu->pgm.s.cNetwareWp0Hacks++;
828 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; });
829 return rc;
830 }
831 AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
832 Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
833 }
834# endif
835
836 /* Interpret the access. */
837 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
838 Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK)));
839 if (RT_SUCCESS(rc))
840 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ);
841 else
842 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3);
843 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; });
844 return rc;
845 }
846# endif
847 /// @todo count the above case; else
848 if (uErr & X86_TRAP_PF_US)
849 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
850 else /* supervisor */
851 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
852
853 /*
854 * Sync the page.
855 *
856 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
857 * page is not present, which is not true in this case.
858 */
859# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
860 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
861# else
862 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
863# endif
864 if (RT_SUCCESS(rc))
865 {
866 /*
867 * Page was successfully synced, return to guest but invalidate
868 * the TLB first as the page is very likely to be in it.
869 */
870# if PGM_SHW_TYPE == PGM_TYPE_EPT
871 HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
872# else
873 PGM_INVL_PG(pVCpu, pvFault);
874# endif
875# ifdef VBOX_STRICT
876 PGMPTWALK GstPageWalk;
877 GstPageWalk.GCPhys = RTGCPHYS_MAX;
878 if (!pVM->pgm.s.fNestedPaging)
879 {
880 rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
881 AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) || ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective));
882 LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective));
883 }
884# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
885 uint64_t fPageShw = 0;
886 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
887 AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
888 ("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault));
889# endif
890# endif /* VBOX_STRICT */
891 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
892 return VINF_SUCCESS;
893 }
894 }
895# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
896 /*
897 * Check for Netware WP0+RO+US hack from above and undo it when user
898 * mode accesses the page again.
899 */
900 else if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
901 && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
902 && pVCpu->pgm.s.cNetwareWp0Hacks > 0
903 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
904 && CPUMGetGuestCPL(pVCpu) == 3
905 && pVM->cCpus == 1
906 )
907 {
908 Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
909 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
910 if (RT_SUCCESS(rc))
911 {
912 PGM_INVL_PG(pVCpu, pvFault);
913 pVCpu->pgm.s.cNetwareWp0Hacks--;
914 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; });
915 return VINF_SUCCESS;
916 }
917 }
918# endif /* PGM_WITH_PAGING */
919
920 /** @todo else: why are we here? */
921
922# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
923 /*
924 * Check for VMM page flags vs. Guest page flags consistency.
925 * Currently only for debug purposes.
926 */
927 if (RT_SUCCESS(rc))
928 {
929 /* Get guest page flags. */
930 PGMPTWALK GstPageWalk;
931 int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
932 if (RT_SUCCESS(rc2))
933 {
934 uint64_t fPageShw = 0;
935 rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
936
937#if 0
938 /*
939 * Compare page flags.
940 * Note: we have AVL, A, D bits desynced.
941 */
942 AssertMsg( (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
943 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
944 || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
945 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
946 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
947 && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW
948 && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US),
949 ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
950 pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
95101:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
95201:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
953
95401:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
95501:01:15.625516 00:08:43.268051 Location :
956e:\vbox\svn\trunk\srcPage flags mismatch!
957pvFault=fffff801b0d7b000
958 uErr=11 X86_TRAP_PF_ID | X86_TRAP_PF_P
959GCPhys=0000000019b52000
960fPageShw=0
961fPageGst=77b0000000000121
962rc=0
963#endif
964
965 }
966 else
967 AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
968 }
969 else
970 AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
971# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
972 }
973
974
975 /*
976 * If we get here it is because something failed above, i.e. most like guru
977 * meditiation time.
978 */
979 LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
980 __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pCtx->cs.Sel, pCtx->rip));
981 return rc;
982
983# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
984 NOREF(uErr); NOREF(pCtx); NOREF(pvFault);
985 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
986 return VERR_PGM_NOT_USED_IN_MODE;
987# endif
988}
989
990
991# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
992/**
993 * Deals with a nested-guest \#PF fault for a guest-physical page with a handler.
994 *
995 * @returns Strict VBox status code.
996 * @param pVCpu The cross context virtual CPU structure.
997 * @param uErr The error code.
998 * @param pCtx Pointer to the register context for the CPU.
999 * @param GCPhysNestedFault The nested-guest physical address of the fault.
1000 * @param pPage The guest page at @a GCPhysNestedFault.
1001 * @param GCPhysFault The guest-physical address of the fault.
1002 * @param pGstWalkAll The guest page walk result.
1003 * @param pfLockTaken Where to store whether the PGM is still held when
1004 * this function completes.
1005 *
1006 * @note The caller has taken the PGM lock.
1007 */
1008static VBOXSTRICTRC PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
1009 RTGCPHYS GCPhysNestedFault, PPGMPAGE pPage,
1010 RTGCPHYS GCPhysFault, PPGMPTWALKGST pGstWalkAll,
1011 bool *pfLockTaken)
1012{
1013# if PGM_GST_TYPE == PGM_TYPE_PROT \
1014 && PGM_SHW_TYPE == PGM_TYPE_EPT
1015
1016 /** @todo Assert uErr isn't X86_TRAP_PF_RSVD and remove release checks. */
1017 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysFault);
1018 AssertMsgReturn(PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage), ("%RGp %RGp uErr=%u\n", GCPhysNestedFault, GCPhysFault, uErr),
1019 VERR_PGM_HANDLER_IPE_1);
1020
1021 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1022 RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1023 RTGCPHYS const GCPhysPage = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1024
1025 /*
1026 * Physical page access handler.
1027 */
1028 PPGMPHYSHANDLER pCur;
1029 VBOXSTRICTRC rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pCur);
1030 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
1031
1032 PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
1033 Assert(pCurType);
1034
1035 /*
1036 * If the region is write protected and we got a page not present fault, then sync
1037 * the pages. If the fault was caused by a read, then restart the instruction.
1038 * In case of write access continue to the GC write handler.
1039 */
1040 if ( !(uErr & X86_TRAP_PF_P)
1041 && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1042 {
1043 Log7Func(("Syncing Monitored: GCPhysNestedPage=%RGp GCPhysPage=%RGp uErr=%#x\n", GCPhysNestedPage, GCPhysPage, uErr));
1044 rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
1045 Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1046 if ( RT_FAILURE(rcStrict)
1047 || !(uErr & X86_TRAP_PF_RW))
1048 {
1049 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1050 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1051 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1052 return rcStrict;
1053 }
1054 }
1055 else if ( !(uErr & X86_TRAP_PF_RSVD)
1056 && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE)
1057 {
1058 /*
1059 * If the access was NOT through an EPT misconfig (i.e. RSVD), sync the page.
1060 * This can happen for the VMX APIC-access page.
1061 */
1062 Log7Func(("Syncing MMIO: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1063 rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
1064 Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1065 if (RT_FAILURE(rcStrict))
1066 {
1067 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1068 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1069 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1070 return rcStrict;
1071 }
1072 }
1073
1074 AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
1075 || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
1076 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
1077 GCPhysNestedFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
1078 if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1079 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
1080 else
1081 {
1082 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
1083 if (uErr & X86_TRAP_PF_RSVD)
1084 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
1085 }
1086
1087 if (pCurType->pfnPfHandler)
1088 {
1089 STAM_PROFILE_START(&pCur->Stat, h);
1090 uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
1091 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
1092
1093 if (pCurType->fKeepPgmLock)
1094 {
1095 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1096 STAM_PROFILE_STOP(&pCur->Stat, h);
1097 }
1098 else
1099 {
1100 PGM_UNLOCK(pVM);
1101 *pfLockTaken = false;
1102 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1103 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
1104 }
1105 }
1106 else
1107 {
1108 AssertMsgFailed(("What's going on here!? Fault falls outside handler range!?\n"));
1109 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
1110 }
1111
1112 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
1113 return rcStrict;
1114
1115# else
1116 RT_NOREF8(pVCpu, uErr, pCtx, GCPhysNestedFault, pPage, GCPhysFault, pGstWalkAll, pfLockTaken);
1117 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1118 return VERR_PGM_NOT_USED_IN_MODE;
1119# endif
1120}
1121# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1122
1123
1124/**
1125 * Nested \#PF handler for nested-guest hardware-assisted execution using nested
1126 * paging.
1127 *
1128 * @returns VBox status code (appropriate for trap handling and GC return).
1129 * @param pVCpu The cross context virtual CPU structure.
1130 * @param uErr The fault error (X86_TRAP_PF_*).
1131 * @param pCtx Pointer to the register context for the CPU.
1132 * @param GCPhysNestedFault The nested-guest physical address of the fault.
1133 * @param fIsLinearAddrValid Whether translation of a nested-guest linear address
1134 * caused this fault. If @c false, GCPtrNestedFault
1135 * must be 0.
1136 * @param GCPtrNestedFault The nested-guest linear address of this fault.
1137 * @param pWalk The guest page table walk result.
1138 * @param pfLockTaken Where to store whether the PGM lock is still held
1139 * when this function completes.
1140 */
1141PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
1142 bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken)
1143{
1144 *pfLockTaken = false;
1145# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \
1146 && PGM_GST_TYPE == PGM_TYPE_PROT \
1147 && PGM_SHW_TYPE == PGM_TYPE_EPT
1148
1149 Assert(CPUMIsGuestVmxEptPagingEnabled(pVCpu));
1150 Assert(PGM_A20_IS_ENABLED(pVCpu));
1151
1152 /* We don't support mode-based execute control for EPT yet. */
1153 Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
1154 Assert(!(uErr & X86_TRAP_PF_US));
1155
1156 /* Take the big lock now. */
1157 *pfLockTaken = true;
1158 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1159 PGM_LOCK_VOID(pVM);
1160
1161 /*
1162 * Walk the guest EPT tables and check if it's an EPT violation or misconfiguration.
1163 */
1164 if (fIsLinearAddrValid)
1165 Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x GCPtrNestedFault=%RGv\n",
1166 pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr, GCPtrNestedFault));
1167 else
1168 Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x\n",
1169 pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr));
1170 PGMPTWALKGST GstWalkAll;
1171 int rc = pgmGstSlatWalk(pVCpu, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk, &GstWalkAll);
1172 if (RT_FAILURE(rc))
1173 return rc;
1174
1175 Assert(GstWalkAll.enmType == PGMPTWALKGSTTYPE_EPT);
1176 Assert(pWalk->fSucceeded);
1177 Assert(pWalk->fEffective & (PGM_PTATTRS_EPT_R_MASK | PGM_PTATTRS_EPT_W_MASK | PGM_PTATTRS_EPT_X_SUPER_MASK));
1178 Assert(pWalk->fIsSlat);
1179
1180# ifdef DEBUG_ramshankar
1181 /* Paranoia. */
1182 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_R_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_R_MASK));
1183 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_W_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_W_MASK));
1184 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_NX_MASK) == !RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_X_SUPER_MASK));
1185# endif
1186
1187 Log7Func(("SLAT: GCPhysNestedFault=%RGp -> GCPhys=%#RGp\n", GCPhysNestedFault, pWalk->GCPhys));
1188
1189 /*
1190 * Check page-access permissions.
1191 */
1192 if ( ((uErr & X86_TRAP_PF_RW) && !(pWalk->fEffective & PGM_PTATTRS_W_MASK))
1193 || ((uErr & X86_TRAP_PF_ID) && (pWalk->fEffective & PGM_PTATTRS_NX_MASK)))
1194 {
1195 Log7Func(("Permission failed! GCPtrNested=%RGv GCPhysNested=%RGp uErr=%#x fEffective=%#RX64\n", GCPtrNestedFault,
1196 GCPhysNestedFault, uErr, pWalk->fEffective));
1197 pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION;
1198 return VERR_ACCESS_DENIED;
1199 }
1200
1201 PGM_A20_ASSERT_MASKED(pVCpu, pWalk->GCPhys);
1202 RTGCPHYS const GCPhysPage = pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1203 RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1204
1205 /*
1206 * If we were called via an EPT misconfig, it should've already resulted in a nested-guest VM-exit.
1207 */
1208 AssertMsgReturn(!(uErr & X86_TRAP_PF_RSVD),
1209 ("Unexpected EPT misconfig VM-exit. GCPhysPage=%RGp GCPhysNestedPage=%RGp\n", GCPhysPage, GCPhysNestedPage),
1210 VERR_PGM_MAPPING_IPE);
1211
1212 /*
1213 * Fetch and sync the nested-guest EPT page directory pointer.
1214 */
1215 PEPTPD pEptPd;
1216 rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL /*ppPdpt*/, &pEptPd, &GstWalkAll);
1217 AssertRCReturn(rc, rc);
1218 Assert(pEptPd);
1219
1220 /*
1221 * A common case is the not-present error caused by lazy page table syncing.
1222 *
1223 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
1224 * here so we can safely assume that the shadow PT is present when calling
1225 * NestedSyncPage later.
1226 *
1227 * NOTE: It's possible we will be syncing the VMX APIC-access page here.
1228 * In that case, we would sync the page but will NOT go ahead with emulating
1229 * the APIC-access VM-exit through IEM. However, once the page is mapped in
1230 * the shadow tables, subsequent APIC-access VM-exits for the nested-guest
1231 * will be triggered by hardware. Maybe calling the IEM #PF handler can be
1232 * considered as an optimization later.
1233 */
1234 unsigned const iPde = (GCPhysNestedPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1235 if ( !(uErr & X86_TRAP_PF_P)
1236 && !(pEptPd->a[iPde].u & EPT_PRESENT_MASK))
1237 {
1238 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
1239 Log7Func(("NestedSyncPT: Lazy. GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1240 rc = PGM_BTH_NAME(NestedSyncPT)(pVCpu, GCPhysNestedPage, GCPhysPage, &GstWalkAll);
1241 if (RT_SUCCESS(rc))
1242 return rc;
1243 AssertMsgFailedReturn(("NestedSyncPT: %RGv failed! rc=%Rrc\n", GCPhysNestedPage, rc), VERR_PGM_MAPPING_IPE);
1244 }
1245
1246 /*
1247 * Check if this fault address is flagged for special treatment.
1248 * This handles faults on an MMIO or write-monitored page.
1249 *
1250 * If this happens to be the VMX APIC-access page, we don't treat is as MMIO
1251 * but rather sync it further below (as a regular guest page) which lets
1252 * hardware-assisted execution trigger the APIC-access VM-exits of the
1253 * nested-guest directly.
1254 */
1255 PPGMPAGE pPage;
1256 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1257 if (RT_FAILURE(rc))
1258 {
1259 /*
1260 * We failed to get the physical page which means it's a reserved/invalid
1261 * page address (not MMIO even). This can typically be observed with
1262 * Microsoft Hyper-V enabled Windows guests. We must fall back to emulating
1263 * the instruction, see @bugref{10318#c7}.
1264 */
1265 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
1266 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
1267 return VINF_EM_RAW_EMULATE_INSTR;
1268 }
1269 /* Check if this is an MMIO page and NOT the VMX APIC-access page. */
1270 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1271 {
1272 Log7Func(("MMIO: Calling NestedTrap0eHandlerDoAccessHandlers for GCPhys %RGp\n", GCPhysPage));
1273 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, GCPhysNestedFault,
1274 pPage, pWalk->GCPhys, &GstWalkAll,
1275 pfLockTaken));
1276 }
1277
1278 /*
1279 * We are here only if page is present in nested-guest page tables but the
1280 * trap is not handled by our handlers. Check for page out-of-sync situation.
1281 */
1282 if (!(uErr & X86_TRAP_PF_P))
1283 {
1284 Assert(!PGM_PAGE_IS_BALLOONED(pPage));
1285 Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1286 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
1287
1288 Log7Func(("SyncPage: Not-Present: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedFault, GCPhysPage));
1289 rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, PGM_SYNC_NR_PAGES, uErr, &GstWalkAll);
1290 if (RT_SUCCESS(rc))
1291 {
1292 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
1293 return VINF_SUCCESS;
1294 }
1295 }
1296 else if (uErr & X86_TRAP_PF_RW)
1297 {
1298 /*
1299 * Write protected pages are made writable when the guest makes the
1300 * first write to it. This happens for pages that are shared, write
1301 * monitored or not yet allocated.
1302 *
1303 * We may also end up here when CR0.WP=0 in the guest.
1304 *
1305 * Also, a side effect of not flushing global PDEs are out of sync
1306 * pages due to physical monitored regions, that are no longer valid.
1307 * Assume for now it only applies to the read/write flag.
1308 */
1309 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1310 {
1311 /* This is a read-only page. */
1312 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysPage));
1313 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
1314
1315 Log7Func(("Calling pgmPhysPageMakeWritable for GCPhysPage=%RGp\n", GCPhysPage));
1316 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1317 if (rc != VINF_SUCCESS)
1318 {
1319 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
1320 return rc;
1321 }
1322 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
1323 return VINF_EM_NO_MEMORY;
1324 }
1325
1326 Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1327 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1328
1329 /*
1330 * Sync the write-protected page.
1331 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1332 * page is not present, which is not true in this case.
1333 */
1334 Log7Func(("SyncPage: RW: cs:rip=%04x:%#RX64 GCPhysNestedPage=%RGp uErr=%#RX32 GCPhysPage=%RGp WalkGCPhys=%RGp\n",
1335 pCtx->cs.Sel, pCtx->rip, GCPhysNestedPage, (uint32_t)uErr, GCPhysPage, pWalk->GCPhys));
1336 rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /* cPages */, uErr, &GstWalkAll);
1337 if (RT_SUCCESS(rc))
1338 {
1339 HMInvalidatePhysPage(pVM, GCPhysPage);
1340 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
1341 return VINF_SUCCESS;
1342 }
1343 }
1344
1345 /*
1346 * If we get here it is because something failed above => guru meditation time.
1347 */
1348 LogRelFunc(("rc=%Rrc GCPhysNestedFault=%#RGp (%#RGp) uErr=%#RX32 cs:rip=%04x:%08RX64\n", rc, GCPhysNestedFault, GCPhysPage,
1349 (uint32_t)uErr, pCtx->cs.Sel, pCtx->rip));
1350 return VERR_PGM_MAPPING_IPE;
1351
1352# else /* !VBOX_WITH_NESTED_HWVIRT_VMX_EPT || PGM_GST_TYPE != PGM_TYPE_PROT || PGM_SHW_TYPE != PGM_TYPE_EPT */
1353 RT_NOREF7(pVCpu, uErr, pCtx, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk);
1354 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1355 return VERR_PGM_NOT_USED_IN_MODE;
1356# endif
1357}
1358
1359#endif /* !IN_RING3 */
1360
1361
1362/**
1363 * Emulation of the invlpg instruction.
1364 *
1365 *
1366 * @returns VBox status code.
1367 *
1368 * @param pVCpu The cross context virtual CPU structure.
1369 * @param GCPtrPage Page to invalidate.
1370 *
1371 * @remark ASSUMES that the guest is updating before invalidating. This order
1372 * isn't required by the CPU, so this is speculative and could cause
1373 * trouble.
1374 * @remark No TLB shootdown is done on any other VCPU as we assume that
1375 * invlpg emulation is the *only* reason for calling this function.
1376 * (The guest has to shoot down TLB entries on other CPUs itself)
1377 * Currently true, but keep in mind!
1378 *
1379 * @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1380 * Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1381 */
1382PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1383{
1384#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1385 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1386 && PGM_SHW_TYPE != PGM_TYPE_NONE
1387 int rc;
1388 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1389 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1390
1391 PGM_LOCK_ASSERT_OWNER(pVM);
1392
1393 LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1394
1395 /*
1396 * Get the shadow PD entry and skip out if this PD isn't present.
1397 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1398 */
1399# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1400 const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1401 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1402
1403 /* Fetch the pgm pool shadow descriptor. */
1404 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1405# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1406 if (!pShwPde)
1407 {
1408 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1409 return VINF_SUCCESS;
1410 }
1411# else
1412 Assert(pShwPde);
1413# endif
1414
1415# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1416 const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1417 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1418
1419 /* If the shadow PDPE isn't present, then skip the invalidate. */
1420# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1421 if (!pPdptDst || !(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1422# else
1423 if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1424# endif
1425 {
1426 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1427 PGM_INVL_PG(pVCpu, GCPtrPage);
1428 return VINF_SUCCESS;
1429 }
1430
1431 /* Fetch the pgm pool shadow descriptor. */
1432 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1433 AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1434
1435 PX86PDPAE pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1436 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1437 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1438
1439# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1440 /* PML4 */
1441 /*const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/
1442 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1443 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1444 PX86PDPAE pPDDst;
1445 PX86PDPT pPdptDst;
1446 PX86PML4E pPml4eDst;
1447 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1448 if (rc != VINF_SUCCESS)
1449 {
1450 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1451 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1452 PGM_INVL_PG(pVCpu, GCPtrPage);
1453 return VINF_SUCCESS;
1454 }
1455 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1456 Assert(pPDDst);
1457 Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P);
1458
1459 /* Fetch the pgm pool shadow descriptor. */
1460 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1461 Assert(pShwPde);
1462
1463# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1464
1465 const SHWPDE PdeDst = *pPdeDst;
1466 if (!(PdeDst.u & X86_PDE_P))
1467 {
1468 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1469 PGM_INVL_PG(pVCpu, GCPtrPage);
1470 return VINF_SUCCESS;
1471 }
1472
1473 /*
1474 * Get the guest PD entry and calc big page.
1475 */
1476# if PGM_GST_TYPE == PGM_TYPE_32BIT
1477 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1478 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1479 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1480# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1481 unsigned iPDSrc = 0;
1482# if PGM_GST_TYPE == PGM_TYPE_PAE
1483 X86PDPE PdpeSrcIgn;
1484 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1485# else /* AMD64 */
1486 PX86PML4E pPml4eSrcIgn;
1487 X86PDPE PdpeSrcIgn;
1488 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1489# endif
1490 GSTPDE PdeSrc;
1491
1492 if (pPDSrc)
1493 PdeSrc = pPDSrc->a[iPDSrc];
1494 else
1495 PdeSrc.u = 0;
1496# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1497 const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1498 const bool fIsBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1499 if (fWasBigPage != fIsBigPage)
1500 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1501
1502# ifdef IN_RING3
1503 /*
1504 * If a CR3 Sync is pending we may ignore the invalidate page operation
1505 * depending on the kind of sync and if it's a global page or not.
1506 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
1507 */
1508# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1509 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1510 || ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1511 && fIsBigPage
1512 && (PdeSrc.u & X86_PDE4M_G)
1513 )
1514 )
1515# else
1516 if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1517# endif
1518 {
1519 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1520 return VINF_SUCCESS;
1521 }
1522# endif /* IN_RING3 */
1523
1524 /*
1525 * Deal with the Guest PDE.
1526 */
1527 rc = VINF_SUCCESS;
1528 if (PdeSrc.u & X86_PDE_P)
1529 {
1530 Assert( (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1531 && ((PdeSrc.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW) || pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1532 if (!fIsBigPage)
1533 {
1534 /*
1535 * 4KB - page.
1536 */
1537 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1538 RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1539
1540# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1541 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1542 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1543# endif
1544 if (pShwPage->GCPhys == GCPhys)
1545 {
1546 /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1547 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1548
1549 PGSTPT pPTSrc;
1550 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1551 if (RT_SUCCESS(rc))
1552 {
1553 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1554 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1555 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1556 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1557 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1558 GCPtrPage, PteSrc.u & X86_PTE_P,
1559 (PteSrc.u & PdeSrc.u & X86_PTE_RW),
1560 (PteSrc.u & PdeSrc.u & X86_PTE_US),
1561 (uint64_t)PteSrc.u,
1562 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1563 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1564 }
1565 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1566 PGM_INVL_PG(pVCpu, GCPtrPage);
1567 }
1568 else
1569 {
1570 /*
1571 * The page table address changed.
1572 */
1573 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1574 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1575 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1576 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1577 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1578 PGM_INVL_VCPU_TLBS(pVCpu);
1579 }
1580 }
1581 else
1582 {
1583 /*
1584 * 2/4MB - page.
1585 */
1586 /* Before freeing the page, check if anything really changed. */
1587 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1588 RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1589# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1590 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1591 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1592# endif
1593 if ( pShwPage->GCPhys == GCPhys
1594 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1595 {
1596 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1597 /** @todo This test is wrong as it cannot check the G bit!
1598 * FIXME */
1599 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1600 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1601 && ( (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1602 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1603 {
1604 LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1605 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1606 return VINF_SUCCESS;
1607 }
1608 }
1609
1610 /*
1611 * Ok, the page table is present and it's been changed in the guest.
1612 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1613 * We could do this for some flushes in GC too, but we need an algorithm for
1614 * deciding which 4MB pages containing code likely to be executed very soon.
1615 */
1616 LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1617 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1618 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1619 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1620 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1621 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1622 }
1623 }
1624 else
1625 {
1626 /*
1627 * Page directory is not present, mark shadow PDE not present.
1628 */
1629 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1630 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1631 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1632 PGM_INVL_PG(pVCpu, GCPtrPage);
1633 }
1634 return rc;
1635
1636#else /* guest real and protected mode, nested + ept, none. */
1637 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1638 NOREF(pVCpu); NOREF(GCPtrPage);
1639 return VINF_SUCCESS;
1640#endif
1641}
1642
1643#if PGM_SHW_TYPE != PGM_TYPE_NONE
1644
1645/**
1646 * Update the tracking of shadowed pages.
1647 *
1648 * @param pVCpu The cross context virtual CPU structure.
1649 * @param pShwPage The shadow page.
1650 * @param HCPhys The physical page we is being dereferenced.
1651 * @param iPte Shadow PTE index
1652 * @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1653 */
1654DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1655 RTGCPHYS GCPhysPage)
1656{
1657 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1658
1659# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1660 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1661 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1662
1663 /* Use the hint we retrieved from the cached guest PT. */
1664 if (pShwPage->fDirty)
1665 {
1666 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1667
1668 Assert(pShwPage->cPresent);
1669 Assert(pPool->cPresent);
1670 pShwPage->cPresent--;
1671 pPool->cPresent--;
1672
1673 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1674 AssertRelease(pPhysPage);
1675 pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1676 return;
1677 }
1678# else
1679 NOREF(GCPhysPage);
1680# endif
1681
1682 STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a);
1683 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1684
1685 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1686 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1687 * 2. write protect all shadowed pages. I.e. implement caching.
1688 */
1689 /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1690
1691 /*
1692 * Find the guest address.
1693 */
1694 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1695 pRam;
1696 pRam = pRam->CTX_SUFF(pNext))
1697 {
1698 unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT;
1699 while (iPage-- > 0)
1700 {
1701 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1702 {
1703 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1704
1705 Assert(pShwPage->cPresent);
1706 Assert(pPool->cPresent);
1707 pShwPage->cPresent--;
1708 pPool->cPresent--;
1709
1710 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1711 STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a);
1712 return;
1713 }
1714 }
1715 }
1716
1717 for (;;)
1718 AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1719}
1720
1721
1722/**
1723 * Update the tracking of shadowed pages.
1724 *
1725 * @param pVCpu The cross context virtual CPU structure.
1726 * @param pShwPage The shadow page.
1727 * @param u16 The top 16-bit of the pPage->HCPhys.
1728 * @param pPage Pointer to the guest page. this will be modified.
1729 * @param iPTDst The index into the shadow table.
1730 */
1731DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16,
1732 PPGMPAGE pPage, const unsigned iPTDst)
1733{
1734 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1735
1736 /*
1737 * Just deal with the simple first time here.
1738 */
1739 if (!u16)
1740 {
1741 STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin);
1742 u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1743 /* Save the page table index. */
1744 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1745 }
1746 else
1747 u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1748
1749 /* write back */
1750 Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1751 PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1752
1753 /* update statistics. */
1754 pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1755 pShwPage->cPresent++;
1756 if (pShwPage->iFirstPresent > iPTDst)
1757 pShwPage->iFirstPresent = iPTDst;
1758}
1759
1760
1761/**
1762 * Modifies a shadow PTE to account for access handlers.
1763 *
1764 * @param pVM The cross context VM structure.
1765 * @param pVCpu The cross context virtual CPU structure.
1766 * @param pPage The page in question.
1767 * @param GCPhysPage The guest-physical address of the page.
1768 * @param fPteSrc The shadowed flags of the source PTE. Must include the
1769 * A (accessed) bit so it can be emulated correctly.
1770 * @param pPteDst The shadow PTE (output). This is temporary storage and
1771 * does not need to be set atomically.
1772 */
1773DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PVMCPUCC pVCpu, PCPGMPAGE pPage, RTGCPHYS GCPhysPage, uint64_t fPteSrc,
1774 PSHWPTE pPteDst)
1775{
1776 RT_NOREF_PV(pVM); RT_NOREF_PV(fPteSrc); RT_NOREF_PV(pVCpu); RT_NOREF_PV(GCPhysPage);
1777
1778 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1779 * Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1780 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1781 {
1782 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1783# if PGM_SHW_TYPE == PGM_TYPE_EPT
1784 pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) | EPT_E_READ | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
1785# else
1786 if (fPteSrc & X86_PTE_A)
1787 {
1788 SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage));
1789 SHW_PTE_SET_RO(*pPteDst);
1790 }
1791 else
1792 SHW_PTE_SET(*pPteDst, 0);
1793# endif
1794 }
1795# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1796# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
1797 else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1798 && ( BTH_IS_NP_ACTIVE(pVM)
1799 || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1800# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1801 && pVM->pgm.s.fLessThan52PhysicalAddressBits
1802# endif
1803 )
1804 {
1805 LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1806# if PGM_SHW_TYPE == PGM_TYPE_EPT
1807 /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1808 pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg
1809 /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1810 | EPT_E_WRITE
1811 /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1812 | EPT_E_MEMTYPE_INVALID_3;
1813# else
1814 /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1815 SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P);
1816# endif
1817 }
1818# endif
1819# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1820 else
1821 {
1822 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1823 SHW_PTE_SET(*pPteDst, 0);
1824 }
1825 /** @todo count these kinds of entries. */
1826}
1827
1828
1829/**
1830 * Creates a 4K shadow page for a guest page.
1831 *
1832 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1833 * physical address. The PdeSrc argument only the flags are used. No page
1834 * structured will be mapped in this function.
1835 *
1836 * @param pVCpu The cross context virtual CPU structure.
1837 * @param pPteDst Destination page table entry.
1838 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1839 * Can safely assume that only the flags are being used.
1840 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1841 * @param pShwPage Pointer to the shadow page.
1842 * @param iPTDst The index into the shadow table.
1843 *
1844 * @remark Not used for 2/4MB pages!
1845 */
1846# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
1847static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1848 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1849# else
1850static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1851 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1852# endif
1853{
1854 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1855 RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1856
1857# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1858 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1859 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1860
1861 if (pShwPage->fDirty)
1862 {
1863 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1864 PGSTPT pGstPT;
1865
1866 /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1867 pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1868 GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1869 pGstPT->a[iPTDst].u = PteSrc.u;
1870 }
1871# else
1872 Assert(!pShwPage->fDirty);
1873# endif
1874
1875# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1876 if ( (PteSrc.u & X86_PTE_P)
1877 && GST_IS_PTE_VALID(pVCpu, PteSrc))
1878# endif
1879 {
1880# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1881 RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1882# endif
1883 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1884
1885 /*
1886 * Find the ram range.
1887 */
1888 PPGMPAGE pPage;
1889 int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1890 if (RT_SUCCESS(rc))
1891 {
1892 /* Ignore ballooned pages.
1893 Don't return errors or use a fatal assert here as part of a
1894 shadow sync range might included ballooned pages. */
1895 if (PGM_PAGE_IS_BALLOONED(pPage))
1896 {
1897 Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */
1898 return;
1899 }
1900
1901# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1902 /* Make the page writable if necessary. */
1903 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1904 && ( PGM_PAGE_IS_ZERO(pPage)
1905# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1906 || ( (PteSrc.u & X86_PTE_RW)
1907# else
1908 || ( 1
1909# endif
1910 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1911# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1912 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1913# endif
1914# ifdef VBOX_WITH_PAGE_SHARING
1915 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1916# endif
1917 )
1918 )
1919 )
1920 {
1921 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1922 AssertRC(rc);
1923 }
1924# endif
1925
1926 /*
1927 * Make page table entry.
1928 */
1929 SHWPTE PteDst;
1930# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1931 uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1932# else
1933 uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D;
1934# endif
1935 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1936 {
1937# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1938 /*
1939 * If the page or page directory entry is not marked accessed,
1940 * we mark the page not present.
1941 */
1942 if (!(PteSrc.u & X86_PTE_A) || !(PdeSrc.u & X86_PDE_A))
1943 {
1944 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1945 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage));
1946 SHW_PTE_SET(PteDst, 0);
1947 }
1948 /*
1949 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1950 * when the page is modified.
1951 */
1952 else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW))
1953 {
1954 AssertCompile(X86_PTE_RW == X86_PDE_RW);
1955 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage));
1956 SHW_PTE_SET(PteDst,
1957 fGstShwPteFlags
1958 | PGM_PAGE_GET_HCPHYS(pPage)
1959 | PGM_PTFLAGS_TRACK_DIRTY);
1960 SHW_PTE_SET_RO(PteDst);
1961 }
1962 else
1963# endif
1964 {
1965 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped));
1966# if PGM_SHW_TYPE == PGM_TYPE_EPT
1967 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage)
1968 | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
1969# else
1970 SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage));
1971# endif
1972 }
1973
1974 /*
1975 * Make sure only allocated pages are mapped writable.
1976 */
1977 if ( SHW_PTE_IS_P_RW(PteDst)
1978 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1979 {
1980 /* Still applies to shared pages. */
1981 Assert(!PGM_PAGE_IS_ZERO(pPage));
1982 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1983 Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1984 }
1985 }
1986 else
1987 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhysPage, fGstShwPteFlags, &PteDst);
1988
1989 /*
1990 * Keep user track up to date.
1991 */
1992 if (SHW_PTE_IS_P(PteDst))
1993 {
1994 if (!SHW_PTE_IS_P(*pPteDst))
1995 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1996 else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1997 {
1998 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst)));
1999 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2000 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2001 }
2002 }
2003 else if (SHW_PTE_IS_P(*pPteDst))
2004 {
2005 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
2006 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2007 }
2008
2009 /*
2010 * Update statistics and commit the entry.
2011 */
2012# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2013 if (!(PteSrc.u & X86_PTE_G))
2014 pShwPage->fSeenNonGlobal = true;
2015# endif
2016 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2017 return;
2018 }
2019
2020/** @todo count these three different kinds. */
2021 Log2(("SyncPageWorker: invalid address in Pte\n"));
2022 }
2023# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2024 else if (!(PteSrc.u & X86_PTE_P))
2025 Log2(("SyncPageWorker: page not present in Pte\n"));
2026 else
2027 Log2(("SyncPageWorker: invalid Pte\n"));
2028# endif
2029
2030 /*
2031 * The page is not present or the PTE is bad. Replace the shadow PTE by
2032 * an empty entry, making sure to keep the user tracking up to date.
2033 */
2034 if (SHW_PTE_IS_P(*pPteDst))
2035 {
2036 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
2037 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2038 }
2039 SHW_PTE_ATOMIC_SET(*pPteDst, 0);
2040}
2041
2042
2043/**
2044 * Syncs a guest OS page.
2045 *
2046 * There are no conflicts at this point, neither is there any need for
2047 * page table allocations.
2048 *
2049 * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
2050 * When called in AMD64 guest mode, the guest PML4E shall be valid.
2051 *
2052 * @returns VBox status code.
2053 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
2054 * @param pVCpu The cross context virtual CPU structure.
2055 * @param PdeSrc Page directory entry of the guest.
2056 * @param GCPtrPage Guest context page address.
2057 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2058 * @param uErr Fault error (X86_TRAP_PF_*).
2059 */
2060static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
2061{
2062 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2063 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2064 LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
2065 RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
2066
2067 PGM_LOCK_ASSERT_OWNER(pVM);
2068
2069# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2070 || PGM_GST_TYPE == PGM_TYPE_PAE \
2071 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2072 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
2073
2074 /*
2075 * Assert preconditions.
2076 */
2077 Assert(PdeSrc.u & X86_PDE_P);
2078 Assert(cPages);
2079# if 0 /* rarely useful; leave for debugging. */
2080 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
2081# endif
2082
2083 /*
2084 * Get the shadow PDE, find the shadow page table in the pool.
2085 */
2086# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2087 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2088 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2089
2090 /* Fetch the pgm pool shadow descriptor. */
2091 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2092 Assert(pShwPde);
2093
2094# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2095 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2096 PPGMPOOLPAGE pShwPde = NULL;
2097 PX86PDPAE pPDDst;
2098
2099 /* Fetch the pgm pool shadow descriptor. */
2100 int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2101 AssertRCSuccessReturn(rc2, rc2);
2102 Assert(pShwPde);
2103
2104 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2105 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2106
2107# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2108 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2109 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2110 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2111 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2112
2113 int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2114 AssertRCSuccessReturn(rc2, rc2);
2115 Assert(pPDDst && pPdptDst);
2116 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2117# endif
2118 SHWPDE PdeDst = *pPdeDst;
2119
2120 /*
2121 * - In the guest SMP case we could have blocked while another VCPU reused
2122 * this page table.
2123 * - With W7-64 we may also take this path when the A bit is cleared on
2124 * higher level tables (PDPE/PML4E). The guest does not invalidate the
2125 * relevant TLB entries. If we're write monitoring any page mapped by
2126 * the modified entry, we may end up here with a "stale" TLB entry.
2127 */
2128 if (!(PdeDst.u & X86_PDE_P))
2129 {
2130 Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2131 AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW),
2132 ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2133 if (uErr & X86_TRAP_PF_P)
2134 PGM_INVL_PG(pVCpu, GCPtrPage);
2135 return VINF_SUCCESS; /* force the instruction to be executed again. */
2136 }
2137
2138 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2139 Assert(pShwPage);
2140
2141# if PGM_GST_TYPE == PGM_TYPE_AMD64
2142 /* Fetch the pgm pool shadow descriptor. */
2143 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2144 Assert(pShwPde);
2145# endif
2146
2147 /*
2148 * Check that the page is present and that the shadow PDE isn't out of sync.
2149 */
2150 const bool fBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
2151 const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
2152 RTGCPHYS GCPhys;
2153 if (!fBigPage)
2154 {
2155 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2156# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2157 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2158 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
2159# endif
2160 }
2161 else
2162 {
2163 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2164# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2165 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2166 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2167# endif
2168 }
2169 /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2170 if ( fPdeValid
2171 && pShwPage->GCPhys == GCPhys
2172 && (PdeSrc.u & X86_PDE_P)
2173 && (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
2174 && ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW))
2175# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2176 && ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) || !GST_IS_NX_ACTIVE(pVCpu))
2177# endif
2178 )
2179 {
2180 /*
2181 * Check that the PDE is marked accessed already.
2182 * Since we set the accessed bit *before* getting here on a #PF, this
2183 * check is only meant for dealing with non-#PF'ing paths.
2184 */
2185 if (PdeSrc.u & X86_PDE_A)
2186 {
2187 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2188 if (!fBigPage)
2189 {
2190 /*
2191 * 4KB Page - Map the guest page table.
2192 */
2193 PGSTPT pPTSrc;
2194 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2195 if (RT_SUCCESS(rc))
2196 {
2197# ifdef PGM_SYNC_N_PAGES
2198 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2199 if ( cPages > 1
2200 && !(uErr & X86_TRAP_PF_P)
2201 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2202 {
2203 /*
2204 * This code path is currently only taken when the caller is PGMTrap0eHandler
2205 * for non-present pages!
2206 *
2207 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2208 * deal with locality.
2209 */
2210 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2211# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2212 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2213 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2214# else
2215 const unsigned offPTSrc = 0;
2216# endif
2217 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2218 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2219 iPTDst = 0;
2220 else
2221 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2222
2223 for (; iPTDst < iPTDstEnd; iPTDst++)
2224 {
2225 const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2226
2227 if ( (pPteSrc->u & X86_PTE_P)
2228 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2229 {
2230 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT))
2231 | ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT);
2232 NOREF(GCPtrCurPage);
2233 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2234 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2235 GCPtrCurPage, pPteSrc->u & X86_PTE_P,
2236 !!(pPteSrc->u & PdeSrc.u & X86_PTE_RW),
2237 !!(pPteSrc->u & PdeSrc.u & X86_PTE_US),
2238 (uint64_t)pPteSrc->u,
2239 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2240 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2241 }
2242 }
2243 }
2244 else
2245# endif /* PGM_SYNC_N_PAGES */
2246 {
2247 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2248 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2249 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2250 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2251 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2252 GCPtrPage, PteSrc.u & X86_PTE_P,
2253 !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
2254 !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
2255 (uint64_t)PteSrc.u,
2256 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2257 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2258 }
2259 }
2260 else /* MMIO or invalid page: emulated in #PF handler. */
2261 {
2262 LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2263 Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2264 }
2265 }
2266 else
2267 {
2268 /*
2269 * 4/2MB page - lazy syncing shadow 4K pages.
2270 * (There are many causes of getting here, it's no longer only CSAM.)
2271 */
2272 /* Calculate the GC physical address of this 4KB shadow page. */
2273 GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2274 /* Find ram range. */
2275 PPGMPAGE pPage;
2276 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2277 if (RT_SUCCESS(rc))
2278 {
2279 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2280
2281# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2282 /* Try to make the page writable if necessary. */
2283 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2284 && ( PGM_PAGE_IS_ZERO(pPage)
2285 || ( (PdeSrc.u & X86_PDE_RW)
2286 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2287# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2288 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2289# endif
2290# ifdef VBOX_WITH_PAGE_SHARING
2291 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2292# endif
2293 )
2294 )
2295 )
2296 {
2297 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2298 AssertRC(rc);
2299 }
2300# endif
2301
2302 /*
2303 * Make shadow PTE entry.
2304 */
2305 SHWPTE PteDst;
2306 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2307 SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
2308 else
2309 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2310
2311 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2312 if ( SHW_PTE_IS_P(PteDst)
2313 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2314 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2315
2316 /* Make sure only allocated pages are mapped writable. */
2317 if ( SHW_PTE_IS_P_RW(PteDst)
2318 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2319 {
2320 /* Still applies to shared pages. */
2321 Assert(!PGM_PAGE_IS_ZERO(pPage));
2322 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2323 Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2324 }
2325
2326 SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2327
2328 /*
2329 * If the page is not flagged as dirty and is writable, then make it read-only
2330 * at PD level, so we can set the dirty bit when the page is modified.
2331 *
2332 * ASSUMES that page access handlers are implemented on page table entry level.
2333 * Thus we will first catch the dirty access and set PDE.D and restart. If
2334 * there is an access handler, we'll trap again and let it work on the problem.
2335 */
2336 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2337 * As for invlpg, it simply frees the whole shadow PT.
2338 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2339 if ((PdeSrc.u & (X86_PDE4M_D | X86_PDE_RW)) == X86_PDE_RW)
2340 {
2341 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2342 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2343 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2344 }
2345 else
2346 {
2347 PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY | X86_PDE_RW);
2348 PdeDst.u |= PdeSrc.u & X86_PDE_RW;
2349 }
2350 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2351 Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2352 GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US),
2353 (uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2354 }
2355 else
2356 {
2357 LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2358 /** @todo must wipe the shadow page table entry in this
2359 * case. */
2360 }
2361 }
2362 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2363 return VINF_SUCCESS;
2364 }
2365
2366 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs));
2367 }
2368 else if (fPdeValid)
2369 {
2370 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2371 Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2372 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2373 }
2374 else
2375 {
2376/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2377 Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2378 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2379 }
2380
2381 /*
2382 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2383 * Yea, I'm lazy.
2384 */
2385 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2386 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
2387
2388 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2389 PGM_INVL_VCPU_TLBS(pVCpu);
2390 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2391
2392
2393# elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2394 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2395 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
2396 NOREF(PdeSrc);
2397
2398# ifdef PGM_SYNC_N_PAGES
2399 /*
2400 * Get the shadow PDE, find the shadow page table in the pool.
2401 */
2402# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2403 X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2404
2405# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2406 X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2407
2408# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2409 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2410 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2411 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2412 X86PDEPAE PdeDst;
2413 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2414
2415 int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2416 AssertRCSuccessReturn(rc, rc);
2417 Assert(pPDDst && pPdptDst);
2418 PdeDst = pPDDst->a[iPDDst];
2419
2420# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2421 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2422 PEPTPD pPDDst;
2423 EPTPDE PdeDst;
2424
2425 int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2426 if (rc != VINF_SUCCESS)
2427 {
2428 AssertRC(rc);
2429 return rc;
2430 }
2431 Assert(pPDDst);
2432 PdeDst = pPDDst->a[iPDDst];
2433# endif
2434 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2435 if (!SHW_PDE_IS_P(PdeDst))
2436 {
2437 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2438 Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2439 return VINF_SUCCESS; /* force the instruction to be executed again. */
2440 }
2441
2442 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2443 if (SHW_PDE_IS_BIG(PdeDst))
2444 {
2445 Assert(pVM->pgm.s.fNestedPaging);
2446 Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2447 return VINF_SUCCESS;
2448 }
2449
2450 /* Mask away the page offset. */
2451 GCPtrPage &= ~((RTGCPTR)0xfff);
2452
2453 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2454 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2455
2456 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2457 if ( cPages > 1
2458 && !(uErr & X86_TRAP_PF_P)
2459 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2460 {
2461 /*
2462 * This code path is currently only taken when the caller is PGMTrap0eHandler
2463 * for non-present pages!
2464 *
2465 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2466 * deal with locality.
2467 */
2468 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2469 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2470 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2471 iPTDst = 0;
2472 else
2473 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2474 for (; iPTDst < iPTDstEnd; iPTDst++)
2475 {
2476 if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2477 {
2478 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2479 | (iPTDst << GUEST_PAGE_SHIFT));
2480
2481 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2482 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2483 GCPtrCurPage,
2484 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2485 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2486
2487 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2488 break;
2489 }
2490 else
2491 Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n",
2492 (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2493 }
2494 }
2495 else
2496# endif /* PGM_SYNC_N_PAGES */
2497 {
2498 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2499 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2500 | (iPTDst << GUEST_PAGE_SHIFT));
2501
2502 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2503
2504 Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2505 GCPtrPage,
2506 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2507 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2508 }
2509 return VINF_SUCCESS;
2510
2511# else
2512 NOREF(PdeSrc);
2513 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2514 return VERR_PGM_NOT_USED_IN_MODE;
2515# endif
2516}
2517
2518#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
2519
2520#if !defined(IN_RING3) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
2521
2522/**
2523 * Sync a shadow page for a nested-guest page.
2524 *
2525 * @param pVCpu The cross context virtual CPU structure.
2526 * @param pPte The shadow page table entry.
2527 * @param GCPhysPage The guest-physical address of the page.
2528 * @param pShwPage The shadow page of the page table.
2529 * @param iPte The index of the page table entry.
2530 * @param pGstWalkAll The guest page table walk result.
2531 *
2532 * @note Not to be used for 2/4MB pages!
2533 */
2534static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
2535 unsigned iPte, PCPGMPTWALKGST pGstWalkAll)
2536{
2537 /*
2538 * Do not make assumptions about anything other than the final PTE entry in the
2539 * guest page table walk result. For instance, while mapping 2M PDEs as 4K pages,
2540 * the PDE might still be having its leaf bit set.
2541 *
2542 * In the future, we could consider introducing a generic SLAT macro like PSLATPTE
2543 * and using that instead of passing the full SLAT translation result.
2544 */
2545 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2546 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
2547 Assert(!pShwPage->fDirty);
2548 Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
2549 AssertMsg((pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK) == GCPhysPage,
2550 ("PTE address mismatch. GCPhysPage=%RGp Pte=%RX64\n", GCPhysPage, pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK));
2551
2552 /*
2553 * Find the ram range.
2554 */
2555 PPGMPAGE pPage;
2556 int rc = pgmPhysGetPageEx(pVCpu->CTX_SUFF(pVM), GCPhysPage, &pPage);
2557 if (RT_SUCCESS(rc))
2558 { /* likely */ }
2559 else
2560 {
2561 /*
2562 * This is a RAM hole/invalid/reserved address (not MMIO).
2563 * Nested Microsoft Hyper-V maps addresses like 0xf0220000 as RW WB memory.
2564 * Shadow a not-present page similar to MMIO, see @bugref{10318#c7}.
2565 */
2566 Assert(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS);
2567 if (SHW_PTE_IS_P(*pPte))
2568 {
2569 Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
2570 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2571 }
2572 Log7Func(("RAM hole/reserved %RGp -> ShwPte=0\n", GCPhysPage));
2573 SHW_PTE_ATOMIC_SET(*pPte, 0);
2574 return;
2575 }
2576
2577 Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2578
2579 /*
2580 * Make page table entry.
2581 */
2582 SHWPTE Pte;
2583 uint64_t const fGstShwPteFlags = pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptShadowedPteMask;
2584 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2585 {
2586# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2587 /* Page wasn't allocated, write protect it. */
2588 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2589 && ( PGM_PAGE_IS_ZERO(pPage)
2590 || ( (pGstWalkAll->u.Ept.Pte.u & EPT_E_WRITE)
2591 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2592# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2593 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2594# endif
2595# ifdef VBOX_WITH_PAGE_SHARING
2596 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2597# endif
2598 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_BALLOONED
2599 )
2600 )
2601 )
2602 {
2603 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE);
2604 Log7Func(("zero page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2605 }
2606 else
2607# endif
2608 {
2609 /** @todo access bit. */
2610 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
2611 Log7Func(("regular page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2612 }
2613 }
2614 else if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2615 {
2616 /** @todo access bit. */
2617 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE);
2618 Log7Func(("monitored page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2619 }
2620 else
2621 {
2622 /** @todo Do MMIO optimizations here too? */
2623 Log7Func(("mmio/all page (%R[pgmpage]) at %RGp -> 0\n", pPage, GCPhysPage));
2624 Pte.u = 0;
2625 }
2626
2627 /* Make sure only allocated pages are mapped writable. */
2628 Assert(!SHW_PTE_IS_P_RW(Pte) || PGM_PAGE_IS_ALLOCATED(pPage));
2629
2630 /*
2631 * Keep user track up to date.
2632 */
2633 if (SHW_PTE_IS_P(Pte))
2634 {
2635 if (!SHW_PTE_IS_P(*pPte))
2636 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2637 else if (SHW_PTE_GET_HCPHYS(*pPte) != SHW_PTE_GET_HCPHYS(Pte))
2638 {
2639 Log2(("SyncPageWorker: deref! *pPte=%RX64 Pte=%RX64\n", SHW_PTE_LOG64(*pPte), SHW_PTE_LOG64(Pte)));
2640 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2641 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2642 }
2643 }
2644 else if (SHW_PTE_IS_P(*pPte))
2645 {
2646 Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
2647 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2648 }
2649
2650 /*
2651 * Commit the entry.
2652 */
2653 SHW_PTE_ATOMIC_SET2(*pPte, Pte);
2654 return;
2655}
2656
2657
2658/**
2659 * Syncs a nested-guest page.
2660 *
2661 * There are no conflicts at this point, neither is there any need for
2662 * page table allocations.
2663 *
2664 * @returns VBox status code.
2665 * @param pVCpu The cross context virtual CPU structure.
2666 * @param GCPhysNestedPage The nested-guest physical address of the page being
2667 * synced.
2668 * @param GCPhysPage The guest-physical address of the page being synced.
2669 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2670 * @param uErr The page fault error (X86_TRAP_PF_XXX).
2671 * @param pGstWalkAll The guest page table walk result.
2672 */
2673static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
2674 uint32_t uErr, PPGMPTWALKGST pGstWalkAll)
2675{
2676 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2677 Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2678 Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2679
2680 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2681 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2682 Log7Func(("GCPhysNestedPage=%RGv GCPhysPage=%RGp cPages=%u uErr=%#x\n", GCPhysNestedPage, GCPhysPage, cPages, uErr));
2683 RT_NOREF_PV(uErr); RT_NOREF_PV(cPages);
2684
2685 PGM_LOCK_ASSERT_OWNER(pVM);
2686
2687 /*
2688 * Get the shadow PDE, find the shadow page table in the pool.
2689 */
2690 unsigned const iPde = ((GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK);
2691 PEPTPD pPd;
2692 int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL, &pPd, pGstWalkAll);
2693 if (RT_SUCCESS(rc))
2694 { /* likely */ }
2695 else
2696 {
2697 Log(("Failed to fetch EPT PD for %RGp (%RGp) rc=%Rrc\n", GCPhysNestedPage, GCPhysPage, rc));
2698 return rc;
2699 }
2700 Assert(pPd);
2701 EPTPDE Pde = pPd->a[iPde];
2702
2703 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2704 if (!SHW_PDE_IS_P(Pde))
2705 {
2706 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)Pde.u));
2707 Log7Func(("CPU%d: SyncPage: Pde at %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2708 return VINF_SUCCESS; /* force the instruction to be executed again. */
2709 }
2710
2711 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2712 if (SHW_PDE_IS_BIG(Pde))
2713 {
2714 Log7Func(("CPU%d: SyncPage: %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2715 return VINF_SUCCESS;
2716 }
2717
2718 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, Pde.u & EPT_PDE_PG_MASK);
2719 PEPTPT pPt = (PEPTPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2720
2721 /*
2722 * If we've shadowed a guest EPT PDE that maps a 2M page using a 4K table,
2723 * then sync the 4K sub-page in the 2M range.
2724 */
2725 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2726 {
2727 Assert(!SHW_PDE_IS_BIG(Pde));
2728
2729 Assert(pGstWalkAll->u.Ept.Pte.u == 0);
2730 Assert((Pde.u & EPT_PRESENT_MASK) == (pGstWalkAll->u.Ept.Pde.u & EPT_PRESENT_MASK));
2731 Assert(pShwPage->GCPhys == (pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK));
2732
2733#if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
2734 PPGMPAGE pPage;
2735 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); AssertRC(rc);
2736 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
2737 Assert(pShwPage->enmKind == PGMPOOLKIND_EPT_PT_FOR_EPT_2MB);
2738#endif
2739 uint64_t const fGstPteFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask & ~EPT_E_LEAF;
2740 pGstWalkAll->u.Ept.Pte.u = GCPhysPage | fGstPteFlags;
2741
2742 unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2743 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
2744 Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2745
2746 /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
2747 pGstWalkAll->u.Ept.Pte.u = 0;
2748 return VINF_SUCCESS;
2749 }
2750
2751 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2752# ifdef PGM_SYNC_N_PAGES
2753 if ( cPages > 1
2754 && !(uErr & X86_TRAP_PF_P)
2755 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2756 {
2757 /*
2758 * This code path is currently only taken for non-present pages!
2759 *
2760 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2761 * deal with locality.
2762 */
2763 unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2764 unsigned const iPteEnd = RT_MIN(iPte + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPt->a));
2765 if (iPte < PGM_SYNC_NR_PAGES / 2)
2766 iPte = 0;
2767 else
2768 iPte -= PGM_SYNC_NR_PAGES / 2;
2769 for (; iPte < iPteEnd; iPte++)
2770 {
2771 if (!SHW_PTE_IS_P(pPt->a[iPte]))
2772 {
2773 PGMPTWALKGST GstWalkPt;
2774 PGMPTWALK WalkPt;
2775 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2776 GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
2777 rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt,
2778 &GstWalkPt);
2779 if (RT_SUCCESS(rc))
2780 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], WalkPt.GCPhys, pShwPage, iPte, &GstWalkPt);
2781 else
2782 {
2783 /*
2784 * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
2785 * Ensure the shadow tables entry is not-present.
2786 */
2787 /** @todo Potential room for optimization (explained in NestedSyncPT). */
2788 AssertMsg(!pPt->a[iPte].u, ("%RX64\n", pPt->a[iPte].u));
2789 }
2790 Log7Func(("Many: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2791 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2792 break;
2793 }
2794 else
2795 {
2796# ifdef VBOX_STRICT
2797 /* Paranoia - Verify address of the page is what it should be. */
2798 PGMPTWALKGST GstWalkPt;
2799 PGMPTWALK WalkPt;
2800 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2801 GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
2802 rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt, &GstWalkPt);
2803 AssertRC(rc);
2804 PPGMPAGE pPage;
2805 rc = pgmPhysGetPageEx(pVM, WalkPt.GCPhys, &pPage);
2806 AssertRC(rc);
2807 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]),
2808 ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp HCPhys=%RHp Shw=%RHp\n",
2809 GCPhysNestedPage, WalkPt.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
2810# endif
2811 Log7Func(("Many3: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2812 }
2813 }
2814 }
2815 else
2816# endif /* PGM_SYNC_N_PAGES */
2817 {
2818 unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2819 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
2820 Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2821 }
2822
2823 return VINF_SUCCESS;
2824}
2825
2826
2827/**
2828 * Sync a shadow page table for a nested-guest page table.
2829 *
2830 * The shadow page table is not present in the shadow PDE.
2831 *
2832 * Handles mapping conflicts.
2833 *
2834 * A precondition for this method is that the shadow PDE is not present. The
2835 * caller must take the PGM lock before checking this and continue to hold it
2836 * when calling this method.
2837 *
2838 * @returns VBox status code.
2839 * @param pVCpu The cross context virtual CPU structure.
2840 * @param GCPhysNestedPage The nested-guest physical page address of the page
2841 * being synced.
2842 * @param GCPhysPage The guest-physical address of the page being synced.
2843 * @param pGstWalkAll The guest page table walk result.
2844 */
2845static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll)
2846{
2847 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2848 Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2849 Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2850
2851 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2852 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2853
2854 Log7Func(("GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
2855
2856 PGM_LOCK_ASSERT_OWNER(pVM);
2857 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2858
2859 PEPTPD pPd;
2860 PEPTPDPT pPdpt;
2861 unsigned const iPde = (GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK;
2862 int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, &pPdpt, &pPd, pGstWalkAll);
2863 if (RT_SUCCESS(rc))
2864 { /* likely */ }
2865 else
2866 {
2867 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2868 AssertRC(rc);
2869 return rc;
2870 }
2871 Assert(pPd);
2872 PSHWPDE pPde = &pPd->a[iPde];
2873
2874 unsigned const iPdpt = (GCPhysNestedPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2875 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdpt->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2876 Assert(pShwPde->enmKind == PGMPOOLKIND_EPT_PD_FOR_EPT_PD);
2877
2878 SHWPDE Pde = *pPde;
2879 Assert(!SHW_PDE_IS_P(Pde)); /* We're only supposed to call SyncPT on PDE!P and conflicts. */
2880
2881# ifdef PGM_WITH_LARGE_PAGES
2882 if (BTH_IS_NP_ACTIVE(pVM))
2883 {
2884 /*
2885 * Check if the guest is mapping a 2M page here.
2886 */
2887 PPGMPAGE pPage;
2888 rc = pgmPhysGetPageEx(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK, &pPage);
2889 AssertRCReturn(rc, rc);
2890 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2891 {
2892 /* A20 is always enabled in VMX root and non-root operation. */
2893 Assert(PGM_A20_IS_ENABLED(pVCpu));
2894
2895 RTHCPHYS HCPhys = NIL_RTHCPHYS;
2896 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
2897 {
2898 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
2899 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2900 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2901 }
2902 else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2903 {
2904 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
2905 rc = pgmPhysRecheckLargePage(pVM, GCPhysPage, pPage);
2906 if (RT_SUCCESS(rc))
2907 {
2908 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2909 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2910 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2911 }
2912 }
2913 else if (PGMIsUsingLargePages(pVM))
2914 {
2915 rc = pgmPhysAllocLargePage(pVM, GCPhysPage);
2916 if (RT_SUCCESS(rc))
2917 {
2918 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2919 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2920 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2921 }
2922 }
2923
2924 /*
2925 * If we have a 2M large page, we can map the guest's 2M large page right away.
2926 */
2927 uint64_t const fShwBigPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask;
2928 if (HCPhys != NIL_RTHCPHYS)
2929 {
2930 Pde.u = HCPhys | fShwBigPdeFlags;
2931 Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzBigPdeMask));
2932 Assert(Pde.u & EPT_E_LEAF);
2933 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
2934
2935 /* Add a reference to the first page only. */
2936 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPde);
2937
2938 Assert(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED);
2939
2940 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2941 Log7Func(("GstPde=%RGp ShwPde=%RX64 [2M]\n", pGstWalkAll->u.Ept.Pde.u, Pde.u));
2942 return VINF_SUCCESS;
2943 }
2944
2945 /*
2946 * We didn't get a perfect 2M fit. Split the 2M page into 4K pages.
2947 * The page ought not to be marked as a big (2M) page at this point.
2948 */
2949 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
2950
2951 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2952 PGMPOOLACCESS enmAccess;
2953 {
2954 Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_USER_EXECUTE)); /* Mode-based execute control for EPT not supported. */
2955 bool const fNoExecute = !(pGstWalkAll->u.Ept.Pde.u & EPT_E_EXECUTE);
2956 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_WRITE)
2957 enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2958 else
2959 enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2960 }
2961
2962 /*
2963 * Allocate & map a 4K shadow table to cover the 2M guest page.
2964 */
2965 PPGMPOOLPAGE pShwPage;
2966 RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK;
2967 rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_2MB, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2968 pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
2969 if ( rc == VINF_SUCCESS
2970 || rc == VINF_PGM_CACHED_PAGE)
2971 { /* likely */ }
2972 else
2973 {
2974 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2975 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2976 }
2977
2978 PSHWPT pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2979 Assert(pPt);
2980 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
2981 if (rc == VINF_SUCCESS)
2982 {
2983 /* The 4K PTEs shall inherit the flags of the 2M PDE page sans the leaf bit. */
2984 uint64_t const fShwPteFlags = fShwBigPdeFlags & ~EPT_E_LEAF;
2985
2986 /* Sync each 4K pages in the 2M range. */
2987 for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++)
2988 {
2989 RTGCPHYS const GCPhysSubPage = GCPhysPt | (iPte << GUEST_PAGE_SHIFT);
2990 pGstWalkAll->u.Ept.Pte.u = GCPhysSubPage | fShwPteFlags;
2991 Assert(!(pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptMbzPteMask));
2992 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysSubPage, pShwPage, iPte, pGstWalkAll);
2993 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [2M->4K]\n", pGstWalkAll->u.Ept.Pte, pPt->a[iPte].u, iPte));
2994 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2995 break;
2996 }
2997
2998 /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
2999 pGstWalkAll->u.Ept.Pte.u = 0;
3000 }
3001 else
3002 {
3003 Assert(rc == VINF_PGM_CACHED_PAGE);
3004# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3005 /* Paranoia - Verify address of each of the subpages are what they should be. */
3006 RTGCPHYS GCPhysSubPage = GCPhysPt;
3007 for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++, GCPhysSubPage += GUEST_PAGE_SIZE)
3008 {
3009 PPGMPAGE pSubPage;
3010 rc = pgmPhysGetPageEx(pVM, GCPhysSubPage, &pSubPage);
3011 AssertRC(rc);
3012 AssertMsg( PGM_PAGE_GET_HCPHYS(pSubPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte])
3013 || !SHW_PTE_IS_P(pPt->a[iPte]),
3014 ("PGM 2M page and shadow PTE conflict. GCPhysSubPage=%RGp Page=%RHp Shw=%RHp\n",
3015 GCPhysSubPage, PGM_PAGE_GET_HCPHYS(pSubPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3016 }
3017# endif
3018 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3019 }
3020
3021 /* Save the new PDE. */
3022 uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3023 Pde.u = pShwPage->Core.Key | fShwPdeFlags;
3024 Assert(!(Pde.u & EPT_E_LEAF));
3025 Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3026 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3027 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3028 Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3029 return rc;
3030 }
3031 }
3032# endif /* PGM_WITH_LARGE_PAGES */
3033
3034 /*
3035 * Allocate & map the shadow page table.
3036 */
3037 PSHWPT pPt;
3038 PPGMPOOLPAGE pShwPage;
3039
3040 RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE_PG_MASK;
3041 rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_PT, PGMPOOLACCESS_DONTCARE,
3042 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
3043 if ( rc == VINF_SUCCESS
3044 || rc == VINF_PGM_CACHED_PAGE)
3045 { /* likely */ }
3046 else
3047 {
3048 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3049 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3050 }
3051
3052 pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3053 Assert(pPt);
3054 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
3055
3056 if (rc == VINF_SUCCESS)
3057 {
3058 /* Sync the page we've already translated through SLAT. */
3059 const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3060 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
3061 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3062
3063 /* Sync the rest of page table (expensive but might be cheaper than nested-guest VM-exits in hardware). */
3064 for (unsigned iPteCur = 0; iPteCur < RT_ELEMENTS(pPt->a); iPteCur++)
3065 {
3066 if (iPteCur != iPte)
3067 {
3068 PGMPTWALKGST GstWalkPt;
3069 PGMPTWALK WalkPt;
3070 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
3071 GCPhysNestedPage |= (iPteCur << GUEST_PAGE_SHIFT);
3072 int const rc2 = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/,
3073 &WalkPt, &GstWalkPt);
3074 if (RT_SUCCESS(rc2))
3075 {
3076 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPteCur], WalkPt.GCPhys, pShwPage, iPteCur, &GstWalkPt);
3077 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", GstWalkPt.u.Ept.Pte.u, pPt->a[iPteCur].u, iPteCur));
3078 }
3079 else
3080 {
3081 /*
3082 * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
3083 * Ensure the shadow tables entry is not-present.
3084 */
3085 /** @todo We currently don't configure these to cause EPT misconfigs but rather trap
3086 * them using EPT violations and walk the guest EPT tables to determine
3087 * whether they are EPT misconfigs VM-exits for the nested-hypervisor. We
3088 * could optimize this by using a specific combination of reserved bits
3089 * which we could immediately identify as EPT misconfigs of the
3090 * nested-hypervisor without having to walk its EPT tables. However, tracking
3091 * non-present entries might be tricky...
3092 */
3093 AssertMsg(!pPt->a[iPteCur].u, ("%RX64\n", pPt->a[iPteCur].u));
3094 }
3095 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3096 break;
3097 }
3098 }
3099 }
3100 else
3101 {
3102 Assert(rc == VINF_PGM_CACHED_PAGE);
3103# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3104 /* Paranoia - Verify address of the page is what it should be. */
3105 PPGMPAGE pPage;
3106 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
3107 AssertRC(rc);
3108 const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3109 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]) || !SHW_PTE_IS_P(pPt->a[iPte]),
3110 ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp Page=%RHp Shw=%RHp\n",
3111 GCPhysNestedPage, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3112 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [cache]\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3113# endif
3114 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3115 }
3116
3117 /* Save the new PDE. */
3118 uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3119 Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF));
3120 Assert(!(pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3121 Pde.u = pShwPage->Core.Key | fShwPdeFlags;
3122 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3123 Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3124
3125 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3126 return rc;
3127}
3128
3129#endif /* !IN_RING3 && VBOX_WITH_NESTED_HWVIRT_VMX_EPT && PGM_SHW_TYPE == PGM_TYPE_EPT*/
3130#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
3131
3132/**
3133 * Handle dirty bit tracking faults.
3134 *
3135 * @returns VBox status code.
3136 * @param pVCpu The cross context virtual CPU structure.
3137 * @param uErr Page fault error code.
3138 * @param pPdeSrc Guest page directory entry.
3139 * @param pPdeDst Shadow page directory entry.
3140 * @param GCPtrPage Guest context page address.
3141 */
3142static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
3143 RTGCPTR GCPtrPage)
3144{
3145 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3146 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3147 NOREF(uErr);
3148
3149 PGM_LOCK_ASSERT_OWNER(pVM);
3150
3151 /*
3152 * Handle big page.
3153 */
3154 if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
3155 {
3156 if ((pPdeDst->u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3157 {
3158 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3159 Assert(pPdeSrc->u & X86_PDE_RW);
3160
3161 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
3162 * fault again and take this path to only invalidate the entry (see below). */
3163 SHWPDE PdeDst = *pPdeDst;
3164 PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY;
3165 PdeDst.u |= X86_PDE_RW | X86_PDE_A;
3166 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3167 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
3168 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3169 }
3170
3171# ifdef IN_RING0
3172 /* Check for stale TLB entry; only applies to the SMP guest case. */
3173 if ( pVM->cCpus > 1
3174 && (pPdeDst->u & (X86_PDE_P | X86_PDE_RW | X86_PDE_A)) == (X86_PDE_P | X86_PDE_RW | X86_PDE_A))
3175 {
3176 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3177 if (pShwPage)
3178 {
3179 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3180 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3181 if (SHW_PTE_IS_P_RW(*pPteDst))
3182 {
3183 /* Stale TLB entry. */
3184 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3185 PGM_INVL_PG(pVCpu, GCPtrPage);
3186 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3187 }
3188 }
3189 }
3190# endif /* IN_RING0 */
3191 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3192 }
3193
3194 /*
3195 * Map the guest page table.
3196 */
3197 PGSTPT pPTSrc;
3198 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
3199 AssertRCReturn(rc, rc);
3200
3201 if (SHW_PDE_IS_P(*pPdeDst))
3202 {
3203 GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
3204 const GSTPTE PteSrc = *pPteSrc;
3205
3206 /*
3207 * Map shadow page table.
3208 */
3209 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3210 if (pShwPage)
3211 {
3212 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3213 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3214 if (SHW_PTE_IS_P(*pPteDst)) /** @todo Optimize accessed bit emulation? */
3215 {
3216 if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
3217 {
3218 PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
3219 SHWPTE PteDst = *pPteDst;
3220
3221 LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
3222 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3223
3224 Assert(PteSrc.u & X86_PTE_RW);
3225
3226 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB
3227 * entry will not harm; write access will simply fault again and
3228 * take this path to only invalidate the entry.
3229 */
3230 if (RT_LIKELY(pPage))
3231 {
3232 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3233 {
3234 //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
3235 Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
3236 /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
3237 SHW_PTE_SET_RO(PteDst);
3238 }
3239 else
3240 {
3241 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
3242 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
3243 {
3244 rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
3245 AssertRC(rc);
3246 }
3247 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
3248 SHW_PTE_SET_RW(PteDst);
3249 else
3250 {
3251 /* Still applies to shared pages. */
3252 Assert(!PGM_PAGE_IS_ZERO(pPage));
3253 SHW_PTE_SET_RO(PteDst);
3254 }
3255 }
3256 }
3257 else
3258 SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
3259
3260 SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
3261 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
3262 PGM_INVL_PG(pVCpu, GCPtrPage);
3263 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3264 }
3265
3266# ifdef IN_RING0
3267 /* Check for stale TLB entry; only applies to the SMP guest case. */
3268 if ( pVM->cCpus > 1
3269 && SHW_PTE_IS_RW(*pPteDst)
3270 && SHW_PTE_IS_A(*pPteDst))
3271 {
3272 /* Stale TLB entry. */
3273 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3274 PGM_INVL_PG(pVCpu, GCPtrPage);
3275 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3276 }
3277# endif
3278 }
3279 }
3280 else
3281 AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
3282 }
3283
3284 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3285}
3286
3287#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
3288
3289/**
3290 * Sync a shadow page table.
3291 *
3292 * The shadow page table is not present in the shadow PDE.
3293 *
3294 * Handles mapping conflicts.
3295 *
3296 * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
3297 * conflict), and Trap0eHandler.
3298 *
3299 * A precondition for this method is that the shadow PDE is not present. The
3300 * caller must take the PGM lock before checking this and continue to hold it
3301 * when calling this method.
3302 *
3303 * @returns VBox status code.
3304 * @param pVCpu The cross context virtual CPU structure.
3305 * @param iPDSrc Page directory index.
3306 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
3307 * Assume this is a temporary mapping.
3308 * @param GCPtrPage GC Pointer of the page that caused the fault
3309 */
3310static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
3311{
3312 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3313 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3314
3315#if 0 /* rarely useful; leave for debugging. */
3316 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
3317#endif
3318 LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
3319
3320 PGM_LOCK_ASSERT_OWNER(pVM);
3321
3322#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3323 || PGM_GST_TYPE == PGM_TYPE_PAE \
3324 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
3325 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3326 && PGM_SHW_TYPE != PGM_TYPE_NONE
3327 int rc = VINF_SUCCESS;
3328
3329 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3330
3331 /*
3332 * Some input validation first.
3333 */
3334 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
3335
3336 /*
3337 * Get the relevant shadow PDE entry.
3338 */
3339# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3340 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
3341 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3342
3343 /* Fetch the pgm pool shadow descriptor. */
3344 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3345 Assert(pShwPde);
3346
3347# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3348 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3349 PPGMPOOLPAGE pShwPde = NULL;
3350 PX86PDPAE pPDDst;
3351 PSHWPDE pPdeDst;
3352
3353 /* Fetch the pgm pool shadow descriptor. */
3354 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3355 AssertRCSuccessReturn(rc, rc);
3356 Assert(pShwPde);
3357
3358 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3359 pPdeDst = &pPDDst->a[iPDDst];
3360
3361# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3362 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3363 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3364 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3365 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
3366 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3367 AssertRCSuccessReturn(rc, rc);
3368 Assert(pPDDst);
3369 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3370
3371# endif
3372 SHWPDE PdeDst = *pPdeDst;
3373
3374# if PGM_GST_TYPE == PGM_TYPE_AMD64
3375 /* Fetch the pgm pool shadow descriptor. */
3376 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3377 Assert(pShwPde);
3378# endif
3379
3380 Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/
3381
3382 /*
3383 * Sync the page directory entry.
3384 */
3385 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3386 const bool fPageTable = !(PdeSrc.u & X86_PDE_PS) || !GST_IS_PSE_ACTIVE(pVCpu);
3387 if ( (PdeSrc.u & X86_PDE_P)
3388 && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
3389 {
3390 /*
3391 * Allocate & map the page table.
3392 */
3393 PSHWPT pPTDst;
3394 PPGMPOOLPAGE pShwPage;
3395 RTGCPHYS GCPhys;
3396 if (fPageTable)
3397 {
3398 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
3399# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3400 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3401 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
3402# endif
3403 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
3404 pShwPde->idx, iPDDst, false /*fLockPage*/,
3405 &pShwPage);
3406 }
3407 else
3408 {
3409 PGMPOOLACCESS enmAccess;
3410# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
3411 const bool fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu);
3412# else
3413 const bool fNoExecute = false;
3414# endif
3415
3416 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3417# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3418 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
3419 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
3420# endif
3421 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
3422 if (PdeSrc.u & X86_PDE_US)
3423 {
3424 if (PdeSrc.u & X86_PDE_RW)
3425 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
3426 else
3427 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
3428 }
3429 else
3430 {
3431 if (PdeSrc.u & X86_PDE_RW)
3432 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
3433 else
3434 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
3435 }
3436 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
3437 pShwPde->idx, iPDDst, false /*fLockPage*/,
3438 &pShwPage);
3439 }
3440 if (rc == VINF_SUCCESS)
3441 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3442 else if (rc == VINF_PGM_CACHED_PAGE)
3443 {
3444 /*
3445 * The PT was cached, just hook it up.
3446 */
3447 if (fPageTable)
3448 PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3449 else
3450 {
3451 PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3452 /* (see explanation and assumptions further down.) */
3453 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
3454 {
3455 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3456 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
3457 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3458 }
3459 }
3460 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3461 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3462 return VINF_SUCCESS;
3463 }
3464 else
3465 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3466 /** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
3467 * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
3468 * irrelevant at this point. */
3469 PdeDst.u &= X86_PDE_AVL_MASK;
3470 PdeDst.u |= pShwPage->Core.Key;
3471
3472 /*
3473 * Page directory has been accessed (this is a fault situation, remember).
3474 */
3475 /** @todo
3476 * Well, when the caller is PrefetchPage or InvalidatePage is isn't a
3477 * fault situation. What's more, the Trap0eHandler has already set the
3478 * accessed bit. So, it's actually just VerifyAccessSyncPage which
3479 * might need setting the accessed flag.
3480 *
3481 * The best idea is to leave this change to the caller and add an
3482 * assertion that it's set already. */
3483 pPDSrc->a[iPDSrc].u |= X86_PDE_A;
3484 if (fPageTable)
3485 {
3486 /*
3487 * Page table - 4KB.
3488 *
3489 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
3490 */
3491 Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
3492 GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u));
3493 PGSTPT pPTSrc;
3494 rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
3495 if (RT_SUCCESS(rc))
3496 {
3497 /*
3498 * Start by syncing the page directory entry so CSAM's TLB trick works.
3499 */
3500 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
3501 | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3502 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3503 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3504
3505 /*
3506 * Directory/page user or supervisor privilege: (same goes for read/write)
3507 *
3508 * Directory Page Combined
3509 * U/S U/S U/S
3510 * 0 0 0
3511 * 0 1 0
3512 * 1 0 0
3513 * 1 1 1
3514 *
3515 * Simple AND operation. Table listed for completeness.
3516 *
3517 */
3518 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K));
3519# ifdef PGM_SYNC_N_PAGES
3520 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3521 unsigned iPTDst = iPTBase;
3522 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
3523 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
3524 iPTDst = 0;
3525 else
3526 iPTDst -= PGM_SYNC_NR_PAGES / 2;
3527# else /* !PGM_SYNC_N_PAGES */
3528 unsigned iPTDst = 0;
3529 const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
3530# endif /* !PGM_SYNC_N_PAGES */
3531 RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
3532 | ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT);
3533# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3534 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3535 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
3536# else
3537 const unsigned offPTSrc = 0;
3538# endif
3539 for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE)
3540 {
3541 const unsigned iPTSrc = iPTDst + offPTSrc;
3542 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
3543 if (PteSrc.u & X86_PTE_P)
3544 {
3545 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
3546 Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
3547 GCPtrCur,
3548 PteSrc.u & X86_PTE_P,
3549 !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
3550 !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
3551 (uint64_t)PteSrc.u,
3552 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
3553 (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
3554 }
3555 /* else: the page table was cleared by the pool */
3556 } /* for PTEs */
3557 }
3558 }
3559 else
3560 {
3561 /*
3562 * Big page - 2/4MB.
3563 *
3564 * We'll walk the ram range list in parallel and optimize lookups.
3565 * We will only sync one shadow page table at a time.
3566 */
3567 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M));
3568
3569 /**
3570 * @todo It might be more efficient to sync only a part of the 4MB
3571 * page (similar to what we do for 4KB PDs).
3572 */
3573
3574 /*
3575 * Start by syncing the page directory entry.
3576 */
3577 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
3578 | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3579
3580 /*
3581 * If the page is not flagged as dirty and is writable, then make it read-only
3582 * at PD level, so we can set the dirty bit when the page is modified.
3583 *
3584 * ASSUMES that page access handlers are implemented on page table entry level.
3585 * Thus we will first catch the dirty access and set PDE.D and restart. If
3586 * there is an access handler, we'll trap again and let it work on the problem.
3587 */
3588 /** @todo move the above stuff to a section in the PGM documentation. */
3589 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
3590 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
3591 {
3592 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3593 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
3594 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3595 }
3596 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3597 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3598
3599 /*
3600 * Fill the shadow page table.
3601 */
3602 /* Get address and flags from the source PDE. */
3603 SHWPTE PteDstBase;
3604 SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
3605
3606 /* Loop thru the entries in the shadow PT. */
3607 const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
3608 Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
3609 GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr,
3610 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
3611 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3612 unsigned iPTDst = 0;
3613 while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3614 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3615 {
3616 if (pRam && GCPhys >= pRam->GCPhys)
3617 {
3618# ifndef PGM_WITH_A20
3619 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
3620# endif
3621 do
3622 {
3623 /* Make shadow PTE. */
3624# ifdef PGM_WITH_A20
3625 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
3626# else
3627 PPGMPAGE pPage = &pRam->aPages[iHCPage];
3628# endif
3629 SHWPTE PteDst;
3630
3631# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3632 /* Try to make the page writable if necessary. */
3633 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3634 && ( PGM_PAGE_IS_ZERO(pPage)
3635 || ( SHW_PTE_IS_RW(PteDstBase)
3636 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3637# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3638 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3639# endif
3640# ifdef VBOX_WITH_PAGE_SHARING
3641 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3642# endif
3643 && !PGM_PAGE_IS_BALLOONED(pPage))
3644 )
3645 )
3646 {
3647 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3648 AssertRCReturn(rc, rc);
3649 if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3650 break;
3651 }
3652# endif
3653
3654 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3655 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, SHW_PTE_GET_U(PteDstBase), &PteDst);
3656 else if (PGM_PAGE_IS_BALLOONED(pPage))
3657 SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3658 else
3659 SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase));
3660
3661 /* Only map writable pages writable. */
3662 if ( SHW_PTE_IS_P_RW(PteDst)
3663 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3664 {
3665 /* Still applies to shared pages. */
3666 Assert(!PGM_PAGE_IS_ZERO(pPage));
3667 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3668 Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
3669 }
3670
3671 if (SHW_PTE_IS_P(PteDst))
3672 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3673
3674 /* commit it (not atomic, new table) */
3675 pPTDst->a[iPTDst] = PteDst;
3676 Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3677 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3678 SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3679
3680 /* advance */
3681 GCPhys += GUEST_PAGE_SIZE;
3682 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3683# ifndef PGM_WITH_A20
3684 iHCPage++;
3685# endif
3686 iPTDst++;
3687 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3688 && GCPhys <= pRam->GCPhysLast);
3689
3690 /* Advance ram range list. */
3691 while (pRam && GCPhys > pRam->GCPhysLast)
3692 pRam = pRam->CTX_SUFF(pNext);
3693 }
3694 else if (pRam)
3695 {
3696 Log(("Invalid pages at %RGp\n", GCPhys));
3697 do
3698 {
3699 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3700 GCPhys += GUEST_PAGE_SIZE;
3701 iPTDst++;
3702 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3703 && GCPhys < pRam->GCPhys);
3704 PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3705 }
3706 else
3707 {
3708 Log(("Invalid pages at %RGp (2)\n", GCPhys));
3709 for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3710 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3711 }
3712 } /* while more PTEs */
3713 } /* 4KB / 4MB */
3714 }
3715 else
3716 AssertRelease(!SHW_PDE_IS_P(PdeDst));
3717
3718 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3719 if (RT_FAILURE(rc))
3720 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3721 return rc;
3722
3723#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
3724 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
3725 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
3726 && PGM_SHW_TYPE != PGM_TYPE_NONE
3727 NOREF(iPDSrc); NOREF(pPDSrc);
3728
3729 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3730
3731 /*
3732 * Validate input a little bit.
3733 */
3734 int rc = VINF_SUCCESS;
3735# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3736 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3737 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3738
3739 /* Fetch the pgm pool shadow descriptor. */
3740 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3741 Assert(pShwPde);
3742
3743# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3744 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3745 PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3746 PX86PDPAE pPDDst;
3747 PSHWPDE pPdeDst;
3748
3749 /* Fetch the pgm pool shadow descriptor. */
3750 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3751 AssertRCSuccessReturn(rc, rc);
3752 Assert(pShwPde);
3753
3754 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3755 pPdeDst = &pPDDst->a[iPDDst];
3756
3757# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3758 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3759 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3760 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3761 PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3762 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3763 AssertRCSuccessReturn(rc, rc);
3764 Assert(pPDDst);
3765 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3766
3767 /* Fetch the pgm pool shadow descriptor. */
3768 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3769 Assert(pShwPde);
3770
3771# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3772 const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3773 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3774 PEPTPD pPDDst;
3775 PEPTPDPT pPdptDst;
3776
3777 rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3778 if (rc != VINF_SUCCESS)
3779 {
3780 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3781 AssertRC(rc);
3782 return rc;
3783 }
3784 Assert(pPDDst);
3785 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3786
3787 /* Fetch the pgm pool shadow descriptor. */
3788 /** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
3789 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3790 Assert(pShwPde);
3791# endif
3792 SHWPDE PdeDst = *pPdeDst;
3793
3794 Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3795
3796# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3797 if (BTH_IS_NP_ACTIVE(pVM))
3798 {
3799 Assert(!VM_IS_NEM_ENABLED(pVM));
3800
3801 /* Check if we allocated a big page before for this 2 MB range. */
3802 PPGMPAGE pPage;
3803 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3804 if (RT_SUCCESS(rc))
3805 {
3806 RTHCPHYS HCPhys = NIL_RTHCPHYS;
3807 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3808 {
3809 if (PGM_A20_IS_ENABLED(pVCpu))
3810 {
3811 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3812 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3813 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3814 }
3815 else
3816 {
3817 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3818 pVM->pgm.s.cLargePagesDisabled++;
3819 }
3820 }
3821 else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3822 && PGM_A20_IS_ENABLED(pVCpu))
3823 {
3824 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3825 rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3826 if (RT_SUCCESS(rc))
3827 {
3828 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3829 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3830 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3831 }
3832 }
3833 else if ( PGMIsUsingLargePages(pVM)
3834 && PGM_A20_IS_ENABLED(pVCpu))
3835 {
3836 rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3837 if (RT_SUCCESS(rc))
3838 {
3839 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3840 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3841 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3842 }
3843 else
3844 LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3845 }
3846
3847 if (HCPhys != NIL_RTHCPHYS)
3848 {
3849# if PGM_SHW_TYPE == PGM_TYPE_EPT
3850 PdeDst.u = HCPhys | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_LEAF | EPT_E_IGNORE_PAT | EPT_E_MEMTYPE_WB
3851 | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
3852# else
3853 PdeDst.u = HCPhys | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PS
3854 | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */;
3855# endif
3856 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3857
3858 Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3859 /* Add a reference to the first page only. */
3860 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3861
3862 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3863 return VINF_SUCCESS;
3864 }
3865 }
3866 }
3867# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
3868
3869 /*
3870 * Allocate & map the page table.
3871 */
3872 PSHWPT pPTDst;
3873 PPGMPOOLPAGE pShwPage;
3874 RTGCPHYS GCPhys;
3875
3876 /* Virtual address = physical address */
3877 GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3878 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3879 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/,
3880 &pShwPage);
3881 if ( rc == VINF_SUCCESS
3882 || rc == VINF_PGM_CACHED_PAGE)
3883 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3884 else
3885 {
3886 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3887 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3888 }
3889
3890 if (rc == VINF_SUCCESS)
3891 {
3892 /* New page table; fully set it up. */
3893 Assert(pPTDst);
3894
3895 /* Mask away the page offset. */
3896 GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
3897
3898 for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3899 {
3900 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3901 | (iPTDst << GUEST_PAGE_SHIFT));
3902
3903 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3904 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3905 GCPtrCurPage,
3906 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3907 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3908
3909 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3910 break;
3911 }
3912 }
3913 else
3914 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3915
3916 /* Save the new PDE. */
3917# if PGM_SHW_TYPE == PGM_TYPE_EPT
3918 PdeDst.u = pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE
3919 | (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
3920# else
3921 PdeDst.u = pShwPage->Core.Key | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A
3922 | (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */);
3923# endif
3924 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3925
3926 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3927 if (RT_FAILURE(rc))
3928 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3929 return rc;
3930
3931#else
3932 NOREF(iPDSrc); NOREF(pPDSrc);
3933 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3934 return VERR_PGM_NOT_USED_IN_MODE;
3935#endif
3936}
3937
3938
3939
3940/**
3941 * Prefetch a page/set of pages.
3942 *
3943 * Typically used to sync commonly used pages before entering raw mode
3944 * after a CR3 reload.
3945 *
3946 * @returns VBox status code.
3947 * @param pVCpu The cross context virtual CPU structure.
3948 * @param GCPtrPage Page to invalidate.
3949 */
3950PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
3951{
3952#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3953 || PGM_GST_TYPE == PGM_TYPE_REAL \
3954 || PGM_GST_TYPE == PGM_TYPE_PROT \
3955 || PGM_GST_TYPE == PGM_TYPE_PAE \
3956 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3957 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3958 && PGM_SHW_TYPE != PGM_TYPE_NONE
3959 /*
3960 * Check that all Guest levels thru the PDE are present, getting the
3961 * PD and PDE in the processes.
3962 */
3963 int rc = VINF_SUCCESS;
3964# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3965# if PGM_GST_TYPE == PGM_TYPE_32BIT
3966 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3967 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3968# elif PGM_GST_TYPE == PGM_TYPE_PAE
3969 unsigned iPDSrc;
3970 X86PDPE PdpeSrc;
3971 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3972 if (!pPDSrc)
3973 return VINF_SUCCESS; /* not present */
3974# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3975 unsigned iPDSrc;
3976 PX86PML4E pPml4eSrc;
3977 X86PDPE PdpeSrc;
3978 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3979 if (!pPDSrc)
3980 return VINF_SUCCESS; /* not present */
3981# endif
3982 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3983# else
3984 PGSTPD pPDSrc = NULL;
3985 const unsigned iPDSrc = 0;
3986 GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
3987# endif
3988
3989 if ((PdeSrc.u & (X86_PDE_P | X86_PDE_A)) == (X86_PDE_P | X86_PDE_A))
3990 {
3991 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3992 PGM_LOCK_VOID(pVM);
3993
3994# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3995 const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3996# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3997 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3998 PX86PDPAE pPDDst;
3999 X86PDEPAE PdeDst;
4000# if PGM_GST_TYPE != PGM_TYPE_PAE
4001 X86PDPE PdpeSrc;
4002
4003 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
4004 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
4005# endif
4006 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
4007 if (rc != VINF_SUCCESS)
4008 {
4009 PGM_UNLOCK(pVM);
4010 AssertRC(rc);
4011 return rc;
4012 }
4013 Assert(pPDDst);
4014 PdeDst = pPDDst->a[iPDDst];
4015
4016# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4017 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4018 PX86PDPAE pPDDst;
4019 X86PDEPAE PdeDst;
4020
4021# if PGM_GST_TYPE == PGM_TYPE_PROT
4022 /* AMD-V nested paging */
4023 X86PML4E Pml4eSrc;
4024 X86PDPE PdpeSrc;
4025 PX86PML4E pPml4eSrc = &Pml4eSrc;
4026
4027 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
4028 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
4029 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
4030# endif
4031
4032 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
4033 if (rc != VINF_SUCCESS)
4034 {
4035 PGM_UNLOCK(pVM);
4036 AssertRC(rc);
4037 return rc;
4038 }
4039 Assert(pPDDst);
4040 PdeDst = pPDDst->a[iPDDst];
4041# endif
4042 if (!(PdeDst.u & X86_PDE_P))
4043 {
4044 /** @todo r=bird: This guy will set the A bit on the PDE,
4045 * probably harmless. */
4046 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
4047 }
4048 else
4049 {
4050 /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
4051 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
4052 * makes no sense to prefetch more than one page.
4053 */
4054 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
4055 if (RT_SUCCESS(rc))
4056 rc = VINF_SUCCESS;
4057 }
4058 PGM_UNLOCK(pVM);
4059 }
4060 return rc;
4061
4062#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4063 NOREF(pVCpu); NOREF(GCPtrPage);
4064 return VINF_SUCCESS; /* ignore */
4065#else
4066 AssertCompile(0);
4067#endif
4068}
4069
4070
4071
4072
4073/**
4074 * Syncs a page during a PGMVerifyAccess() call.
4075 *
4076 * @returns VBox status code (informational included).
4077 * @param pVCpu The cross context virtual CPU structure.
4078 * @param GCPtrPage The address of the page to sync.
4079 * @param fPage The effective guest page flags.
4080 * @param uErr The trap error code.
4081 * @remarks This will normally never be called on invalid guest page
4082 * translation entries.
4083 */
4084PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
4085{
4086 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4087
4088 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
4089 RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
4090
4091 Assert(!pVM->pgm.s.fNestedPaging);
4092#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
4093 || PGM_GST_TYPE == PGM_TYPE_REAL \
4094 || PGM_GST_TYPE == PGM_TYPE_PROT \
4095 || PGM_GST_TYPE == PGM_TYPE_PAE \
4096 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
4097 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
4098 && PGM_SHW_TYPE != PGM_TYPE_NONE
4099
4100 /*
4101 * Get guest PD and index.
4102 */
4103 /** @todo Performance: We've done all this a jiffy ago in the
4104 * PGMGstGetPage call. */
4105# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4106# if PGM_GST_TYPE == PGM_TYPE_32BIT
4107 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
4108 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4109
4110# elif PGM_GST_TYPE == PGM_TYPE_PAE
4111 unsigned iPDSrc = 0;
4112 X86PDPE PdpeSrc;
4113 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
4114 if (RT_UNLIKELY(!pPDSrc))
4115 {
4116 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
4117 return VINF_EM_RAW_GUEST_TRAP;
4118 }
4119
4120# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4121 unsigned iPDSrc = 0; /* shut up gcc */
4122 PX86PML4E pPml4eSrc = NULL; /* ditto */
4123 X86PDPE PdpeSrc;
4124 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
4125 if (RT_UNLIKELY(!pPDSrc))
4126 {
4127 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
4128 return VINF_EM_RAW_GUEST_TRAP;
4129 }
4130# endif
4131
4132# else /* !PGM_WITH_PAGING */
4133 PGSTPD pPDSrc = NULL;
4134 const unsigned iPDSrc = 0;
4135# endif /* !PGM_WITH_PAGING */
4136 int rc = VINF_SUCCESS;
4137
4138 PGM_LOCK_VOID(pVM);
4139
4140 /*
4141 * First check if the shadow pd is present.
4142 */
4143# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4144 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
4145
4146# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4147 PX86PDEPAE pPdeDst;
4148 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4149 PX86PDPAE pPDDst;
4150# if PGM_GST_TYPE != PGM_TYPE_PAE
4151 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
4152 X86PDPE PdpeSrc;
4153 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
4154# endif
4155 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
4156 if (rc != VINF_SUCCESS)
4157 {
4158 PGM_UNLOCK(pVM);
4159 AssertRC(rc);
4160 return rc;
4161 }
4162 Assert(pPDDst);
4163 pPdeDst = &pPDDst->a[iPDDst];
4164
4165# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4166 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4167 PX86PDPAE pPDDst;
4168 PX86PDEPAE pPdeDst;
4169
4170# if PGM_GST_TYPE == PGM_TYPE_PROT
4171 /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
4172 X86PML4E Pml4eSrc;
4173 X86PDPE PdpeSrc;
4174 PX86PML4E pPml4eSrc = &Pml4eSrc;
4175 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
4176 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
4177# endif
4178
4179 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
4180 if (rc != VINF_SUCCESS)
4181 {
4182 PGM_UNLOCK(pVM);
4183 AssertRC(rc);
4184 return rc;
4185 }
4186 Assert(pPDDst);
4187 pPdeDst = &pPDDst->a[iPDDst];
4188# endif
4189
4190 if (!(pPdeDst->u & X86_PDE_P))
4191 {
4192 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
4193 if (rc != VINF_SUCCESS)
4194 {
4195 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
4196 PGM_UNLOCK(pVM);
4197 AssertRC(rc);
4198 return rc;
4199 }
4200 }
4201
4202# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4203 /* Check for dirty bit fault */
4204 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
4205 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
4206 Log(("PGMVerifyAccess: success (dirty)\n"));
4207 else
4208# endif
4209 {
4210# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4211 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
4212# else
4213 GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
4214# endif
4215
4216 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
4217 if (uErr & X86_TRAP_PF_US)
4218 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
4219 else /* supervisor */
4220 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
4221
4222 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
4223 if (RT_SUCCESS(rc))
4224 {
4225 /* Page was successfully synced */
4226 Log2(("PGMVerifyAccess: success (sync)\n"));
4227 rc = VINF_SUCCESS;
4228 }
4229 else
4230 {
4231 Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
4232 rc = VINF_EM_RAW_GUEST_TRAP;
4233 }
4234 }
4235 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
4236 PGM_UNLOCK(pVM);
4237 return rc;
4238
4239#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
4240
4241 AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
4242 return VERR_PGM_NOT_USED_IN_MODE;
4243#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
4244}
4245
4246
4247/**
4248 * Syncs the paging hierarchy starting at CR3.
4249 *
4250 * @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
4251 * informational status codes.
4252 * @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
4253 * the VMM into guest context.
4254 * @param pVCpu The cross context virtual CPU structure.
4255 * @param cr0 Guest context CR0 register.
4256 * @param cr3 Guest context CR3 register. Not subjected to the A20
4257 * mask.
4258 * @param cr4 Guest context CR4 register.
4259 * @param fGlobal Including global page directories or not
4260 */
4261PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
4262{
4263 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4264 NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
4265
4266 LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
4267
4268#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
4269# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4270 PGM_LOCK_VOID(pVM);
4271 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4272 if (pPool->cDirtyPages)
4273 pgmPoolResetDirtyPages(pVM);
4274 PGM_UNLOCK(pVM);
4275# endif
4276#endif /* !NESTED && !EPT */
4277
4278#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4279 /*
4280 * Nested / EPT / None - No work.
4281 */
4282 return VINF_SUCCESS;
4283
4284#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4285 /*
4286 * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
4287 * out the shadow parts when the guest modifies its tables.
4288 */
4289 return VINF_SUCCESS;
4290
4291#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4292
4293 return VINF_SUCCESS;
4294#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4295}
4296
4297
4298
4299
4300#ifdef VBOX_STRICT
4301
4302/**
4303 * Checks that the shadow page table is in sync with the guest one.
4304 *
4305 * @returns The number of errors.
4306 * @param pVCpu The cross context virtual CPU structure.
4307 * @param cr3 Guest context CR3 register.
4308 * @param cr4 Guest context CR4 register.
4309 * @param GCPtr Where to start. Defaults to 0.
4310 * @param cb How much to check. Defaults to everything.
4311 */
4312PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
4313{
4314 NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
4315#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4316 return 0;
4317#else
4318 unsigned cErrors = 0;
4319 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4320 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
4321
4322# if PGM_GST_TYPE == PGM_TYPE_PAE
4323 /** @todo currently broken; crashes below somewhere */
4324 AssertFailed();
4325# endif
4326
4327# if PGM_GST_TYPE == PGM_TYPE_32BIT \
4328 || PGM_GST_TYPE == PGM_TYPE_PAE \
4329 || PGM_GST_TYPE == PGM_TYPE_AMD64
4330
4331 bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
4332 PPGMCPU pPGM = &pVCpu->pgm.s;
4333 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
4334 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
4335# ifndef IN_RING0
4336 RTHCPHYS HCPhys; /* general usage. */
4337# endif
4338 int rc;
4339
4340 /*
4341 * Check that the Guest CR3 and all its mappings are correct.
4342 */
4343 AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
4344 ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
4345 false);
4346# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
4347# if 0
4348# if PGM_GST_TYPE == PGM_TYPE_32BIT
4349 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
4350# else
4351 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
4352# endif
4353 AssertRCReturn(rc, 1);
4354 HCPhys = NIL_RTHCPHYS;
4355 rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
4356 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
4357# endif
4358# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
4359 pgmGstGet32bitPDPtr(pVCpu);
4360 RTGCPHYS GCPhys;
4361 rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
4362 AssertRCReturn(rc, 1);
4363 AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
4364# endif
4365# endif /* !IN_RING0 */
4366
4367 /*
4368 * Get and check the Shadow CR3.
4369 */
4370# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4371 unsigned cPDEs = X86_PG_ENTRIES;
4372 unsigned cIncrement = X86_PG_ENTRIES * GUEST_PAGE_SIZE;
4373# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4374# if PGM_GST_TYPE == PGM_TYPE_32BIT
4375 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
4376# else
4377 unsigned cPDEs = X86_PG_PAE_ENTRIES;
4378# endif
4379 unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4380# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4381 unsigned cPDEs = X86_PG_PAE_ENTRIES;
4382 unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4383# endif
4384 if (cb != ~(RTGCPTR)0)
4385 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
4386
4387/** @todo call the other two PGMAssert*() functions. */
4388
4389# if PGM_GST_TYPE == PGM_TYPE_AMD64
4390 unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4391
4392 for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
4393 {
4394 PPGMPOOLPAGE pShwPdpt = NULL;
4395 PX86PML4E pPml4eSrc;
4396 PX86PML4E pPml4eDst;
4397 RTGCPHYS GCPhysPdptSrc;
4398
4399 pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
4400 pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
4401
4402 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
4403 if (!(pPml4eDst->u & X86_PML4E_P))
4404 {
4405 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4406 continue;
4407 }
4408
4409 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
4410 GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
4411
4412 if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P))
4413 {
4414 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4415 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4416 cErrors++;
4417 continue;
4418 }
4419
4420 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
4421 {
4422 AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
4423 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4424 cErrors++;
4425 continue;
4426 }
4427
4428 if ( (pPml4eDst->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX))
4429 != (pPml4eSrc->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX)))
4430 {
4431 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4432 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4433 cErrors++;
4434 continue;
4435 }
4436# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4437 {
4438# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4439
4440# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
4441 /*
4442 * Check the PDPTEs too.
4443 */
4444 unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
4445
4446 for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
4447 {
4448 unsigned iPDSrc = 0; /* initialized to shut up gcc */
4449 PPGMPOOLPAGE pShwPde = NULL;
4450 PX86PDPE pPdpeDst;
4451 RTGCPHYS GCPhysPdeSrc;
4452 X86PDPE PdpeSrc;
4453 PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
4454# if PGM_GST_TYPE == PGM_TYPE_PAE
4455 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
4456 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
4457# else
4458 PX86PML4E pPml4eSrcIgn;
4459 PX86PDPT pPdptDst;
4460 PX86PDPAE pPDDst;
4461 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
4462
4463 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
4464 if (rc != VINF_SUCCESS)
4465 {
4466 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
4467 GCPtr += 512 * _2M;
4468 continue; /* next PDPTE */
4469 }
4470 Assert(pPDDst);
4471# endif
4472 Assert(iPDSrc == 0);
4473
4474 pPdpeDst = &pPdptDst->a[iPdpt];
4475
4476 if (!(pPdpeDst->u & X86_PDPE_P))
4477 {
4478 GCPtr += 512 * _2M;
4479 continue; /* next PDPTE */
4480 }
4481
4482 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
4483 GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
4484
4485 if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P))
4486 {
4487 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4488 GCPtr += 512 * _2M;
4489 cErrors++;
4490 continue;
4491 }
4492
4493 if (GCPhysPdeSrc != pShwPde->GCPhys)
4494 {
4495# if PGM_GST_TYPE == PGM_TYPE_AMD64
4496 AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4497# else
4498 AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4499# endif
4500 GCPtr += 512 * _2M;
4501 cErrors++;
4502 continue;
4503 }
4504
4505# if PGM_GST_TYPE == PGM_TYPE_AMD64
4506 if ( (pPdpeDst->u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX))
4507 != (PdpeSrc.u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX)))
4508 {
4509 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4510 GCPtr += 512 * _2M;
4511 cErrors++;
4512 continue;
4513 }
4514# endif
4515
4516# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4517 {
4518# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4519# if PGM_GST_TYPE == PGM_TYPE_32BIT
4520 GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4521# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4522 PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
4523# endif
4524# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
4525 /*
4526 * Iterate the shadow page directory.
4527 */
4528 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
4529 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
4530
4531 for (;
4532 iPDDst < cPDEs;
4533 iPDDst++, GCPtr += cIncrement)
4534 {
4535# if PGM_SHW_TYPE == PGM_TYPE_PAE
4536 const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4537# else
4538 const SHWPDE PdeDst = pPDDst->a[iPDDst];
4539# endif
4540 if ( (PdeDst.u & X86_PDE_P)
4541 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) )
4542 {
4543 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4544 PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4545 if (!pPoolPage)
4546 {
4547 AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4548 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4549 cErrors++;
4550 continue;
4551 }
4552 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4553
4554 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
4555 {
4556 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4557 GCPtr, (uint64_t)PdeDst.u));
4558 cErrors++;
4559 }
4560
4561 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
4562 {
4563 AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4564 GCPtr, (uint64_t)PdeDst.u));
4565 cErrors++;
4566 }
4567
4568 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4569 if (!(PdeSrc.u & X86_PDE_P))
4570 {
4571 AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4572 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4573 cErrors++;
4574 continue;
4575 }
4576
4577 if ( !(PdeSrc.u & X86_PDE_PS)
4578 || !fBigPagesSupported)
4579 {
4580 GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4581# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4582 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
4583# endif
4584 }
4585 else
4586 {
4587# if PGM_GST_TYPE == PGM_TYPE_32BIT
4588 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4589 {
4590 AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4591 GCPtr, (uint64_t)PdeSrc.u));
4592 cErrors++;
4593 continue;
4594 }
4595# endif
4596 GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4597# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4598 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4599# endif
4600 }
4601
4602 if ( pPoolPage->enmKind
4603 != (!(PdeSrc.u & X86_PDE_PS) || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4604 {
4605 AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4606 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4607 cErrors++;
4608 }
4609
4610 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4611 if (!pPhysPage)
4612 {
4613 AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4614 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4615 cErrors++;
4616 continue;
4617 }
4618
4619 if (GCPhysGst != pPoolPage->GCPhys)
4620 {
4621 AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4622 GCPhysGst, pPoolPage->GCPhys, GCPtr));
4623 cErrors++;
4624 continue;
4625 }
4626
4627 if ( !(PdeSrc.u & X86_PDE_PS)
4628 || !fBigPagesSupported)
4629 {
4630 /*
4631 * Page Table.
4632 */
4633 const GSTPT *pPTSrc;
4634 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)),
4635 &pPTSrc);
4636 if (RT_FAILURE(rc))
4637 {
4638 AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4639 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4640 cErrors++;
4641 continue;
4642 }
4643 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
4644 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
4645 {
4646 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4647 // (This problem will go away when/if we shadow multiple CR3s.)
4648 AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4649 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4650 cErrors++;
4651 continue;
4652 }
4653 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4654 {
4655 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4656 GCPtr, (uint64_t)PdeDst.u));
4657 cErrors++;
4658 continue;
4659 }
4660
4661 /* iterate the page table. */
4662# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4663 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4664 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4665# else
4666 const unsigned offPTSrc = 0;
4667# endif
4668 for (unsigned iPT = 0, off = 0;
4669 iPT < RT_ELEMENTS(pPTDst->a);
4670 iPT++, off += GUEST_PAGE_SIZE)
4671 {
4672 const SHWPTE PteDst = pPTDst->a[iPT];
4673
4674 /* skip not-present and dirty tracked entries. */
4675 if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4676 continue;
4677 Assert(SHW_PTE_IS_P(PteDst));
4678
4679 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4680 if (!(PteSrc.u & X86_PTE_P))
4681 {
4682# ifdef IN_RING3
4683 PGMAssertHandlerAndFlagsInSync(pVM);
4684 DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE
4685 | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3,
4686 0, 0, UINT64_MAX, 99, NULL);
4687# endif
4688 AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4689 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4690 (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4691 cErrors++;
4692 continue;
4693 }
4694
4695 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
4696# if 1 /** @todo sync accessed bit properly... */
4697 fIgnoreFlags |= X86_PTE_A;
4698# endif
4699
4700 /* match the physical addresses */
4701 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4702 GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4703
4704# ifdef IN_RING3
4705 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4706 if (RT_FAILURE(rc))
4707 {
4708# if 0
4709 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4710 {
4711 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4712 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4713 cErrors++;
4714 continue;
4715 }
4716# endif
4717 }
4718 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4719 {
4720 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4721 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4722 cErrors++;
4723 continue;
4724 }
4725# endif
4726
4727 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4728 if (!pPhysPage)
4729 {
4730# if 0
4731 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4732 {
4733 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4734 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4735 cErrors++;
4736 continue;
4737 }
4738# endif
4739 if (SHW_PTE_IS_RW(PteDst))
4740 {
4741 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4742 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4743 cErrors++;
4744 }
4745 fIgnoreFlags |= X86_PTE_RW;
4746 }
4747 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4748 {
4749 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4750 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4751 cErrors++;
4752 continue;
4753 }
4754
4755 /* flags */
4756 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
4757 {
4758 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4759 {
4760 if (SHW_PTE_IS_RW(PteDst))
4761 {
4762 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4763 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4764 cErrors++;
4765 continue;
4766 }
4767 fIgnoreFlags |= X86_PTE_RW;
4768 }
4769 else
4770 {
4771 if ( SHW_PTE_IS_P(PteDst)
4772# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
4773 && !PGM_PAGE_IS_MMIO(pPhysPage)
4774# endif
4775 )
4776 {
4777 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4778 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4779 cErrors++;
4780 continue;
4781 }
4782 fIgnoreFlags |= X86_PTE_P;
4783 }
4784 }
4785 else
4786 {
4787 if ((PteSrc.u & (X86_PTE_RW | X86_PTE_D)) == X86_PTE_RW)
4788 {
4789 if (SHW_PTE_IS_RW(PteDst))
4790 {
4791 AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4792 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4793 cErrors++;
4794 continue;
4795 }
4796 if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4797 {
4798 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4799 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4800 cErrors++;
4801 continue;
4802 }
4803 if (SHW_PTE_IS_D(PteDst))
4804 {
4805 AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4806 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4807 cErrors++;
4808 }
4809# if 0 /** @todo sync access bit properly... */
4810 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4811 {
4812 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4813 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4814 cErrors++;
4815 }
4816 fIgnoreFlags |= X86_PTE_RW;
4817# else
4818 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4819# endif
4820 }
4821 else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4822 {
4823 /* access bit emulation (not implemented). */
4824 if ((PteSrc.u & X86_PTE_A) || SHW_PTE_IS_P(PteDst))
4825 {
4826 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4827 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4828 cErrors++;
4829 continue;
4830 }
4831 if (!SHW_PTE_IS_A(PteDst))
4832 {
4833 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4834 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4835 cErrors++;
4836 }
4837 fIgnoreFlags |= X86_PTE_P;
4838 }
4839# ifdef DEBUG_sandervl
4840 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4841# endif
4842 }
4843
4844 if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4845 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4846 )
4847 {
4848 AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4849 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4850 fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4851 cErrors++;
4852 continue;
4853 }
4854 } /* foreach PTE */
4855 }
4856 else
4857 {
4858 /*
4859 * Big Page.
4860 */
4861 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4862 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
4863 {
4864 if (PdeDst.u & X86_PDE_RW)
4865 {
4866 AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4867 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4868 cErrors++;
4869 continue;
4870 }
4871 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4872 {
4873 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4874 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4875 cErrors++;
4876 continue;
4877 }
4878# if 0 /** @todo sync access bit properly... */
4879 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4880 {
4881 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4882 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4883 cErrors++;
4884 }
4885 fIgnoreFlags |= X86_PTE_RW;
4886# else
4887 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4888# endif
4889 }
4890 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4891 {
4892 /* access bit emulation (not implemented). */
4893 if ((PdeSrc.u & X86_PDE_A) || SHW_PDE_IS_P(PdeDst))
4894 {
4895 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4896 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4897 cErrors++;
4898 continue;
4899 }
4900 if (!SHW_PDE_IS_A(PdeDst))
4901 {
4902 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4903 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4904 cErrors++;
4905 }
4906 fIgnoreFlags |= X86_PTE_P;
4907 }
4908
4909 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4910 {
4911 AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4912 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4913 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4914 cErrors++;
4915 }
4916
4917 /* iterate the page table. */
4918 for (unsigned iPT = 0, off = 0;
4919 iPT < RT_ELEMENTS(pPTDst->a);
4920 iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE))
4921 {
4922 const SHWPTE PteDst = pPTDst->a[iPT];
4923
4924 if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4925 {
4926 AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4927 GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4928 cErrors++;
4929 }
4930
4931 /* skip not-present entries. */
4932 if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4933 continue;
4934
4935 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
4936
4937 /* match the physical addresses */
4938 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4939
4940# ifdef IN_RING3
4941 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4942 if (RT_FAILURE(rc))
4943 {
4944# if 0
4945 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4946 {
4947 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4948 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4949 cErrors++;
4950 }
4951# endif
4952 }
4953 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4954 {
4955 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4956 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4957 cErrors++;
4958 continue;
4959 }
4960# endif
4961 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4962 if (!pPhysPage)
4963 {
4964# if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4965 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4966 {
4967 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4968 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4969 cErrors++;
4970 continue;
4971 }
4972# endif
4973 if (SHW_PTE_IS_RW(PteDst))
4974 {
4975 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4976 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4977 cErrors++;
4978 }
4979 fIgnoreFlags |= X86_PTE_RW;
4980 }
4981 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4982 {
4983 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4984 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4985 cErrors++;
4986 continue;
4987 }
4988
4989 /* flags */
4990 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4991 {
4992 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4993 {
4994 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4995 {
4996 if ( SHW_PTE_IS_RW(PteDst)
4997 && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
4998 {
4999 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
5000 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5001 cErrors++;
5002 continue;
5003 }
5004 fIgnoreFlags |= X86_PTE_RW;
5005 }
5006 }
5007 else
5008 {
5009 if ( SHW_PTE_IS_P(PteDst)
5010 && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage)
5011# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
5012 && !PGM_PAGE_IS_MMIO(pPhysPage)
5013# endif
5014 )
5015 {
5016 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
5017 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5018 cErrors++;
5019 continue;
5020 }
5021 fIgnoreFlags |= X86_PTE_P;
5022 }
5023 }
5024
5025 if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
5026 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
5027 )
5028 {
5029 AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
5030 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
5031 fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5032 cErrors++;
5033 continue;
5034 }
5035 } /* for each PTE */
5036 }
5037 }
5038 /* not present */
5039
5040 } /* for each PDE */
5041
5042 } /* for each PDPTE */
5043
5044 } /* for each PML4E */
5045
5046# ifdef DEBUG
5047 if (cErrors)
5048 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
5049# endif
5050# endif /* GST is in {32BIT, PAE, AMD64} */
5051 return cErrors;
5052#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
5053}
5054#endif /* VBOX_STRICT */
5055
5056
5057/**
5058 * Sets up the CR3 for shadow paging
5059 *
5060 * @returns Strict VBox status code.
5061 * @retval VINF_SUCCESS.
5062 *
5063 * @param pVCpu The cross context virtual CPU structure.
5064 * @param GCPhysCR3 The physical address in the CR3 register. (A20 mask
5065 * already applied.)
5066 */
5067PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
5068{
5069 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
5070 int rc = VINF_SUCCESS;
5071
5072 /* Update guest paging info. */
5073#if PGM_GST_TYPE == PGM_TYPE_32BIT \
5074 || PGM_GST_TYPE == PGM_TYPE_PAE \
5075 || PGM_GST_TYPE == PGM_TYPE_AMD64
5076
5077 LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
5078 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
5079
5080# if PGM_GST_TYPE == PGM_TYPE_PAE
5081 if ( !pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped)
5082 || pVCpu->pgm.s.GCPhysPaeCR3 != GCPhysCR3)
5083# endif
5084 {
5085 /*
5086 * Map the page CR3 points at.
5087 */
5088 RTHCPTR HCPtrGuestCR3;
5089 rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3);
5090 if (RT_SUCCESS(rc))
5091 {
5092# if PGM_GST_TYPE == PGM_TYPE_32BIT
5093# ifdef IN_RING3
5094 pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
5095 pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
5096# else
5097 pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
5098 pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
5099# endif
5100
5101# elif PGM_GST_TYPE == PGM_TYPE_PAE
5102# ifdef IN_RING3
5103 pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
5104 pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
5105# else
5106 pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
5107 pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
5108# endif
5109
5110 X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
5111#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5112 /*
5113 * When EPT is enabled by the nested-hypervisor and the nested-guest is in PAE mode,
5114 * the guest-CPU context would've already been updated with the 4 PAE PDPEs specified
5115 * in the virtual VMCS. The PDPEs can differ from those in guest memory referenced by
5116 * the translated nested-guest CR3. We -MUST- use the PDPEs provided in the virtual VMCS
5117 * rather than those in guest memory.
5118 *
5119 * See Intel spec. 26.3.2.4 "Loading Page-Directory-Pointer-Table Entries".
5120 */
5121 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
5122 CPUMGetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5123 else
5124#endif
5125 {
5126 /* Update CPUM with the PAE PDPEs referenced by CR3. */
5127 memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
5128 CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5129 }
5130
5131 /*
5132 * Map the 4 PAE PDPEs.
5133 */
5134 rc = PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
5135 if (RT_SUCCESS(rc))
5136 {
5137# ifdef IN_RING3
5138 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
5139 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5140# else
5141 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5142 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
5143# endif
5144 pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
5145 }
5146
5147# elif PGM_GST_TYPE == PGM_TYPE_AMD64
5148# ifdef IN_RING3
5149 pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
5150 pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
5151# else
5152 pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
5153 pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
5154# endif
5155# endif
5156 }
5157 else
5158 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
5159 }
5160#endif
5161
5162 /*
5163 * Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
5164 */
5165# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5166 || PGM_SHW_TYPE == PGM_TYPE_PAE \
5167 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
5168 && ( PGM_GST_TYPE != PGM_TYPE_REAL \
5169 && PGM_GST_TYPE != PGM_TYPE_PROT))
5170
5171 Assert(!pVM->pgm.s.fNestedPaging);
5172 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
5173
5174 /*
5175 * Update the shadow root page as well since that's not fixed.
5176 */
5177 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5178 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
5179 PPGMPOOLPAGE pNewShwPageCR3;
5180
5181 PGM_LOCK_VOID(pVM);
5182
5183# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5184 if (pPool->cDirtyPages)
5185 pgmPoolResetDirtyPages(pVM);
5186# endif
5187
5188 Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */
5189 int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE,
5190 PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/, &pNewShwPageCR3);
5191 AssertFatalRC(rc2);
5192
5193 pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
5194 pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
5195
5196 /* Set the current hypervisor CR3. */
5197 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
5198
5199 /* Clean up the old CR3 root. */
5200 if ( pOldShwPageCR3
5201 && pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
5202 {
5203 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
5204
5205 /* Mark the page as unlocked; allow flushing again. */
5206 pgmPoolUnlockPage(pPool, pOldShwPageCR3);
5207
5208 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
5209 }
5210 PGM_UNLOCK(pVM);
5211# else
5212 NOREF(GCPhysCR3);
5213# endif
5214
5215 return rc;
5216}
5217
5218/**
5219 * Unmaps the shadow CR3.
5220 *
5221 * @returns VBox status, no specials.
5222 * @param pVCpu The cross context virtual CPU structure.
5223 */
5224PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
5225{
5226 LogFlow(("UnmapCR3\n"));
5227
5228 int rc = VINF_SUCCESS;
5229 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
5230
5231 /*
5232 * Update guest paging info.
5233 */
5234#if PGM_GST_TYPE == PGM_TYPE_32BIT
5235 pVCpu->pgm.s.pGst32BitPdR3 = 0;
5236 pVCpu->pgm.s.pGst32BitPdR0 = 0;
5237
5238#elif PGM_GST_TYPE == PGM_TYPE_PAE
5239 pVCpu->pgm.s.pGstPaePdptR3 = 0;
5240 pVCpu->pgm.s.pGstPaePdptR0 = 0;
5241 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
5242 {
5243 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
5244 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
5245 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
5246 }
5247
5248#elif PGM_GST_TYPE == PGM_TYPE_AMD64
5249 pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
5250 pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
5251
5252#else /* prot/real mode stub */
5253 /* nothing to do */
5254#endif
5255
5256 /*
5257 * PAE PDPEs (and CR3) might have been mapped via PGMGstMapPaePdpesAtCr3()
5258 * prior to switching to PAE in pfnMapCr3(), so we need to clear them here.
5259 */
5260 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5261 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5262 pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
5263
5264 /*
5265 * Update shadow paging info.
5266 */
5267#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5268 || PGM_SHW_TYPE == PGM_TYPE_PAE \
5269 || PGM_SHW_TYPE == PGM_TYPE_AMD64))
5270# if PGM_GST_TYPE != PGM_TYPE_REAL
5271 Assert(!pVM->pgm.s.fNestedPaging);
5272# endif
5273 PGM_LOCK_VOID(pVM);
5274
5275 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
5276 {
5277 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5278
5279# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5280 if (pPool->cDirtyPages)
5281 pgmPoolResetDirtyPages(pVM);
5282# endif
5283
5284 /* Mark the page as unlocked; allow flushing again. */
5285 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
5286
5287 pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
5288 pVCpu->pgm.s.pShwPageCR3R3 = 0;
5289 pVCpu->pgm.s.pShwPageCR3R0 = 0;
5290 }
5291
5292 PGM_UNLOCK(pVM);
5293#endif
5294
5295 return rc;
5296}
5297
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette