VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 99132

Last change on this file since 99132 was 99132, checked in by vboxsync, 21 months ago

VMM: Nested VMX: bugref:10318 PGM fixes for supporting Hyper-V in a VM using hardware-assisted execution.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 233.1 KB
Line 
1/* $Id: PGMAllBth.h 99132 2023-03-23 09:00:20Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6 * PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7 * bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8 *
9 * @remarks This file is one big \#ifdef-orgy!
10 *
11 */
12
13/*
14 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
15 *
16 * This file is part of VirtualBox base platform packages, as
17 * available from https://www.virtualbox.org.
18 *
19 * This program is free software; you can redistribute it and/or
20 * modify it under the terms of the GNU General Public License
21 * as published by the Free Software Foundation, in version 3 of the
22 * License.
23 *
24 * This program is distributed in the hope that it will be useful, but
25 * WITHOUT ANY WARRANTY; without even the implied warranty of
26 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
27 * General Public License for more details.
28 *
29 * You should have received a copy of the GNU General Public License
30 * along with this program; if not, see <https://www.gnu.org/licenses>.
31 *
32 * SPDX-License-Identifier: GPL-3.0-only
33 */
34
35#ifdef _MSC_VER
36/** @todo we're generating unnecessary code in nested/ept shadow mode and for
37 * real/prot-guest+RC mode. */
38# pragma warning(disable: 4505)
39#endif
40
41
42/*********************************************************************************************************************************
43* Internal Functions *
44*********************************************************************************************************************************/
45RT_C_DECLS_BEGIN
46PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
47#ifndef IN_RING3
48PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken);
49PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
50 bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken);
51# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
52static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
53 unsigned iPte, PCPGMPTWALKGST pGstWalkAll);
54static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
55 uint32_t uErr, PPGMPTWALKGST pGstWalkAll);
56static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll);
57# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
58#endif
59PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
60static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
61static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
62static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
63#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
64static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
65#else
66static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
67#endif
68PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
69PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
70PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
71#ifdef VBOX_STRICT
72PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
73#endif
74PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
75PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
76
77#ifdef IN_RING3
78PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
79#endif
80RT_C_DECLS_END
81
82
83
84
85/*
86 * Filter out some illegal combinations of guest and shadow paging, so we can
87 * remove redundant checks inside functions.
88 */
89#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
90 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
91# error "Invalid combination; PAE guest implies PAE shadow"
92#endif
93
94#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
95 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 \
96 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
97# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
98#endif
99
100#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
101 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE \
102 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
103# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
104#endif
105
106#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
107 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
108# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
109#endif
110
111
112/**
113 * Enters the shadow+guest mode.
114 *
115 * @returns VBox status code.
116 * @param pVCpu The cross context virtual CPU structure.
117 * @param GCPhysCR3 The physical address from the CR3 register.
118 */
119PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
120{
121 /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
122 * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
123 */
124#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
125 || PGM_SHW_TYPE == PGM_TYPE_PAE \
126 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
127 && ( PGM_GST_TYPE == PGM_TYPE_REAL \
128 || PGM_GST_TYPE == PGM_TYPE_PROT))
129
130 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
131
132 Assert(!pVM->pgm.s.fNestedPaging);
133
134 PGM_LOCK_VOID(pVM);
135 /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
136 * but any calls to GC need a proper shadow page setup as well.
137 */
138 /* Free the previous root mapping if still active. */
139 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
140 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
141 if (pOldShwPageCR3)
142 {
143 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
144
145 /* Mark the page as unlocked; allow flushing again. */
146 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
147
148 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
149 pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
150 pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
151 }
152
153 /* construct a fake address. */
154 GCPhysCR3 = RT_BIT_64(63);
155 PPGMPOOLPAGE pNewShwPageCR3;
156 int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
157 NIL_PGMPOOL_IDX, UINT32_MAX, false /*fLockPage*/,
158 &pNewShwPageCR3);
159 AssertRCReturn(rc, rc);
160
161 pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
162 pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
163
164 /* Mark the page as locked; disallow flushing. */
165 pgmPoolLockPage(pPool, pNewShwPageCR3);
166
167 /* Set the current hypervisor CR3. */
168 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
169
170 PGM_UNLOCK(pVM);
171 return rc;
172#else
173 NOREF(pVCpu); NOREF(GCPhysCR3);
174 return VINF_SUCCESS;
175#endif
176}
177
178
179#ifndef IN_RING3
180
181# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
182/**
183 * Deal with a guest page fault.
184 *
185 * @returns Strict VBox status code.
186 * @retval VINF_EM_RAW_GUEST_TRAP
187 * @retval VINF_EM_RAW_EMULATE_INSTR
188 *
189 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
190 * @param pWalk The guest page table walk result.
191 * @param uErr The error code.
192 */
193PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr)
194{
195 /*
196 * Calc the error code for the guest trap.
197 */
198 uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
199 ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)
200 : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
201 if ( pWalk->fRsvdError
202 || pWalk->fBadPhysAddr)
203 {
204 uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P;
205 Assert(!pWalk->fNotPresent);
206 }
207 else if (!pWalk->fNotPresent)
208 uNewErr |= X86_TRAP_PF_P;
209 TRPMSetErrorCode(pVCpu, uNewErr);
210
211 LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel));
212 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
213 return VINF_EM_RAW_GUEST_TRAP;
214}
215# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
216
217
218#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
219/**
220 * Deal with a guest page fault.
221 *
222 * The caller has taken the PGM lock.
223 *
224 * @returns Strict VBox status code.
225 *
226 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
227 * @param uErr The error code.
228 * @param pCtx Pointer to the register context for the CPU.
229 * @param pvFault The fault address.
230 * @param pPage The guest page at @a pvFault.
231 * @param pWalk The guest page table walk result.
232 * @param pGstWalk The guest paging-mode specific walk information.
233 * @param pfLockTaken PGM lock taken here or not (out). This is true
234 * when we're called.
235 */
236static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
237 RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
238# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
239 , PPGMPTWALK pWalk
240 , PGSTPTWALK pGstWalk
241# endif
242 )
243{
244# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
245 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
246# endif
247 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
248 VBOXSTRICTRC rcStrict;
249
250 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
251 {
252 /*
253 * Physical page access handler.
254 */
255# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
256 const RTGCPHYS GCPhysFault = pWalk->GCPhys;
257# else
258 const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
259# endif
260 PPGMPHYSHANDLER pCur;
261 rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysFault, &pCur);
262 if (RT_SUCCESS(rcStrict))
263 {
264 PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
265
266# ifdef PGM_SYNC_N_PAGES
267 /*
268 * If the region is write protected and we got a page not present fault, then sync
269 * the pages. If the fault was caused by a read, then restart the instruction.
270 * In case of write access continue to the GC write handler.
271 *
272 * ASSUMES that there is only one handler per page or that they have similar write properties.
273 */
274 if ( !(uErr & X86_TRAP_PF_P)
275 && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
276 {
277# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
278 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
279# else
280 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
281# endif
282 if ( RT_FAILURE(rcStrict)
283 || !(uErr & X86_TRAP_PF_RW)
284 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
285 {
286 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
287 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
288 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
289 return rcStrict;
290 }
291 }
292# endif
293# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
294 /*
295 * If the access was not thru a #PF(RSVD|...) resync the page.
296 */
297 if ( !(uErr & X86_TRAP_PF_RSVD)
298 && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
299# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
300 && (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK))
301 == PGM_PTATTRS_W_MASK /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
302# endif
303 )
304 {
305# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
306 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
307# else
308 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
309# endif
310 if ( RT_FAILURE(rcStrict)
311 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
312 {
313 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
314 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
315 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
316 return rcStrict;
317 }
318 }
319# endif
320
321 AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
322 || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
323 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
324 pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
325 if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
326 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
327 else
328 {
329 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
330 if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
331 }
332
333 if (pCurType->pfnPfHandler)
334 {
335 STAM_PROFILE_START(&pCur->Stat, h);
336
337 if (pCurType->fKeepPgmLock)
338 {
339 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault,
340 !pCurType->fRing0DevInsIdx ? pCur->uUser
341 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser));
342
343 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
344 }
345 else
346 {
347 uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
348 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
349 PGM_UNLOCK(pVM);
350 *pfLockTaken = false;
351
352 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault, uUser);
353
354 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
355 }
356 }
357 else
358 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
359
360 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
361 return rcStrict;
362 }
363 AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
364 }
365
366 /*
367 * There is a handled area of the page, but this fault doesn't belong to it.
368 * We must emulate the instruction.
369 *
370 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
371 * we first check if this was a page-not-present fault for a page with only
372 * write access handlers. Restart the instruction if it wasn't a write access.
373 */
374 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled);
375
376 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
377 && !(uErr & X86_TRAP_PF_P))
378 {
379# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
380 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
381# else
382 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
383# endif
384 if ( RT_FAILURE(rcStrict)
385 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
386 || !(uErr & X86_TRAP_PF_RW))
387 {
388 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
389 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
390 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
391 return rcStrict;
392 }
393 }
394
395 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
396 * It's writing to an unhandled part of the LDT page several million times.
397 */
398 rcStrict = PGMInterpretInstruction(pVCpu, pvFault);
399 LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
400 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; });
401 return rcStrict;
402} /* if any kind of handler */
403# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
404
405
406/**
407 * \#PF Handler for raw-mode guest execution.
408 *
409 * @returns VBox status code (appropriate for trap handling and GC return).
410 *
411 * @param pVCpu The cross context virtual CPU structure.
412 * @param uErr The trap error code.
413 * @param pCtx Pointer to the register context for the CPU.
414 * @param pvFault The fault address.
415 * @param pfLockTaken PGM lock taken here or not (out)
416 */
417PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken)
418{
419 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
420
421 *pfLockTaken = false;
422
423# if ( PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \
424 || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
425 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
426 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
427 && PGM_SHW_TYPE != PGM_TYPE_NONE
428 int rc;
429
430# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
431 /*
432 * Walk the guest page translation tables and check if it's a guest fault.
433 */
434 PGMPTWALK Walk;
435 GSTPTWALK GstWalk;
436 rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk);
437 if (RT_FAILURE_NP(rc))
438 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
439
440 /* assert some GstWalk sanity. */
441# if PGM_GST_TYPE == PGM_TYPE_AMD64
442 /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. */
443# endif
444# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
445 /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */
446# endif
447 /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */
448 /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */
449 Assert(Walk.fSucceeded);
450 Assert(Walk.fEffective & PGM_PTATTRS_R_MASK);
451
452 if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
453 {
454 if ( ( (uErr & X86_TRAP_PF_RW)
455 && !(Walk.fEffective & PGM_PTATTRS_W_MASK)
456 && ( (uErr & X86_TRAP_PF_US)
457 || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
458 || ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK))
459 || ((uErr & X86_TRAP_PF_ID) && (Walk.fEffective & PGM_PTATTRS_NX_MASK))
460 )
461 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
462 }
463
464 /* Take the big lock now before we update flags. */
465 *pfLockTaken = true;
466 PGM_LOCK_VOID(pVM);
467
468 /*
469 * Set the accessed and dirty flags.
470 */
471 /** @todo Should probably use cmpxchg logic here as we're potentially racing
472 * other CPUs in SMP configs. (the lock isn't enough, since we take it
473 * after walking and the page tables could be stale already) */
474# if PGM_GST_TYPE == PGM_TYPE_AMD64
475 if (!(GstWalk.Pml4e.u & X86_PML4E_A))
476 {
477 GstWalk.Pml4e.u |= X86_PML4E_A;
478 GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A);
479 }
480 if (!(GstWalk.Pdpe.u & X86_PDPE_A))
481 {
482 GstWalk.Pdpe.u |= X86_PDPE_A;
483 GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A);
484 }
485# endif
486 if (Walk.fBigPage)
487 {
488 Assert(GstWalk.Pde.u & X86_PDE_PS);
489 if (uErr & X86_TRAP_PF_RW)
490 {
491 if ((GstWalk.Pde.u & (X86_PDE4M_A | X86_PDE4M_D)) != (X86_PDE4M_A | X86_PDE4M_D))
492 {
493 GstWalk.Pde.u |= X86_PDE4M_A | X86_PDE4M_D;
494 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A | X86_PDE4M_D);
495 }
496 }
497 else
498 {
499 if (!(GstWalk.Pde.u & X86_PDE4M_A))
500 {
501 GstWalk.Pde.u |= X86_PDE4M_A;
502 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A);
503 }
504 }
505 }
506 else
507 {
508 Assert(!(GstWalk.Pde.u & X86_PDE_PS));
509 if (!(GstWalk.Pde.u & X86_PDE_A))
510 {
511 GstWalk.Pde.u |= X86_PDE_A;
512 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A);
513 }
514
515 if (uErr & X86_TRAP_PF_RW)
516 {
517# ifdef VBOX_WITH_STATISTICS
518 if (GstWalk.Pte.u & X86_PTE_D)
519 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty));
520 else
521 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage));
522# endif
523 if ((GstWalk.Pte.u & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
524 {
525 GstWalk.Pte.u |= X86_PTE_A | X86_PTE_D;
526 GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A | X86_PTE_D);
527 }
528 }
529 else
530 {
531 if (!(GstWalk.Pte.u & X86_PTE_A))
532 {
533 GstWalk.Pte.u |= X86_PTE_A;
534 GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A);
535 }
536 }
537 Assert(GstWalk.Pte.u == GstWalk.pPte->u);
538 }
539#if 0
540 /* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */
541 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u,
542 ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
543#endif
544
545# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
546 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */
547
548 /* Take the big lock now. */
549 *pfLockTaken = true;
550 PGM_LOCK_VOID(pVM);
551# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
552
553# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
554 /*
555 * If it is a reserved bit fault we know that it is an MMIO (access
556 * handler) related fault and can skip some 200 lines of code.
557 */
558 if (uErr & X86_TRAP_PF_RSVD)
559 {
560 Assert(uErr & X86_TRAP_PF_P);
561 PPGMPAGE pPage;
562# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
563 rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage);
564 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
565 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage,
566 pfLockTaken, &Walk, &GstWalk));
567 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
568# else
569 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
570 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
571 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
572 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
573# endif
574 AssertRC(rc);
575 PGM_INVL_PG(pVCpu, pvFault);
576 return rc; /* Restart with the corrected entry. */
577 }
578# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
579
580 /*
581 * Fetch the guest PDE, PDPE and PML4E.
582 */
583# if PGM_SHW_TYPE == PGM_TYPE_32BIT
584 const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
585 PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
586
587# elif PGM_SHW_TYPE == PGM_TYPE_PAE
588 const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
589 PX86PDPAE pPDDst;
590# if PGM_GST_TYPE == PGM_TYPE_PAE
591 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
592# else
593 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
594# endif
595 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
596
597# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
598 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
599 PX86PDPAE pPDDst;
600# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
601 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A,
602 X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst);
603# else
604 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
605# endif
606 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
607
608# elif PGM_SHW_TYPE == PGM_TYPE_EPT
609 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
610 PEPTPD pPDDst;
611 rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
612 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
613# endif
614 Assert(pPDDst);
615
616# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
617 /*
618 * Dirty page handling.
619 *
620 * If we successfully correct the write protection fault due to dirty bit
621 * tracking, then return immediately.
622 */
623 if (uErr & X86_TRAP_PF_RW) /* write fault? */
624 {
625 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
626 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
627 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
628 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
629 {
630 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0
631 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
632 ? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed
633 : &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
634 Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
635 return VINF_SUCCESS;
636 }
637#ifdef DEBUG_bird
638 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
639 AssertMsg(Walk.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
640#endif
641 }
642
643# if 0 /* rarely useful; leave for debugging. */
644 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
645# endif
646# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
647
648 /*
649 * A common case is the not-present error caused by lazy page table syncing.
650 *
651 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
652 * here so we can safely assume that the shadow PT is present when calling
653 * SyncPage later.
654 *
655 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
656 * of mapping conflict and defer to SyncCR3 in R3.
657 * (Again, we do NOT support access handlers for non-present guest pages.)
658 *
659 */
660# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
661 Assert(GstWalk.Pde.u & X86_PDE_P);
662# endif
663 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
664 && !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
665 {
666 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
667# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
668 LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
669 rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
670# else
671 LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
672 rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
673# endif
674 if (RT_SUCCESS(rc))
675 return rc;
676 Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
677 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
678 return VINF_PGM_SYNC_CR3;
679 }
680
681 /*
682 * Check if this fault address is flagged for special treatment,
683 * which means we'll have to figure out the physical address and
684 * check flags associated with it.
685 *
686 * ASSUME that we can limit any special access handling to pages
687 * in page tables which the guest believes to be present.
688 */
689# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
690 RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
691# else
692 RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
693# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
694 PPGMPAGE pPage;
695 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
696 if (RT_FAILURE(rc))
697 {
698 /*
699 * When the guest accesses invalid physical memory (e.g. probing
700 * of RAM or accessing a remapped MMIO range), then we'll fall
701 * back to the recompiler to emulate the instruction.
702 */
703 LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
704 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
705 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
706 return VINF_EM_RAW_EMULATE_INSTR;
707 }
708
709 /*
710 * Any handlers for this page?
711 */
712 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
713# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
714 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken,
715 &Walk, &GstWalk));
716# else
717 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
718# endif
719
720 /*
721 * We are here only if page is present in Guest page tables and
722 * trap is not handled by our handlers.
723 *
724 * Check it for page out-of-sync situation.
725 */
726 if (!(uErr & X86_TRAP_PF_P))
727 {
728 /*
729 * Page is not present in our page tables. Try to sync it!
730 */
731 if (uErr & X86_TRAP_PF_US)
732 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
733 else /* supervisor */
734 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
735
736 if (PGM_PAGE_IS_BALLOONED(pPage))
737 {
738 /* Emulate reads from ballooned pages as they are not present in
739 our shadow page tables. (Required for e.g. Solaris guests; soft
740 ecc, random nr generator.) */
741 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
742 LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
743 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
744 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
745 return rc;
746 }
747
748# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
749 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
750# else
751 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
752# endif
753 if (RT_SUCCESS(rc))
754 {
755 /* The page was successfully synced, return to the guest. */
756 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
757 return VINF_SUCCESS;
758 }
759 }
760 else /* uErr & X86_TRAP_PF_P: */
761 {
762 /*
763 * Write protected pages are made writable when the guest makes the
764 * first write to it. This happens for pages that are shared, write
765 * monitored or not yet allocated.
766 *
767 * We may also end up here when CR0.WP=0 in the guest.
768 *
769 * Also, a side effect of not flushing global PDEs are out of sync
770 * pages due to physical monitored regions, that are no longer valid.
771 * Assume for now it only applies to the read/write flag.
772 */
773 if (uErr & X86_TRAP_PF_RW)
774 {
775 /*
776 * Check if it is a read-only page.
777 */
778 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
779 {
780 Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
781 Assert(!PGM_PAGE_IS_ZERO(pPage));
782 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
783 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
784
785 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
786 if (rc != VINF_SUCCESS)
787 {
788 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
789 return rc;
790 }
791 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
792 return VINF_EM_NO_MEMORY;
793 }
794
795# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
796 /*
797 * Check to see if we need to emulate the instruction if CR0.WP=0.
798 */
799 if ( !(Walk.fEffective & PGM_PTATTRS_W_MASK)
800 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
801 && CPUMGetGuestCPL(pVCpu) < 3)
802 {
803 Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P));
804
805 /*
806 * The Netware WP0+RO+US hack.
807 *
808 * Netware sometimes(/always?) runs with WP0. It has been observed doing
809 * excessive write accesses to pages which are mapped with US=1 and RW=0
810 * while WP=0. This causes a lot of exits and extremely slow execution.
811 * To avoid trapping and emulating every write here, we change the shadow
812 * page table entry to map it as US=0 and RW=1 until user mode tries to
813 * access it again (see further below). We count these shadow page table
814 * changes so we can avoid having to clear the page pool every time the WP
815 * bit changes to 1 (see PGMCr0WpEnabled()).
816 */
817# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) && 1
818 if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
819 && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
820 && pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
821 {
822 Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage));
823 rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
824 if (rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3)
825 {
826 PGM_INVL_PG(pVCpu, pvFault);
827 pVCpu->pgm.s.cNetwareWp0Hacks++;
828 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; });
829 return rc;
830 }
831 AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
832 Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
833 }
834# endif
835
836 /* Interpret the access. */
837 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
838 Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK)));
839 if (RT_SUCCESS(rc))
840 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ);
841 else
842 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3);
843 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; });
844 return rc;
845 }
846# endif
847 /// @todo count the above case; else
848 if (uErr & X86_TRAP_PF_US)
849 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
850 else /* supervisor */
851 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
852
853 /*
854 * Sync the page.
855 *
856 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
857 * page is not present, which is not true in this case.
858 */
859# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
860 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
861# else
862 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
863# endif
864 if (RT_SUCCESS(rc))
865 {
866 /*
867 * Page was successfully synced, return to guest but invalidate
868 * the TLB first as the page is very likely to be in it.
869 */
870# if PGM_SHW_TYPE == PGM_TYPE_EPT
871 HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
872# else
873 PGM_INVL_PG(pVCpu, pvFault);
874# endif
875# ifdef VBOX_STRICT
876 PGMPTWALK GstPageWalk;
877 GstPageWalk.GCPhys = RTGCPHYS_MAX;
878 if (!pVM->pgm.s.fNestedPaging)
879 {
880 rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
881 AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) || ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective));
882 LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective));
883 }
884# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
885 uint64_t fPageShw = 0;
886 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
887 AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
888 ("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault));
889# endif
890# endif /* VBOX_STRICT */
891 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
892 return VINF_SUCCESS;
893 }
894 }
895# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
896 /*
897 * Check for Netware WP0+RO+US hack from above and undo it when user
898 * mode accesses the page again.
899 */
900 else if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
901 && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
902 && pVCpu->pgm.s.cNetwareWp0Hacks > 0
903 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
904 && CPUMGetGuestCPL(pVCpu) == 3
905 && pVM->cCpus == 1
906 )
907 {
908 Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
909 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
910 if (RT_SUCCESS(rc))
911 {
912 PGM_INVL_PG(pVCpu, pvFault);
913 pVCpu->pgm.s.cNetwareWp0Hacks--;
914 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; });
915 return VINF_SUCCESS;
916 }
917 }
918# endif /* PGM_WITH_PAGING */
919
920 /** @todo else: why are we here? */
921
922# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
923 /*
924 * Check for VMM page flags vs. Guest page flags consistency.
925 * Currently only for debug purposes.
926 */
927 if (RT_SUCCESS(rc))
928 {
929 /* Get guest page flags. */
930 PGMPTWALK GstPageWalk;
931 int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
932 if (RT_SUCCESS(rc2))
933 {
934 uint64_t fPageShw = 0;
935 rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
936
937#if 0
938 /*
939 * Compare page flags.
940 * Note: we have AVL, A, D bits desynced.
941 */
942 AssertMsg( (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
943 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
944 || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
945 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
946 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
947 && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW
948 && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US),
949 ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
950 pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
95101:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
95201:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
953
95401:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
95501:01:15.625516 00:08:43.268051 Location :
956e:\vbox\svn\trunk\srcPage flags mismatch!
957pvFault=fffff801b0d7b000
958 uErr=11 X86_TRAP_PF_ID | X86_TRAP_PF_P
959GCPhys=0000000019b52000
960fPageShw=0
961fPageGst=77b0000000000121
962rc=0
963#endif
964
965 }
966 else
967 AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
968 }
969 else
970 AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
971# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
972 }
973
974
975 /*
976 * If we get here it is because something failed above, i.e. most like guru
977 * meditiation time.
978 */
979 LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
980 __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pCtx->cs.Sel, pCtx->rip));
981 return rc;
982
983# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
984 NOREF(uErr); NOREF(pCtx); NOREF(pvFault);
985 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
986 return VERR_PGM_NOT_USED_IN_MODE;
987# endif
988}
989
990
991# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
992/**
993 * Deals with a nested-guest \#PF fault for a guest-physical page with a handler.
994 *
995 * @returns Strict VBox status code.
996 * @param pVCpu The cross context virtual CPU structure.
997 * @param uErr The error code.
998 * @param pCtx Pointer to the register context for the CPU.
999 * @param GCPhysNestedFault The nested-guest physical address of the fault.
1000 * @param pPage The guest page at @a GCPhysNestedFault.
1001 * @param GCPhysFault The guest-physical address of the fault.
1002 * @param pGstWalkAll The guest page walk result.
1003 * @param pfLockTaken Where to store whether the PGM is still held when
1004 * this function completes.
1005 *
1006 * @note The caller has taken the PGM lock.
1007 */
1008static VBOXSTRICTRC PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
1009 RTGCPHYS GCPhysNestedFault, PPGMPAGE pPage,
1010 RTGCPHYS GCPhysFault, PPGMPTWALKGST pGstWalkAll,
1011 bool *pfLockTaken)
1012{
1013# if PGM_GST_TYPE == PGM_TYPE_PROT \
1014 && PGM_SHW_TYPE == PGM_TYPE_EPT
1015
1016 /** @todo Assert uErr isn't X86_TRAP_PF_RSVD and remove release checks. */
1017 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysFault);
1018 AssertMsgReturn(PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage), ("%RGp %RGp uErr=%u\n", GCPhysNestedFault, GCPhysFault, uErr),
1019 VERR_PGM_HANDLER_IPE_1);
1020
1021 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1022 RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1023 RTGCPHYS const GCPhysPage = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1024
1025 /*
1026 * Physical page access handler.
1027 */
1028 PPGMPHYSHANDLER pCur;
1029 VBOXSTRICTRC rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pCur);
1030 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
1031
1032 PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
1033 Assert(pCurType);
1034
1035 /*
1036 * If the region is write protected and we got a page not present fault, then sync
1037 * the pages. If the fault was caused by a read, then restart the instruction.
1038 * In case of write access continue to the GC write handler.
1039 */
1040 if ( !(uErr & X86_TRAP_PF_P)
1041 && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1042 {
1043 Log7Func(("Syncing Monitored: GCPhysNestedPage=%RGp GCPhysPage=%RGp uErr=%#x\n", GCPhysNestedPage, GCPhysPage, uErr));
1044 rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
1045 Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1046 if ( RT_FAILURE(rcStrict)
1047 || !(uErr & X86_TRAP_PF_RW))
1048 {
1049 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1050 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1051 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1052 return rcStrict;
1053 }
1054 }
1055 else if ( !(uErr & X86_TRAP_PF_RSVD)
1056 && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE)
1057 {
1058 /*
1059 * If the access was NOT through an EPT misconfig (i.e. RSVD), sync the page.
1060 * This can happen for the VMX APIC-access page.
1061 */
1062 Log7Func(("Syncing MMIO: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1063 rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
1064 Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1065 if (RT_FAILURE(rcStrict))
1066 {
1067 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1068 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1069 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1070 return rcStrict;
1071 }
1072 }
1073
1074 AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
1075 || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
1076 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
1077 GCPhysNestedFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
1078 if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1079 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
1080 else
1081 {
1082 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
1083 if (uErr & X86_TRAP_PF_RSVD)
1084 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
1085 }
1086
1087 if (pCurType->pfnPfHandler)
1088 {
1089 STAM_PROFILE_START(&pCur->Stat, h);
1090 uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
1091 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
1092
1093 if (pCurType->fKeepPgmLock)
1094 {
1095 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1096 STAM_PROFILE_STOP(&pCur->Stat, h);
1097 }
1098 else
1099 {
1100 PGM_UNLOCK(pVM);
1101 *pfLockTaken = false;
1102 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1103 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
1104 }
1105 }
1106 else
1107 {
1108 AssertMsgFailed(("What's going on here!? Fault falls outside handler range!?\n"));
1109 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
1110 }
1111
1112 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
1113 return rcStrict;
1114
1115# else
1116 RT_NOREF8(pVCpu, uErr, pCtx, GCPhysNestedFault, pPage, GCPhysFault, pGstWalkAll, pfLockTaken);
1117 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1118 return VERR_PGM_NOT_USED_IN_MODE;
1119# endif
1120}
1121# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1122
1123
1124/**
1125 * Nested \#PF handler for nested-guest hardware-assisted execution using nested
1126 * paging.
1127 *
1128 * @returns VBox status code (appropriate for trap handling and GC return).
1129 * @param pVCpu The cross context virtual CPU structure.
1130 * @param uErr The fault error (X86_TRAP_PF_*).
1131 * @param pCtx Pointer to the register context for the CPU.
1132 * @param GCPhysNestedFault The nested-guest physical address of the fault.
1133 * @param fIsLinearAddrValid Whether translation of a nested-guest linear address
1134 * caused this fault. If @c false, GCPtrNestedFault
1135 * must be 0.
1136 * @param GCPtrNestedFault The nested-guest linear address of this fault.
1137 * @param pWalk The guest page table walk result.
1138 * @param pfLockTaken Where to store whether the PGM lock is still held
1139 * when this function completes.
1140 */
1141PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
1142 bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken)
1143{
1144 *pfLockTaken = false;
1145# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \
1146 && PGM_GST_TYPE == PGM_TYPE_PROT \
1147 && PGM_SHW_TYPE == PGM_TYPE_EPT
1148
1149 Assert(CPUMIsGuestVmxEptPagingEnabled(pVCpu));
1150 Assert(PGM_A20_IS_ENABLED(pVCpu));
1151
1152 /* We don't support mode-based execute control for EPT yet. */
1153 Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
1154 Assert(!(uErr & X86_TRAP_PF_US));
1155
1156 /* Take the big lock now. */
1157 *pfLockTaken = true;
1158 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1159 PGM_LOCK_VOID(pVM);
1160
1161 /*
1162 * Walk the guest EPT tables and check if it's an EPT violation or misconfiguration.
1163 */
1164 if (fIsLinearAddrValid)
1165 Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x GCPtrNestedFault=%RGv\n",
1166 pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr, GCPtrNestedFault));
1167 else
1168 Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x\n",
1169 pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr));
1170 PGMPTWALKGST GstWalkAll;
1171 int rc = pgmGstSlatWalk(pVCpu, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk, &GstWalkAll);
1172 if (RT_FAILURE(rc))
1173 return rc;
1174
1175 Assert(GstWalkAll.enmType == PGMPTWALKGSTTYPE_EPT);
1176 Assert(pWalk->fSucceeded);
1177 Assert(pWalk->fEffective & (PGM_PTATTRS_EPT_R_MASK | PGM_PTATTRS_EPT_W_MASK | PGM_PTATTRS_EPT_X_SUPER_MASK));
1178 Assert(pWalk->fIsSlat);
1179
1180# ifdef DEBUG_ramshankar
1181 /* Paranoia. */
1182 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_R_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_R_MASK));
1183 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_W_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_W_MASK));
1184 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_NX_MASK) == !RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_X_SUPER_MASK));
1185# endif
1186
1187 Log7Func(("SLAT: GCPhysNestedFault=%RGp -> GCPhys=%#RGp\n", GCPhysNestedFault, pWalk->GCPhys));
1188
1189 /*
1190 * Check page-access permissions.
1191 */
1192 if ( ((uErr & X86_TRAP_PF_RW) && !(pWalk->fEffective & PGM_PTATTRS_W_MASK))
1193 || ((uErr & X86_TRAP_PF_ID) && (pWalk->fEffective & PGM_PTATTRS_NX_MASK)))
1194 {
1195 Log7Func(("Permission failed! GCPtrNested=%RGv GCPhysNested=%RGp uErr=%#x fEffective=%#RX64\n", GCPtrNestedFault,
1196 GCPhysNestedFault, uErr, pWalk->fEffective));
1197 pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION;
1198 return VERR_ACCESS_DENIED;
1199 }
1200
1201 PGM_A20_ASSERT_MASKED(pVCpu, pWalk->GCPhys);
1202 RTGCPHYS const GCPhysPage = pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1203 RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1204
1205 /*
1206 * If we were called via an EPT misconfig, it should've already resulted in a nested-guest VM-exit.
1207 */
1208 AssertMsgReturn(!(uErr & X86_TRAP_PF_RSVD),
1209 ("Unexpected EPT misconfig VM-exit. GCPhysPage=%RGp GCPhysNestedPage=%RGp\n", GCPhysPage, GCPhysNestedPage),
1210 VERR_PGM_MAPPING_IPE);
1211
1212 /*
1213 * Fetch and sync the nested-guest EPT page directory pointer.
1214 */
1215 PEPTPD pEptPd;
1216 rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL /*ppPdpt*/, &pEptPd, &GstWalkAll);
1217 AssertRCReturn(rc, rc);
1218 Assert(pEptPd);
1219
1220 /*
1221 * A common case is the not-present error caused by lazy page table syncing.
1222 *
1223 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
1224 * here so we can safely assume that the shadow PT is present when calling
1225 * NestedSyncPage later.
1226 *
1227 * NOTE: It's possible we will be syncing the VMX APIC-access page here.
1228 * In that case, we would sync the page but will NOT go ahead with emulating
1229 * the APIC-access VM-exit through IEM. However, once the page is mapped in
1230 * the shadow tables, subsequent APIC-access VM-exits for the nested-guest
1231 * will be triggered by hardware. Maybe calling the IEM #PF handler can be
1232 * considered as an optimization later.
1233 */
1234 unsigned const iPde = (GCPhysNestedPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1235 if ( !(uErr & X86_TRAP_PF_P)
1236 && !(pEptPd->a[iPde].u & EPT_PRESENT_MASK))
1237 {
1238 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
1239 Log7Func(("NestedSyncPT: Lazy. GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1240 rc = PGM_BTH_NAME(NestedSyncPT)(pVCpu, GCPhysNestedPage, GCPhysPage, &GstWalkAll);
1241 if (RT_SUCCESS(rc))
1242 return rc;
1243 AssertMsgFailedReturn(("NestedSyncPT: %RGv failed! rc=%Rrc\n", GCPhysNestedPage, rc), VERR_PGM_MAPPING_IPE);
1244 }
1245
1246 /*
1247 * Check if this fault address is flagged for special treatment.
1248 * This handles faults on an MMIO or write-monitored page.
1249 *
1250 * If this happens to be the VMX APIC-access page, we don't treat is as MMIO
1251 * but rather sync it further below (as a regular guest page) which lets
1252 * hardware-assisted execution trigger the APIC-access VM-exits of the
1253 * nested-guest directly.
1254 */
1255 PPGMPAGE pPage;
1256 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1257 if (RT_FAILURE(rc))
1258 {
1259 /*
1260 * We failed to get the physical page which means it's a reserved/invalid
1261 * page address (not MMIO even). This can typically be observed with
1262 * Microsoft Hyper-V enabled Windows guests. We must fall back to emulating
1263 * the instruction, see @bugref{10318#c7}.
1264 */
1265 return VINF_EM_RAW_EMULATE_INSTR;
1266 }
1267 /* Check if this is an MMIO page and NOT the VMX APIC-access page. */
1268 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1269 {
1270 Log7Func(("MMIO: Calling NestedTrap0eHandlerDoAccessHandlers for GCPhys %RGp\n", GCPhysPage));
1271 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, GCPhysNestedFault,
1272 pPage, pWalk->GCPhys, &GstWalkAll,
1273 pfLockTaken));
1274 }
1275
1276 /*
1277 * We are here only if page is present in nested-guest page tables but the
1278 * trap is not handled by our handlers. Check for page out-of-sync situation.
1279 */
1280 if (!(uErr & X86_TRAP_PF_P))
1281 {
1282 Assert(!PGM_PAGE_IS_BALLOONED(pPage));
1283 Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1284 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
1285
1286 Log7Func(("SyncPage: Not-Present: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedFault, GCPhysPage));
1287 rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, PGM_SYNC_NR_PAGES, uErr, &GstWalkAll);
1288 if (RT_SUCCESS(rc))
1289 {
1290 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
1291 return VINF_SUCCESS;
1292 }
1293 }
1294 else if (uErr & X86_TRAP_PF_RW)
1295 {
1296 /*
1297 * Write protected pages are made writable when the guest makes the
1298 * first write to it. This happens for pages that are shared, write
1299 * monitored or not yet allocated.
1300 *
1301 * We may also end up here when CR0.WP=0 in the guest.
1302 *
1303 * Also, a side effect of not flushing global PDEs are out of sync
1304 * pages due to physical monitored regions, that are no longer valid.
1305 * Assume for now it only applies to the read/write flag.
1306 */
1307 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1308 {
1309 /* This is a read-only page. */
1310 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysPage));
1311 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
1312
1313 Log7Func(("Calling pgmPhysPageMakeWritable for GCPhysPage=%RGp\n", GCPhysPage));
1314 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1315 if (rc != VINF_SUCCESS)
1316 {
1317 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
1318 return rc;
1319 }
1320 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
1321 return VINF_EM_NO_MEMORY;
1322 }
1323
1324 Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1325 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1326
1327 /*
1328 * Sync the write-protected page.
1329 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1330 * page is not present, which is not true in this case.
1331 */
1332 Log7Func(("SyncPage: RW: cs:rip=%04x:%#RX64 GCPhysNestedPage=%RGp uErr=%#RX32 GCPhysPage=%RGp WalkGCPhys=%RGp\n",
1333 pCtx->cs.Sel, pCtx->rip, GCPhysNestedPage, (uint32_t)uErr, GCPhysPage, pWalk->GCPhys));
1334 rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /* cPages */, uErr, &GstWalkAll);
1335 if (RT_SUCCESS(rc))
1336 {
1337 HMInvalidatePhysPage(pVM, GCPhysPage);
1338 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
1339 return VINF_SUCCESS;
1340 }
1341 }
1342
1343 /*
1344 * If we get here it is because something failed above => guru meditation time.
1345 */
1346 LogRelFunc(("rc=%Rrc GCPhysNestedFault=%#RGp (%#RGp) uErr=%#RX32 cs:rip=%04x:%08RX64\n", rc, GCPhysNestedFault, GCPhysPage,
1347 (uint32_t)uErr, pCtx->cs.Sel, pCtx->rip));
1348 return VERR_PGM_MAPPING_IPE;
1349
1350# else /* !VBOX_WITH_NESTED_HWVIRT_VMX_EPT || PGM_GST_TYPE != PGM_TYPE_PROT || PGM_SHW_TYPE != PGM_TYPE_EPT */
1351 RT_NOREF7(pVCpu, uErr, pCtx, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk);
1352 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1353 return VERR_PGM_NOT_USED_IN_MODE;
1354# endif
1355}
1356
1357#endif /* !IN_RING3 */
1358
1359
1360/**
1361 * Emulation of the invlpg instruction.
1362 *
1363 *
1364 * @returns VBox status code.
1365 *
1366 * @param pVCpu The cross context virtual CPU structure.
1367 * @param GCPtrPage Page to invalidate.
1368 *
1369 * @remark ASSUMES that the guest is updating before invalidating. This order
1370 * isn't required by the CPU, so this is speculative and could cause
1371 * trouble.
1372 * @remark No TLB shootdown is done on any other VCPU as we assume that
1373 * invlpg emulation is the *only* reason for calling this function.
1374 * (The guest has to shoot down TLB entries on other CPUs itself)
1375 * Currently true, but keep in mind!
1376 *
1377 * @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1378 * Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1379 */
1380PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1381{
1382#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1383 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1384 && PGM_SHW_TYPE != PGM_TYPE_NONE
1385 int rc;
1386 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1387 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1388
1389 PGM_LOCK_ASSERT_OWNER(pVM);
1390
1391 LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1392
1393 /*
1394 * Get the shadow PD entry and skip out if this PD isn't present.
1395 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1396 */
1397# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1398 const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1399 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1400
1401 /* Fetch the pgm pool shadow descriptor. */
1402 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1403# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1404 if (!pShwPde)
1405 {
1406 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1407 return VINF_SUCCESS;
1408 }
1409# else
1410 Assert(pShwPde);
1411# endif
1412
1413# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1414 const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1415 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1416
1417 /* If the shadow PDPE isn't present, then skip the invalidate. */
1418# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1419 if (!pPdptDst || !(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1420# else
1421 if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1422# endif
1423 {
1424 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1425 PGM_INVL_PG(pVCpu, GCPtrPage);
1426 return VINF_SUCCESS;
1427 }
1428
1429 /* Fetch the pgm pool shadow descriptor. */
1430 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1431 AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1432
1433 PX86PDPAE pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1434 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1435 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1436
1437# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1438 /* PML4 */
1439 /*const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/
1440 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1441 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1442 PX86PDPAE pPDDst;
1443 PX86PDPT pPdptDst;
1444 PX86PML4E pPml4eDst;
1445 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1446 if (rc != VINF_SUCCESS)
1447 {
1448 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1449 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1450 PGM_INVL_PG(pVCpu, GCPtrPage);
1451 return VINF_SUCCESS;
1452 }
1453 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1454 Assert(pPDDst);
1455 Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P);
1456
1457 /* Fetch the pgm pool shadow descriptor. */
1458 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1459 Assert(pShwPde);
1460
1461# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1462
1463 const SHWPDE PdeDst = *pPdeDst;
1464 if (!(PdeDst.u & X86_PDE_P))
1465 {
1466 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1467 PGM_INVL_PG(pVCpu, GCPtrPage);
1468 return VINF_SUCCESS;
1469 }
1470
1471 /*
1472 * Get the guest PD entry and calc big page.
1473 */
1474# if PGM_GST_TYPE == PGM_TYPE_32BIT
1475 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1476 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1477 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1478# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1479 unsigned iPDSrc = 0;
1480# if PGM_GST_TYPE == PGM_TYPE_PAE
1481 X86PDPE PdpeSrcIgn;
1482 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1483# else /* AMD64 */
1484 PX86PML4E pPml4eSrcIgn;
1485 X86PDPE PdpeSrcIgn;
1486 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1487# endif
1488 GSTPDE PdeSrc;
1489
1490 if (pPDSrc)
1491 PdeSrc = pPDSrc->a[iPDSrc];
1492 else
1493 PdeSrc.u = 0;
1494# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1495 const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1496 const bool fIsBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1497 if (fWasBigPage != fIsBigPage)
1498 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1499
1500# ifdef IN_RING3
1501 /*
1502 * If a CR3 Sync is pending we may ignore the invalidate page operation
1503 * depending on the kind of sync and if it's a global page or not.
1504 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
1505 */
1506# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1507 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1508 || ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1509 && fIsBigPage
1510 && (PdeSrc.u & X86_PDE4M_G)
1511 )
1512 )
1513# else
1514 if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1515# endif
1516 {
1517 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1518 return VINF_SUCCESS;
1519 }
1520# endif /* IN_RING3 */
1521
1522 /*
1523 * Deal with the Guest PDE.
1524 */
1525 rc = VINF_SUCCESS;
1526 if (PdeSrc.u & X86_PDE_P)
1527 {
1528 Assert( (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1529 && ((PdeSrc.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW) || pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1530 if (!fIsBigPage)
1531 {
1532 /*
1533 * 4KB - page.
1534 */
1535 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1536 RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1537
1538# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1539 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1540 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1541# endif
1542 if (pShwPage->GCPhys == GCPhys)
1543 {
1544 /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1545 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1546
1547 PGSTPT pPTSrc;
1548 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1549 if (RT_SUCCESS(rc))
1550 {
1551 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1552 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1553 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1554 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1555 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1556 GCPtrPage, PteSrc.u & X86_PTE_P,
1557 (PteSrc.u & PdeSrc.u & X86_PTE_RW),
1558 (PteSrc.u & PdeSrc.u & X86_PTE_US),
1559 (uint64_t)PteSrc.u,
1560 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1561 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1562 }
1563 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1564 PGM_INVL_PG(pVCpu, GCPtrPage);
1565 }
1566 else
1567 {
1568 /*
1569 * The page table address changed.
1570 */
1571 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1572 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1573 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1574 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1575 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1576 PGM_INVL_VCPU_TLBS(pVCpu);
1577 }
1578 }
1579 else
1580 {
1581 /*
1582 * 2/4MB - page.
1583 */
1584 /* Before freeing the page, check if anything really changed. */
1585 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1586 RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1587# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1588 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1589 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1590# endif
1591 if ( pShwPage->GCPhys == GCPhys
1592 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1593 {
1594 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1595 /** @todo This test is wrong as it cannot check the G bit!
1596 * FIXME */
1597 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1598 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1599 && ( (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1600 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1601 {
1602 LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1603 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1604 return VINF_SUCCESS;
1605 }
1606 }
1607
1608 /*
1609 * Ok, the page table is present and it's been changed in the guest.
1610 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1611 * We could do this for some flushes in GC too, but we need an algorithm for
1612 * deciding which 4MB pages containing code likely to be executed very soon.
1613 */
1614 LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1615 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1616 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1617 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1618 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1619 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1620 }
1621 }
1622 else
1623 {
1624 /*
1625 * Page directory is not present, mark shadow PDE not present.
1626 */
1627 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1628 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1629 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1630 PGM_INVL_PG(pVCpu, GCPtrPage);
1631 }
1632 return rc;
1633
1634#else /* guest real and protected mode, nested + ept, none. */
1635 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1636 NOREF(pVCpu); NOREF(GCPtrPage);
1637 return VINF_SUCCESS;
1638#endif
1639}
1640
1641#if PGM_SHW_TYPE != PGM_TYPE_NONE
1642
1643/**
1644 * Update the tracking of shadowed pages.
1645 *
1646 * @param pVCpu The cross context virtual CPU structure.
1647 * @param pShwPage The shadow page.
1648 * @param HCPhys The physical page we is being dereferenced.
1649 * @param iPte Shadow PTE index
1650 * @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1651 */
1652DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1653 RTGCPHYS GCPhysPage)
1654{
1655 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1656
1657# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1658 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1659 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1660
1661 /* Use the hint we retrieved from the cached guest PT. */
1662 if (pShwPage->fDirty)
1663 {
1664 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1665
1666 Assert(pShwPage->cPresent);
1667 Assert(pPool->cPresent);
1668 pShwPage->cPresent--;
1669 pPool->cPresent--;
1670
1671 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1672 AssertRelease(pPhysPage);
1673 pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1674 return;
1675 }
1676# else
1677 NOREF(GCPhysPage);
1678# endif
1679
1680 STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a);
1681 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1682
1683 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1684 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1685 * 2. write protect all shadowed pages. I.e. implement caching.
1686 */
1687 /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1688
1689 /*
1690 * Find the guest address.
1691 */
1692 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1693 pRam;
1694 pRam = pRam->CTX_SUFF(pNext))
1695 {
1696 unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT;
1697 while (iPage-- > 0)
1698 {
1699 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1700 {
1701 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1702
1703 Assert(pShwPage->cPresent);
1704 Assert(pPool->cPresent);
1705 pShwPage->cPresent--;
1706 pPool->cPresent--;
1707
1708 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1709 STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a);
1710 return;
1711 }
1712 }
1713 }
1714
1715 for (;;)
1716 AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1717}
1718
1719
1720/**
1721 * Update the tracking of shadowed pages.
1722 *
1723 * @param pVCpu The cross context virtual CPU structure.
1724 * @param pShwPage The shadow page.
1725 * @param u16 The top 16-bit of the pPage->HCPhys.
1726 * @param pPage Pointer to the guest page. this will be modified.
1727 * @param iPTDst The index into the shadow table.
1728 */
1729DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16,
1730 PPGMPAGE pPage, const unsigned iPTDst)
1731{
1732 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1733
1734 /*
1735 * Just deal with the simple first time here.
1736 */
1737 if (!u16)
1738 {
1739 STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin);
1740 u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1741 /* Save the page table index. */
1742 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1743 }
1744 else
1745 u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1746
1747 /* write back */
1748 Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1749 PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1750
1751 /* update statistics. */
1752 pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1753 pShwPage->cPresent++;
1754 if (pShwPage->iFirstPresent > iPTDst)
1755 pShwPage->iFirstPresent = iPTDst;
1756}
1757
1758
1759/**
1760 * Modifies a shadow PTE to account for access handlers.
1761 *
1762 * @param pVM The cross context VM structure.
1763 * @param pVCpu The cross context virtual CPU structure.
1764 * @param pPage The page in question.
1765 * @param GCPhysPage The guest-physical address of the page.
1766 * @param fPteSrc The shadowed flags of the source PTE. Must include the
1767 * A (accessed) bit so it can be emulated correctly.
1768 * @param pPteDst The shadow PTE (output). This is temporary storage and
1769 * does not need to be set atomically.
1770 */
1771DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PVMCPUCC pVCpu, PCPGMPAGE pPage, RTGCPHYS GCPhysPage, uint64_t fPteSrc,
1772 PSHWPTE pPteDst)
1773{
1774 RT_NOREF_PV(pVM); RT_NOREF_PV(fPteSrc); RT_NOREF_PV(pVCpu); RT_NOREF_PV(GCPhysPage);
1775
1776 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1777 * Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1778 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1779 {
1780 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1781# if PGM_SHW_TYPE == PGM_TYPE_EPT
1782 pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) | EPT_E_READ | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
1783# else
1784 if (fPteSrc & X86_PTE_A)
1785 {
1786 SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage));
1787 SHW_PTE_SET_RO(*pPteDst);
1788 }
1789 else
1790 SHW_PTE_SET(*pPteDst, 0);
1791# endif
1792 }
1793# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1794# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
1795 else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1796 && ( BTH_IS_NP_ACTIVE(pVM)
1797 || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1798# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1799 && pVM->pgm.s.fLessThan52PhysicalAddressBits
1800# endif
1801 )
1802 {
1803 LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1804# if PGM_SHW_TYPE == PGM_TYPE_EPT
1805 /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1806 pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg
1807 /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1808 | EPT_E_WRITE
1809 /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1810 | EPT_E_MEMTYPE_INVALID_3;
1811# else
1812 /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1813 SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P);
1814# endif
1815 }
1816# endif
1817# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1818 else
1819 {
1820 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1821 SHW_PTE_SET(*pPteDst, 0);
1822 }
1823 /** @todo count these kinds of entries. */
1824}
1825
1826
1827/**
1828 * Creates a 4K shadow page for a guest page.
1829 *
1830 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1831 * physical address. The PdeSrc argument only the flags are used. No page
1832 * structured will be mapped in this function.
1833 *
1834 * @param pVCpu The cross context virtual CPU structure.
1835 * @param pPteDst Destination page table entry.
1836 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1837 * Can safely assume that only the flags are being used.
1838 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1839 * @param pShwPage Pointer to the shadow page.
1840 * @param iPTDst The index into the shadow table.
1841 *
1842 * @remark Not used for 2/4MB pages!
1843 */
1844# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
1845static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1846 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1847# else
1848static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1849 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1850# endif
1851{
1852 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1853 RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1854
1855# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1856 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1857 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1858
1859 if (pShwPage->fDirty)
1860 {
1861 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1862 PGSTPT pGstPT;
1863
1864 /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1865 pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1866 GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1867 pGstPT->a[iPTDst].u = PteSrc.u;
1868 }
1869# else
1870 Assert(!pShwPage->fDirty);
1871# endif
1872
1873# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1874 if ( (PteSrc.u & X86_PTE_P)
1875 && GST_IS_PTE_VALID(pVCpu, PteSrc))
1876# endif
1877 {
1878# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1879 RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1880# endif
1881 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1882
1883 /*
1884 * Find the ram range.
1885 */
1886 PPGMPAGE pPage;
1887 int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1888 if (RT_SUCCESS(rc))
1889 {
1890 /* Ignore ballooned pages.
1891 Don't return errors or use a fatal assert here as part of a
1892 shadow sync range might included ballooned pages. */
1893 if (PGM_PAGE_IS_BALLOONED(pPage))
1894 {
1895 Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */
1896 return;
1897 }
1898
1899# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1900 /* Make the page writable if necessary. */
1901 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1902 && ( PGM_PAGE_IS_ZERO(pPage)
1903# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1904 || ( (PteSrc.u & X86_PTE_RW)
1905# else
1906 || ( 1
1907# endif
1908 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1909# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1910 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1911# endif
1912# ifdef VBOX_WITH_PAGE_SHARING
1913 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1914# endif
1915 )
1916 )
1917 )
1918 {
1919 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1920 AssertRC(rc);
1921 }
1922# endif
1923
1924 /*
1925 * Make page table entry.
1926 */
1927 SHWPTE PteDst;
1928# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1929 uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1930# else
1931 uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D;
1932# endif
1933 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1934 {
1935# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1936 /*
1937 * If the page or page directory entry is not marked accessed,
1938 * we mark the page not present.
1939 */
1940 if (!(PteSrc.u & X86_PTE_A) || !(PdeSrc.u & X86_PDE_A))
1941 {
1942 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1943 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage));
1944 SHW_PTE_SET(PteDst, 0);
1945 }
1946 /*
1947 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1948 * when the page is modified.
1949 */
1950 else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW))
1951 {
1952 AssertCompile(X86_PTE_RW == X86_PDE_RW);
1953 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage));
1954 SHW_PTE_SET(PteDst,
1955 fGstShwPteFlags
1956 | PGM_PAGE_GET_HCPHYS(pPage)
1957 | PGM_PTFLAGS_TRACK_DIRTY);
1958 SHW_PTE_SET_RO(PteDst);
1959 }
1960 else
1961# endif
1962 {
1963 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped));
1964# if PGM_SHW_TYPE == PGM_TYPE_EPT
1965 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage)
1966 | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
1967# else
1968 SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage));
1969# endif
1970 }
1971
1972 /*
1973 * Make sure only allocated pages are mapped writable.
1974 */
1975 if ( SHW_PTE_IS_P_RW(PteDst)
1976 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1977 {
1978 /* Still applies to shared pages. */
1979 Assert(!PGM_PAGE_IS_ZERO(pPage));
1980 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1981 Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1982 }
1983 }
1984 else
1985 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhysPage, fGstShwPteFlags, &PteDst);
1986
1987 /*
1988 * Keep user track up to date.
1989 */
1990 if (SHW_PTE_IS_P(PteDst))
1991 {
1992 if (!SHW_PTE_IS_P(*pPteDst))
1993 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1994 else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1995 {
1996 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst)));
1997 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1998 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1999 }
2000 }
2001 else if (SHW_PTE_IS_P(*pPteDst))
2002 {
2003 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
2004 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2005 }
2006
2007 /*
2008 * Update statistics and commit the entry.
2009 */
2010# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2011 if (!(PteSrc.u & X86_PTE_G))
2012 pShwPage->fSeenNonGlobal = true;
2013# endif
2014 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2015 return;
2016 }
2017
2018/** @todo count these three different kinds. */
2019 Log2(("SyncPageWorker: invalid address in Pte\n"));
2020 }
2021# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2022 else if (!(PteSrc.u & X86_PTE_P))
2023 Log2(("SyncPageWorker: page not present in Pte\n"));
2024 else
2025 Log2(("SyncPageWorker: invalid Pte\n"));
2026# endif
2027
2028 /*
2029 * The page is not present or the PTE is bad. Replace the shadow PTE by
2030 * an empty entry, making sure to keep the user tracking up to date.
2031 */
2032 if (SHW_PTE_IS_P(*pPteDst))
2033 {
2034 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
2035 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2036 }
2037 SHW_PTE_ATOMIC_SET(*pPteDst, 0);
2038}
2039
2040
2041/**
2042 * Syncs a guest OS page.
2043 *
2044 * There are no conflicts at this point, neither is there any need for
2045 * page table allocations.
2046 *
2047 * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
2048 * When called in AMD64 guest mode, the guest PML4E shall be valid.
2049 *
2050 * @returns VBox status code.
2051 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
2052 * @param pVCpu The cross context virtual CPU structure.
2053 * @param PdeSrc Page directory entry of the guest.
2054 * @param GCPtrPage Guest context page address.
2055 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2056 * @param uErr Fault error (X86_TRAP_PF_*).
2057 */
2058static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
2059{
2060 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2061 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2062 LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
2063 RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
2064
2065 PGM_LOCK_ASSERT_OWNER(pVM);
2066
2067# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2068 || PGM_GST_TYPE == PGM_TYPE_PAE \
2069 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2070 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
2071
2072 /*
2073 * Assert preconditions.
2074 */
2075 Assert(PdeSrc.u & X86_PDE_P);
2076 Assert(cPages);
2077# if 0 /* rarely useful; leave for debugging. */
2078 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
2079# endif
2080
2081 /*
2082 * Get the shadow PDE, find the shadow page table in the pool.
2083 */
2084# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2085 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2086 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2087
2088 /* Fetch the pgm pool shadow descriptor. */
2089 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2090 Assert(pShwPde);
2091
2092# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2093 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2094 PPGMPOOLPAGE pShwPde = NULL;
2095 PX86PDPAE pPDDst;
2096
2097 /* Fetch the pgm pool shadow descriptor. */
2098 int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2099 AssertRCSuccessReturn(rc2, rc2);
2100 Assert(pShwPde);
2101
2102 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2103 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2104
2105# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2106 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2107 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2108 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2109 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2110
2111 int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2112 AssertRCSuccessReturn(rc2, rc2);
2113 Assert(pPDDst && pPdptDst);
2114 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2115# endif
2116 SHWPDE PdeDst = *pPdeDst;
2117
2118 /*
2119 * - In the guest SMP case we could have blocked while another VCPU reused
2120 * this page table.
2121 * - With W7-64 we may also take this path when the A bit is cleared on
2122 * higher level tables (PDPE/PML4E). The guest does not invalidate the
2123 * relevant TLB entries. If we're write monitoring any page mapped by
2124 * the modified entry, we may end up here with a "stale" TLB entry.
2125 */
2126 if (!(PdeDst.u & X86_PDE_P))
2127 {
2128 Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2129 AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW),
2130 ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2131 if (uErr & X86_TRAP_PF_P)
2132 PGM_INVL_PG(pVCpu, GCPtrPage);
2133 return VINF_SUCCESS; /* force the instruction to be executed again. */
2134 }
2135
2136 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2137 Assert(pShwPage);
2138
2139# if PGM_GST_TYPE == PGM_TYPE_AMD64
2140 /* Fetch the pgm pool shadow descriptor. */
2141 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2142 Assert(pShwPde);
2143# endif
2144
2145 /*
2146 * Check that the page is present and that the shadow PDE isn't out of sync.
2147 */
2148 const bool fBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
2149 const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
2150 RTGCPHYS GCPhys;
2151 if (!fBigPage)
2152 {
2153 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2154# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2155 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2156 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
2157# endif
2158 }
2159 else
2160 {
2161 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2162# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2163 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2164 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2165# endif
2166 }
2167 /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2168 if ( fPdeValid
2169 && pShwPage->GCPhys == GCPhys
2170 && (PdeSrc.u & X86_PDE_P)
2171 && (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
2172 && ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW))
2173# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2174 && ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) || !GST_IS_NX_ACTIVE(pVCpu))
2175# endif
2176 )
2177 {
2178 /*
2179 * Check that the PDE is marked accessed already.
2180 * Since we set the accessed bit *before* getting here on a #PF, this
2181 * check is only meant for dealing with non-#PF'ing paths.
2182 */
2183 if (PdeSrc.u & X86_PDE_A)
2184 {
2185 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2186 if (!fBigPage)
2187 {
2188 /*
2189 * 4KB Page - Map the guest page table.
2190 */
2191 PGSTPT pPTSrc;
2192 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2193 if (RT_SUCCESS(rc))
2194 {
2195# ifdef PGM_SYNC_N_PAGES
2196 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2197 if ( cPages > 1
2198 && !(uErr & X86_TRAP_PF_P)
2199 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2200 {
2201 /*
2202 * This code path is currently only taken when the caller is PGMTrap0eHandler
2203 * for non-present pages!
2204 *
2205 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2206 * deal with locality.
2207 */
2208 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2209# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2210 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2211 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2212# else
2213 const unsigned offPTSrc = 0;
2214# endif
2215 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2216 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2217 iPTDst = 0;
2218 else
2219 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2220
2221 for (; iPTDst < iPTDstEnd; iPTDst++)
2222 {
2223 const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2224
2225 if ( (pPteSrc->u & X86_PTE_P)
2226 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2227 {
2228 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT))
2229 | ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT);
2230 NOREF(GCPtrCurPage);
2231 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2232 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2233 GCPtrCurPage, pPteSrc->u & X86_PTE_P,
2234 !!(pPteSrc->u & PdeSrc.u & X86_PTE_RW),
2235 !!(pPteSrc->u & PdeSrc.u & X86_PTE_US),
2236 (uint64_t)pPteSrc->u,
2237 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2238 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2239 }
2240 }
2241 }
2242 else
2243# endif /* PGM_SYNC_N_PAGES */
2244 {
2245 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2246 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2247 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2248 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2249 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2250 GCPtrPage, PteSrc.u & X86_PTE_P,
2251 !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
2252 !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
2253 (uint64_t)PteSrc.u,
2254 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2255 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2256 }
2257 }
2258 else /* MMIO or invalid page: emulated in #PF handler. */
2259 {
2260 LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2261 Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2262 }
2263 }
2264 else
2265 {
2266 /*
2267 * 4/2MB page - lazy syncing shadow 4K pages.
2268 * (There are many causes of getting here, it's no longer only CSAM.)
2269 */
2270 /* Calculate the GC physical address of this 4KB shadow page. */
2271 GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2272 /* Find ram range. */
2273 PPGMPAGE pPage;
2274 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2275 if (RT_SUCCESS(rc))
2276 {
2277 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2278
2279# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2280 /* Try to make the page writable if necessary. */
2281 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2282 && ( PGM_PAGE_IS_ZERO(pPage)
2283 || ( (PdeSrc.u & X86_PDE_RW)
2284 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2285# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2286 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2287# endif
2288# ifdef VBOX_WITH_PAGE_SHARING
2289 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2290# endif
2291 )
2292 )
2293 )
2294 {
2295 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2296 AssertRC(rc);
2297 }
2298# endif
2299
2300 /*
2301 * Make shadow PTE entry.
2302 */
2303 SHWPTE PteDst;
2304 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2305 SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
2306 else
2307 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2308
2309 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2310 if ( SHW_PTE_IS_P(PteDst)
2311 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2312 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2313
2314 /* Make sure only allocated pages are mapped writable. */
2315 if ( SHW_PTE_IS_P_RW(PteDst)
2316 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2317 {
2318 /* Still applies to shared pages. */
2319 Assert(!PGM_PAGE_IS_ZERO(pPage));
2320 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2321 Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2322 }
2323
2324 SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2325
2326 /*
2327 * If the page is not flagged as dirty and is writable, then make it read-only
2328 * at PD level, so we can set the dirty bit when the page is modified.
2329 *
2330 * ASSUMES that page access handlers are implemented on page table entry level.
2331 * Thus we will first catch the dirty access and set PDE.D and restart. If
2332 * there is an access handler, we'll trap again and let it work on the problem.
2333 */
2334 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2335 * As for invlpg, it simply frees the whole shadow PT.
2336 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2337 if ((PdeSrc.u & (X86_PDE4M_D | X86_PDE_RW)) == X86_PDE_RW)
2338 {
2339 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2340 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2341 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2342 }
2343 else
2344 {
2345 PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY | X86_PDE_RW);
2346 PdeDst.u |= PdeSrc.u & X86_PDE_RW;
2347 }
2348 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2349 Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2350 GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US),
2351 (uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2352 }
2353 else
2354 {
2355 LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2356 /** @todo must wipe the shadow page table entry in this
2357 * case. */
2358 }
2359 }
2360 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2361 return VINF_SUCCESS;
2362 }
2363
2364 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs));
2365 }
2366 else if (fPdeValid)
2367 {
2368 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2369 Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2370 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2371 }
2372 else
2373 {
2374/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2375 Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2376 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2377 }
2378
2379 /*
2380 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2381 * Yea, I'm lazy.
2382 */
2383 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2384 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
2385
2386 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2387 PGM_INVL_VCPU_TLBS(pVCpu);
2388 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2389
2390
2391# elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2392 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2393 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
2394 NOREF(PdeSrc);
2395
2396# ifdef PGM_SYNC_N_PAGES
2397 /*
2398 * Get the shadow PDE, find the shadow page table in the pool.
2399 */
2400# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2401 X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2402
2403# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2404 X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2405
2406# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2407 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2408 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2409 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2410 X86PDEPAE PdeDst;
2411 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2412
2413 int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2414 AssertRCSuccessReturn(rc, rc);
2415 Assert(pPDDst && pPdptDst);
2416 PdeDst = pPDDst->a[iPDDst];
2417
2418# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2419 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2420 PEPTPD pPDDst;
2421 EPTPDE PdeDst;
2422
2423 int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2424 if (rc != VINF_SUCCESS)
2425 {
2426 AssertRC(rc);
2427 return rc;
2428 }
2429 Assert(pPDDst);
2430 PdeDst = pPDDst->a[iPDDst];
2431# endif
2432 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2433 if (!SHW_PDE_IS_P(PdeDst))
2434 {
2435 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2436 Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2437 return VINF_SUCCESS; /* force the instruction to be executed again. */
2438 }
2439
2440 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2441 if (SHW_PDE_IS_BIG(PdeDst))
2442 {
2443 Assert(pVM->pgm.s.fNestedPaging);
2444 Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2445 return VINF_SUCCESS;
2446 }
2447
2448 /* Mask away the page offset. */
2449 GCPtrPage &= ~((RTGCPTR)0xfff);
2450
2451 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2452 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2453
2454 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2455 if ( cPages > 1
2456 && !(uErr & X86_TRAP_PF_P)
2457 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2458 {
2459 /*
2460 * This code path is currently only taken when the caller is PGMTrap0eHandler
2461 * for non-present pages!
2462 *
2463 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2464 * deal with locality.
2465 */
2466 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2467 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2468 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2469 iPTDst = 0;
2470 else
2471 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2472 for (; iPTDst < iPTDstEnd; iPTDst++)
2473 {
2474 if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2475 {
2476 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2477 | (iPTDst << GUEST_PAGE_SHIFT));
2478
2479 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2480 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2481 GCPtrCurPage,
2482 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2483 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2484
2485 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2486 break;
2487 }
2488 else
2489 Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n",
2490 (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2491 }
2492 }
2493 else
2494# endif /* PGM_SYNC_N_PAGES */
2495 {
2496 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2497 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2498 | (iPTDst << GUEST_PAGE_SHIFT));
2499
2500 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2501
2502 Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2503 GCPtrPage,
2504 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2505 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2506 }
2507 return VINF_SUCCESS;
2508
2509# else
2510 NOREF(PdeSrc);
2511 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2512 return VERR_PGM_NOT_USED_IN_MODE;
2513# endif
2514}
2515
2516#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
2517
2518#if !defined(IN_RING3) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
2519
2520/**
2521 * Sync a shadow page for a nested-guest page.
2522 *
2523 * @param pVCpu The cross context virtual CPU structure.
2524 * @param pPte The shadow page table entry.
2525 * @param GCPhysPage The guest-physical address of the page.
2526 * @param pShwPage The shadow page of the page table.
2527 * @param iPte The index of the page table entry.
2528 * @param pGstWalkAll The guest page table walk result.
2529 *
2530 * @note Not to be used for 2/4MB pages!
2531 */
2532static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
2533 unsigned iPte, PCPGMPTWALKGST pGstWalkAll)
2534{
2535 /*
2536 * Do not make assumptions about anything other than the final PTE entry in the
2537 * guest page table walk result. For instance, while mapping 2M PDEs as 4K pages,
2538 * the PDE might still be having its leaf bit set.
2539 *
2540 * In the future, we could consider introducing a generic SLAT macro like PSLATPTE
2541 * and using that instead of passing the full SLAT translation result.
2542 */
2543 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2544 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
2545 Assert(!pShwPage->fDirty);
2546 Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
2547 AssertMsg((pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK) == GCPhysPage,
2548 ("PTE address mismatch. GCPhysPage=%RGp Pte=%RX64\n", GCPhysPage, pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK));
2549
2550 /*
2551 * Find the ram range.
2552 */
2553 PPGMPAGE pPage;
2554 int rc = pgmPhysGetPageEx(pVCpu->CTX_SUFF(pVM), GCPhysPage, &pPage);
2555 if (RT_SUCCESS(rc))
2556 { /* likely */ }
2557 else
2558 {
2559 /*
2560 * This is a RAM hole/invalid/reserved address (not MMIO).
2561 * Nested Microsoft Hyper-V maps addresses like 0xf0220000 as RW WB memory.
2562 * Shadow a not-present page similar to MMIO, see @bugref{10318#c7}.
2563 */
2564 Assert(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS);
2565 if (SHW_PTE_IS_P(*pPte))
2566 {
2567 Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
2568 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2569 }
2570 Log7Func(("RAM hole/reserved %RGp -> ShwPte=0\n", GCPhysPage));
2571 SHW_PTE_ATOMIC_SET(*pPte, 0);
2572 return;
2573 }
2574
2575 Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2576
2577 /*
2578 * Make page table entry.
2579 */
2580 SHWPTE Pte;
2581 uint64_t const fGstShwPteFlags = pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptShadowedPteMask;
2582 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2583 {
2584# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2585 /* Page wasn't allocated, write protect it. */
2586 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2587 && ( PGM_PAGE_IS_ZERO(pPage)
2588 || ( (pGstWalkAll->u.Ept.Pte.u & EPT_E_WRITE)
2589 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2590# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2591 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2592# endif
2593# ifdef VBOX_WITH_PAGE_SHARING
2594 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2595# endif
2596 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_BALLOONED
2597 )
2598 )
2599 )
2600 {
2601 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE);
2602 Log7Func(("zero page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2603 }
2604 else
2605# endif
2606 {
2607 /** @todo access bit. */
2608 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
2609 Log7Func(("regular page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2610 }
2611 }
2612 else if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2613 {
2614 /** @todo access bit. */
2615 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE);
2616 Log7Func(("monitored page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2617 }
2618 else
2619 {
2620 /** @todo Do MMIO optimizations here too? */
2621 Log7Func(("mmio/all page (%R[pgmpage]) at %RGp -> 0\n", pPage, GCPhysPage));
2622 Pte.u = 0;
2623 }
2624
2625 /* Make sure only allocated pages are mapped writable. */
2626 Assert(!SHW_PTE_IS_P_RW(Pte) || PGM_PAGE_IS_ALLOCATED(pPage));
2627
2628 /*
2629 * Keep user track up to date.
2630 */
2631 if (SHW_PTE_IS_P(Pte))
2632 {
2633 if (!SHW_PTE_IS_P(*pPte))
2634 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2635 else if (SHW_PTE_GET_HCPHYS(*pPte) != SHW_PTE_GET_HCPHYS(Pte))
2636 {
2637 Log2(("SyncPageWorker: deref! *pPte=%RX64 Pte=%RX64\n", SHW_PTE_LOG64(*pPte), SHW_PTE_LOG64(Pte)));
2638 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2639 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2640 }
2641 }
2642 else if (SHW_PTE_IS_P(*pPte))
2643 {
2644 Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
2645 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2646 }
2647
2648 /*
2649 * Commit the entry.
2650 */
2651 SHW_PTE_ATOMIC_SET2(*pPte, Pte);
2652 return;
2653}
2654
2655
2656/**
2657 * Syncs a nested-guest page.
2658 *
2659 * There are no conflicts at this point, neither is there any need for
2660 * page table allocations.
2661 *
2662 * @returns VBox status code.
2663 * @param pVCpu The cross context virtual CPU structure.
2664 * @param GCPhysNestedPage The nested-guest physical address of the page being
2665 * synced.
2666 * @param GCPhysPage The guest-physical address of the page being synced.
2667 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2668 * @param uErr The page fault error (X86_TRAP_PF_XXX).
2669 * @param pGstWalkAll The guest page table walk result.
2670 */
2671static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
2672 uint32_t uErr, PPGMPTWALKGST pGstWalkAll)
2673{
2674 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2675 Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2676 Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2677
2678 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2679 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2680 Log7Func(("GCPhysNestedPage=%RGv GCPhysPage=%RGp cPages=%u uErr=%#x\n", GCPhysNestedPage, GCPhysPage, cPages, uErr));
2681 RT_NOREF_PV(uErr); RT_NOREF_PV(cPages);
2682
2683 PGM_LOCK_ASSERT_OWNER(pVM);
2684
2685 /*
2686 * Get the shadow PDE, find the shadow page table in the pool.
2687 */
2688 unsigned const iPde = ((GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK);
2689 PEPTPD pPd;
2690 int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL, &pPd, pGstWalkAll);
2691 if (RT_SUCCESS(rc))
2692 { /* likely */ }
2693 else
2694 {
2695 Log(("Failed to fetch EPT PD for %RGp (%RGp) rc=%Rrc\n", GCPhysNestedPage, GCPhysPage, rc));
2696 return rc;
2697 }
2698 Assert(pPd);
2699 EPTPDE Pde = pPd->a[iPde];
2700
2701 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2702 if (!SHW_PDE_IS_P(Pde))
2703 {
2704 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)Pde.u));
2705 Log7Func(("CPU%d: SyncPage: Pde at %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2706 return VINF_SUCCESS; /* force the instruction to be executed again. */
2707 }
2708
2709 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2710 if (SHW_PDE_IS_BIG(Pde))
2711 {
2712 Log7Func(("CPU%d: SyncPage: %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2713 return VINF_SUCCESS;
2714 }
2715
2716 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, Pde.u & EPT_PDE_PG_MASK);
2717 PEPTPT pPt = (PEPTPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2718
2719 /*
2720 * If we've shadowed a guest EPT PDE that maps a 2M page using a 4K table,
2721 * then sync the 4K sub-page in the 2M range.
2722 */
2723 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2724 {
2725 Assert(!SHW_PDE_IS_BIG(Pde));
2726
2727 Assert(pGstWalkAll->u.Ept.Pte.u == 0);
2728 Assert((Pde.u & EPT_PRESENT_MASK) == (pGstWalkAll->u.Ept.Pde.u & EPT_PRESENT_MASK));
2729 Assert(pShwPage->GCPhys == (pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK));
2730
2731#if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
2732 PPGMPAGE pPage;
2733 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); AssertRC(rc);
2734 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
2735 Assert(pShwPage->enmKind == PGMPOOLKIND_EPT_PT_FOR_EPT_2MB);
2736#endif
2737 uint64_t const fGstPteFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask & ~EPT_E_LEAF;
2738 pGstWalkAll->u.Ept.Pte.u = GCPhysPage | fGstPteFlags;
2739
2740 unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2741 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
2742 Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2743
2744 /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
2745 pGstWalkAll->u.Ept.Pte.u = 0;
2746 return VINF_SUCCESS;
2747 }
2748
2749 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2750# ifdef PGM_SYNC_N_PAGES
2751 if ( cPages > 1
2752 && !(uErr & X86_TRAP_PF_P)
2753 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2754 {
2755 /*
2756 * This code path is currently only taken for non-present pages!
2757 *
2758 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2759 * deal with locality.
2760 */
2761 unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2762 unsigned const iPteEnd = RT_MIN(iPte + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPt->a));
2763 if (iPte < PGM_SYNC_NR_PAGES / 2)
2764 iPte = 0;
2765 else
2766 iPte -= PGM_SYNC_NR_PAGES / 2;
2767 for (; iPte < iPteEnd; iPte++)
2768 {
2769 if (!SHW_PTE_IS_P(pPt->a[iPte]))
2770 {
2771 PGMPTWALKGST GstWalkPt;
2772 PGMPTWALK WalkPt;
2773 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2774 GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
2775 rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt,
2776 &GstWalkPt);
2777 if (RT_SUCCESS(rc))
2778 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], WalkPt.GCPhys, pShwPage, iPte, &GstWalkPt);
2779 else
2780 {
2781 /*
2782 * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
2783 * Ensure the shadow tables entry is not-present.
2784 */
2785 /** @todo Potential room for optimization (explained in NestedSyncPT). */
2786 AssertMsg(!pPt->a[iPte].u, ("%RX64\n", pPt->a[iPte].u));
2787 }
2788 Log7Func(("Many: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2789 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2790 break;
2791 }
2792 else
2793 {
2794# ifdef VBOX_STRICT
2795 /* Paranoia - Verify address of the page is what it should be. */
2796 PGMPTWALKGST GstWalkPt;
2797 PGMPTWALK WalkPt;
2798 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2799 GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
2800 rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt, &GstWalkPt);
2801 AssertRC(rc);
2802 PPGMPAGE pPage;
2803 rc = pgmPhysGetPageEx(pVM, WalkPt.GCPhys, &pPage);
2804 AssertRC(rc);
2805 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]),
2806 ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp HCPhys=%RHp Shw=%RHp\n",
2807 GCPhysNestedPage, WalkPt.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
2808# endif
2809 Log7Func(("Many3: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2810 }
2811 }
2812 }
2813 else
2814# endif /* PGM_SYNC_N_PAGES */
2815 {
2816 unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2817 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
2818 Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2819 }
2820
2821 return VINF_SUCCESS;
2822}
2823
2824
2825/**
2826 * Sync a shadow page table for a nested-guest page table.
2827 *
2828 * The shadow page table is not present in the shadow PDE.
2829 *
2830 * Handles mapping conflicts.
2831 *
2832 * A precondition for this method is that the shadow PDE is not present. The
2833 * caller must take the PGM lock before checking this and continue to hold it
2834 * when calling this method.
2835 *
2836 * @returns VBox status code.
2837 * @param pVCpu The cross context virtual CPU structure.
2838 * @param GCPhysNestedPage The nested-guest physical page address of the page
2839 * being synced.
2840 * @param GCPhysPage The guest-physical address of the page being synced.
2841 * @param pGstWalkAll The guest page table walk result.
2842 */
2843static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll)
2844{
2845 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2846 Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2847 Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2848
2849 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2850 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2851
2852 Log7Func(("GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
2853
2854 PGM_LOCK_ASSERT_OWNER(pVM);
2855 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2856
2857 PEPTPD pPd;
2858 PEPTPDPT pPdpt;
2859 unsigned const iPde = (GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK;
2860 int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, &pPdpt, &pPd, pGstWalkAll);
2861 if (RT_SUCCESS(rc))
2862 { /* likely */ }
2863 else
2864 {
2865 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2866 AssertRC(rc);
2867 return rc;
2868 }
2869 Assert(pPd);
2870 PSHWPDE pPde = &pPd->a[iPde];
2871
2872 unsigned const iPdpt = (GCPhysNestedPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2873 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdpt->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2874 Assert(pShwPde->enmKind == PGMPOOLKIND_EPT_PD_FOR_EPT_PD);
2875
2876 SHWPDE Pde = *pPde;
2877 Assert(!SHW_PDE_IS_P(Pde)); /* We're only supposed to call SyncPT on PDE!P and conflicts. */
2878
2879# ifdef PGM_WITH_LARGE_PAGES
2880 if (BTH_IS_NP_ACTIVE(pVM))
2881 {
2882 /*
2883 * Check if the guest is mapping a 2M page here.
2884 */
2885 PPGMPAGE pPage;
2886 rc = pgmPhysGetPageEx(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK, &pPage);
2887 AssertRCReturn(rc, rc);
2888 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2889 {
2890 /* A20 is always enabled in VMX root and non-root operation. */
2891 Assert(PGM_A20_IS_ENABLED(pVCpu));
2892
2893 RTHCPHYS HCPhys = NIL_RTHCPHYS;
2894 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
2895 {
2896 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
2897 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2898 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2899 }
2900 else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2901 {
2902 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
2903 rc = pgmPhysRecheckLargePage(pVM, GCPhysPage, pPage);
2904 if (RT_SUCCESS(rc))
2905 {
2906 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2907 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2908 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2909 }
2910 }
2911 else if (PGMIsUsingLargePages(pVM))
2912 {
2913 rc = pgmPhysAllocLargePage(pVM, GCPhysPage);
2914 if (RT_SUCCESS(rc))
2915 {
2916 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2917 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2918 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2919 }
2920 }
2921
2922 /*
2923 * If we have a 2M large page, we can map the guest's 2M large page right away.
2924 */
2925 uint64_t const fShwBigPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask;
2926 if (HCPhys != NIL_RTHCPHYS)
2927 {
2928 Pde.u = HCPhys | fShwBigPdeFlags;
2929 Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzBigPdeMask));
2930 Assert(Pde.u & EPT_E_LEAF);
2931 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
2932
2933 /* Add a reference to the first page only. */
2934 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPde);
2935
2936 Assert(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED);
2937
2938 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2939 Log7Func(("GstPde=%RGp ShwPde=%RX64 [2M]\n", pGstWalkAll->u.Ept.Pde.u, Pde.u));
2940 return VINF_SUCCESS;
2941 }
2942
2943 /*
2944 * We didn't get a perfect 2M fit. Split the 2M page into 4K pages.
2945 * The page ought not to be marked as a big (2M) page at this point.
2946 */
2947 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
2948
2949 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2950 PGMPOOLACCESS enmAccess;
2951 {
2952 Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_USER_EXECUTE)); /* Mode-based execute control for EPT not supported. */
2953 bool const fNoExecute = !(pGstWalkAll->u.Ept.Pde.u & EPT_E_EXECUTE);
2954 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_WRITE)
2955 enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2956 else
2957 enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2958 }
2959
2960 /*
2961 * Allocate & map a 4K shadow table to cover the 2M guest page.
2962 */
2963 PPGMPOOLPAGE pShwPage;
2964 RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK;
2965 rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_2MB, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2966 pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
2967 if ( rc == VINF_SUCCESS
2968 || rc == VINF_PGM_CACHED_PAGE)
2969 { /* likely */ }
2970 else
2971 {
2972 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2973 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2974 }
2975
2976 PSHWPT pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2977 Assert(pPt);
2978 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
2979 if (rc == VINF_SUCCESS)
2980 {
2981 /* The 4K PTEs shall inherit the flags of the 2M PDE page sans the leaf bit. */
2982 uint64_t const fShwPteFlags = fShwBigPdeFlags & ~EPT_E_LEAF;
2983
2984 /* Sync each 4K pages in the 2M range. */
2985 for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++)
2986 {
2987 RTGCPHYS const GCPhysSubPage = GCPhysPt | (iPte << GUEST_PAGE_SHIFT);
2988 pGstWalkAll->u.Ept.Pte.u = GCPhysSubPage | fShwPteFlags;
2989 Assert(!(pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptMbzPteMask));
2990 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysSubPage, pShwPage, iPte, pGstWalkAll);
2991 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [2M->4K]\n", pGstWalkAll->u.Ept.Pte, pPt->a[iPte].u, iPte));
2992 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2993 break;
2994 }
2995
2996 /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
2997 pGstWalkAll->u.Ept.Pte.u = 0;
2998 }
2999 else
3000 {
3001 Assert(rc == VINF_PGM_CACHED_PAGE);
3002# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3003 /* Paranoia - Verify address of each of the subpages are what they should be. */
3004 RTGCPHYS GCPhysSubPage = GCPhysPt;
3005 for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++, GCPhysSubPage += GUEST_PAGE_SIZE)
3006 {
3007 PPGMPAGE pSubPage;
3008 rc = pgmPhysGetPageEx(pVM, GCPhysSubPage, &pSubPage);
3009 AssertRC(rc);
3010 AssertMsg( PGM_PAGE_GET_HCPHYS(pSubPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte])
3011 || !SHW_PTE_IS_P(pPt->a[iPte]),
3012 ("PGM 2M page and shadow PTE conflict. GCPhysSubPage=%RGp Page=%RHp Shw=%RHp\n",
3013 GCPhysSubPage, PGM_PAGE_GET_HCPHYS(pSubPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3014 }
3015# endif
3016 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3017 }
3018
3019 /* Save the new PDE. */
3020 uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3021 Pde.u = pShwPage->Core.Key | fShwPdeFlags;
3022 Assert(!(Pde.u & EPT_E_LEAF));
3023 Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3024 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3025 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3026 Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3027 return rc;
3028 }
3029 }
3030# endif /* PGM_WITH_LARGE_PAGES */
3031
3032 /*
3033 * Allocate & map the shadow page table.
3034 */
3035 PSHWPT pPt;
3036 PPGMPOOLPAGE pShwPage;
3037
3038 RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE_PG_MASK;
3039 rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_PT, PGMPOOLACCESS_DONTCARE,
3040 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
3041 if ( rc == VINF_SUCCESS
3042 || rc == VINF_PGM_CACHED_PAGE)
3043 { /* likely */ }
3044 else
3045 {
3046 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3047 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3048 }
3049
3050 pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3051 Assert(pPt);
3052 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
3053
3054 if (rc == VINF_SUCCESS)
3055 {
3056 /* Sync the page we've already translated through SLAT. */
3057 const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3058 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
3059 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3060
3061 /* Sync the rest of page table (expensive but might be cheaper than nested-guest VM-exits in hardware). */
3062 for (unsigned iPteCur = 0; iPteCur < RT_ELEMENTS(pPt->a); iPteCur++)
3063 {
3064 if (iPteCur != iPte)
3065 {
3066 PGMPTWALKGST GstWalkPt;
3067 PGMPTWALK WalkPt;
3068 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
3069 GCPhysNestedPage |= (iPteCur << GUEST_PAGE_SHIFT);
3070 int const rc2 = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/,
3071 &WalkPt, &GstWalkPt);
3072 if (RT_SUCCESS(rc2))
3073 {
3074 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPteCur], WalkPt.GCPhys, pShwPage, iPteCur, &GstWalkPt);
3075 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", GstWalkPt.u.Ept.Pte.u, pPt->a[iPteCur].u, iPteCur));
3076 }
3077 else
3078 {
3079 /*
3080 * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
3081 * Ensure the shadow tables entry is not-present.
3082 */
3083 /** @todo We currently don't configure these to cause EPT misconfigs but rather trap
3084 * them using EPT violations and walk the guest EPT tables to determine
3085 * whether they are EPT misconfigs VM-exits for the nested-hypervisor. We
3086 * could optimize this by using a specific combination of reserved bits
3087 * which we could immediately identify as EPT misconfigs of the
3088 * nested-hypervisor without having to walk its EPT tables. However, tracking
3089 * non-present entries might be tricky...
3090 */
3091 AssertMsg(!pPt->a[iPteCur].u, ("%RX64\n", pPt->a[iPteCur].u));
3092 }
3093 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3094 break;
3095 }
3096 }
3097 }
3098 else
3099 {
3100 Assert(rc == VINF_PGM_CACHED_PAGE);
3101# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3102 /* Paranoia - Verify address of the page is what it should be. */
3103 PPGMPAGE pPage;
3104 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
3105 AssertRC(rc);
3106 const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3107 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]) || !SHW_PTE_IS_P(pPt->a[iPte]),
3108 ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp Page=%RHp Shw=%RHp\n",
3109 GCPhysNestedPage, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3110 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [cache]\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3111# endif
3112 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3113 }
3114
3115 /* Save the new PDE. */
3116 uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3117 Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF));
3118 Assert(!(pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3119 Pde.u = pShwPage->Core.Key | fShwPdeFlags;
3120 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3121 Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3122
3123 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3124 return rc;
3125}
3126
3127#endif /* !IN_RING3 && VBOX_WITH_NESTED_HWVIRT_VMX_EPT && PGM_SHW_TYPE == PGM_TYPE_EPT*/
3128#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
3129
3130/**
3131 * Handle dirty bit tracking faults.
3132 *
3133 * @returns VBox status code.
3134 * @param pVCpu The cross context virtual CPU structure.
3135 * @param uErr Page fault error code.
3136 * @param pPdeSrc Guest page directory entry.
3137 * @param pPdeDst Shadow page directory entry.
3138 * @param GCPtrPage Guest context page address.
3139 */
3140static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
3141 RTGCPTR GCPtrPage)
3142{
3143 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3144 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3145 NOREF(uErr);
3146
3147 PGM_LOCK_ASSERT_OWNER(pVM);
3148
3149 /*
3150 * Handle big page.
3151 */
3152 if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
3153 {
3154 if ((pPdeDst->u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3155 {
3156 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3157 Assert(pPdeSrc->u & X86_PDE_RW);
3158
3159 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
3160 * fault again and take this path to only invalidate the entry (see below). */
3161 SHWPDE PdeDst = *pPdeDst;
3162 PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY;
3163 PdeDst.u |= X86_PDE_RW | X86_PDE_A;
3164 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3165 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
3166 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3167 }
3168
3169# ifdef IN_RING0
3170 /* Check for stale TLB entry; only applies to the SMP guest case. */
3171 if ( pVM->cCpus > 1
3172 && (pPdeDst->u & (X86_PDE_P | X86_PDE_RW | X86_PDE_A)) == (X86_PDE_P | X86_PDE_RW | X86_PDE_A))
3173 {
3174 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3175 if (pShwPage)
3176 {
3177 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3178 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3179 if (SHW_PTE_IS_P_RW(*pPteDst))
3180 {
3181 /* Stale TLB entry. */
3182 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3183 PGM_INVL_PG(pVCpu, GCPtrPage);
3184 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3185 }
3186 }
3187 }
3188# endif /* IN_RING0 */
3189 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3190 }
3191
3192 /*
3193 * Map the guest page table.
3194 */
3195 PGSTPT pPTSrc;
3196 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
3197 AssertRCReturn(rc, rc);
3198
3199 if (SHW_PDE_IS_P(*pPdeDst))
3200 {
3201 GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
3202 const GSTPTE PteSrc = *pPteSrc;
3203
3204 /*
3205 * Map shadow page table.
3206 */
3207 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3208 if (pShwPage)
3209 {
3210 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3211 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3212 if (SHW_PTE_IS_P(*pPteDst)) /** @todo Optimize accessed bit emulation? */
3213 {
3214 if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
3215 {
3216 PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
3217 SHWPTE PteDst = *pPteDst;
3218
3219 LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
3220 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3221
3222 Assert(PteSrc.u & X86_PTE_RW);
3223
3224 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB
3225 * entry will not harm; write access will simply fault again and
3226 * take this path to only invalidate the entry.
3227 */
3228 if (RT_LIKELY(pPage))
3229 {
3230 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3231 {
3232 //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
3233 Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
3234 /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
3235 SHW_PTE_SET_RO(PteDst);
3236 }
3237 else
3238 {
3239 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
3240 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
3241 {
3242 rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
3243 AssertRC(rc);
3244 }
3245 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
3246 SHW_PTE_SET_RW(PteDst);
3247 else
3248 {
3249 /* Still applies to shared pages. */
3250 Assert(!PGM_PAGE_IS_ZERO(pPage));
3251 SHW_PTE_SET_RO(PteDst);
3252 }
3253 }
3254 }
3255 else
3256 SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
3257
3258 SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
3259 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
3260 PGM_INVL_PG(pVCpu, GCPtrPage);
3261 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3262 }
3263
3264# ifdef IN_RING0
3265 /* Check for stale TLB entry; only applies to the SMP guest case. */
3266 if ( pVM->cCpus > 1
3267 && SHW_PTE_IS_RW(*pPteDst)
3268 && SHW_PTE_IS_A(*pPteDst))
3269 {
3270 /* Stale TLB entry. */
3271 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3272 PGM_INVL_PG(pVCpu, GCPtrPage);
3273 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3274 }
3275# endif
3276 }
3277 }
3278 else
3279 AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
3280 }
3281
3282 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3283}
3284
3285#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
3286
3287/**
3288 * Sync a shadow page table.
3289 *
3290 * The shadow page table is not present in the shadow PDE.
3291 *
3292 * Handles mapping conflicts.
3293 *
3294 * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
3295 * conflict), and Trap0eHandler.
3296 *
3297 * A precondition for this method is that the shadow PDE is not present. The
3298 * caller must take the PGM lock before checking this and continue to hold it
3299 * when calling this method.
3300 *
3301 * @returns VBox status code.
3302 * @param pVCpu The cross context virtual CPU structure.
3303 * @param iPDSrc Page directory index.
3304 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
3305 * Assume this is a temporary mapping.
3306 * @param GCPtrPage GC Pointer of the page that caused the fault
3307 */
3308static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
3309{
3310 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3311 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3312
3313#if 0 /* rarely useful; leave for debugging. */
3314 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
3315#endif
3316 LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
3317
3318 PGM_LOCK_ASSERT_OWNER(pVM);
3319
3320#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3321 || PGM_GST_TYPE == PGM_TYPE_PAE \
3322 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
3323 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3324 && PGM_SHW_TYPE != PGM_TYPE_NONE
3325 int rc = VINF_SUCCESS;
3326
3327 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3328
3329 /*
3330 * Some input validation first.
3331 */
3332 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
3333
3334 /*
3335 * Get the relevant shadow PDE entry.
3336 */
3337# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3338 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
3339 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3340
3341 /* Fetch the pgm pool shadow descriptor. */
3342 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3343 Assert(pShwPde);
3344
3345# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3346 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3347 PPGMPOOLPAGE pShwPde = NULL;
3348 PX86PDPAE pPDDst;
3349 PSHWPDE pPdeDst;
3350
3351 /* Fetch the pgm pool shadow descriptor. */
3352 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3353 AssertRCSuccessReturn(rc, rc);
3354 Assert(pShwPde);
3355
3356 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3357 pPdeDst = &pPDDst->a[iPDDst];
3358
3359# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3360 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3361 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3362 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3363 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
3364 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3365 AssertRCSuccessReturn(rc, rc);
3366 Assert(pPDDst);
3367 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3368
3369# endif
3370 SHWPDE PdeDst = *pPdeDst;
3371
3372# if PGM_GST_TYPE == PGM_TYPE_AMD64
3373 /* Fetch the pgm pool shadow descriptor. */
3374 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3375 Assert(pShwPde);
3376# endif
3377
3378 Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/
3379
3380 /*
3381 * Sync the page directory entry.
3382 */
3383 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3384 const bool fPageTable = !(PdeSrc.u & X86_PDE_PS) || !GST_IS_PSE_ACTIVE(pVCpu);
3385 if ( (PdeSrc.u & X86_PDE_P)
3386 && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
3387 {
3388 /*
3389 * Allocate & map the page table.
3390 */
3391 PSHWPT pPTDst;
3392 PPGMPOOLPAGE pShwPage;
3393 RTGCPHYS GCPhys;
3394 if (fPageTable)
3395 {
3396 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
3397# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3398 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3399 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
3400# endif
3401 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
3402 pShwPde->idx, iPDDst, false /*fLockPage*/,
3403 &pShwPage);
3404 }
3405 else
3406 {
3407 PGMPOOLACCESS enmAccess;
3408# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
3409 const bool fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu);
3410# else
3411 const bool fNoExecute = false;
3412# endif
3413
3414 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3415# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3416 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
3417 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
3418# endif
3419 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
3420 if (PdeSrc.u & X86_PDE_US)
3421 {
3422 if (PdeSrc.u & X86_PDE_RW)
3423 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
3424 else
3425 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
3426 }
3427 else
3428 {
3429 if (PdeSrc.u & X86_PDE_RW)
3430 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
3431 else
3432 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
3433 }
3434 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
3435 pShwPde->idx, iPDDst, false /*fLockPage*/,
3436 &pShwPage);
3437 }
3438 if (rc == VINF_SUCCESS)
3439 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3440 else if (rc == VINF_PGM_CACHED_PAGE)
3441 {
3442 /*
3443 * The PT was cached, just hook it up.
3444 */
3445 if (fPageTable)
3446 PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3447 else
3448 {
3449 PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3450 /* (see explanation and assumptions further down.) */
3451 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
3452 {
3453 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3454 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
3455 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3456 }
3457 }
3458 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3459 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3460 return VINF_SUCCESS;
3461 }
3462 else
3463 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3464 /** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
3465 * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
3466 * irrelevant at this point. */
3467 PdeDst.u &= X86_PDE_AVL_MASK;
3468 PdeDst.u |= pShwPage->Core.Key;
3469
3470 /*
3471 * Page directory has been accessed (this is a fault situation, remember).
3472 */
3473 /** @todo
3474 * Well, when the caller is PrefetchPage or InvalidatePage is isn't a
3475 * fault situation. What's more, the Trap0eHandler has already set the
3476 * accessed bit. So, it's actually just VerifyAccessSyncPage which
3477 * might need setting the accessed flag.
3478 *
3479 * The best idea is to leave this change to the caller and add an
3480 * assertion that it's set already. */
3481 pPDSrc->a[iPDSrc].u |= X86_PDE_A;
3482 if (fPageTable)
3483 {
3484 /*
3485 * Page table - 4KB.
3486 *
3487 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
3488 */
3489 Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
3490 GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u));
3491 PGSTPT pPTSrc;
3492 rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
3493 if (RT_SUCCESS(rc))
3494 {
3495 /*
3496 * Start by syncing the page directory entry so CSAM's TLB trick works.
3497 */
3498 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
3499 | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3500 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3501 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3502
3503 /*
3504 * Directory/page user or supervisor privilege: (same goes for read/write)
3505 *
3506 * Directory Page Combined
3507 * U/S U/S U/S
3508 * 0 0 0
3509 * 0 1 0
3510 * 1 0 0
3511 * 1 1 1
3512 *
3513 * Simple AND operation. Table listed for completeness.
3514 *
3515 */
3516 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K));
3517# ifdef PGM_SYNC_N_PAGES
3518 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3519 unsigned iPTDst = iPTBase;
3520 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
3521 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
3522 iPTDst = 0;
3523 else
3524 iPTDst -= PGM_SYNC_NR_PAGES / 2;
3525# else /* !PGM_SYNC_N_PAGES */
3526 unsigned iPTDst = 0;
3527 const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
3528# endif /* !PGM_SYNC_N_PAGES */
3529 RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
3530 | ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT);
3531# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3532 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3533 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
3534# else
3535 const unsigned offPTSrc = 0;
3536# endif
3537 for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE)
3538 {
3539 const unsigned iPTSrc = iPTDst + offPTSrc;
3540 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
3541 if (PteSrc.u & X86_PTE_P)
3542 {
3543 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
3544 Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
3545 GCPtrCur,
3546 PteSrc.u & X86_PTE_P,
3547 !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
3548 !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
3549 (uint64_t)PteSrc.u,
3550 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
3551 (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
3552 }
3553 /* else: the page table was cleared by the pool */
3554 } /* for PTEs */
3555 }
3556 }
3557 else
3558 {
3559 /*
3560 * Big page - 2/4MB.
3561 *
3562 * We'll walk the ram range list in parallel and optimize lookups.
3563 * We will only sync one shadow page table at a time.
3564 */
3565 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M));
3566
3567 /**
3568 * @todo It might be more efficient to sync only a part of the 4MB
3569 * page (similar to what we do for 4KB PDs).
3570 */
3571
3572 /*
3573 * Start by syncing the page directory entry.
3574 */
3575 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
3576 | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3577
3578 /*
3579 * If the page is not flagged as dirty and is writable, then make it read-only
3580 * at PD level, so we can set the dirty bit when the page is modified.
3581 *
3582 * ASSUMES that page access handlers are implemented on page table entry level.
3583 * Thus we will first catch the dirty access and set PDE.D and restart. If
3584 * there is an access handler, we'll trap again and let it work on the problem.
3585 */
3586 /** @todo move the above stuff to a section in the PGM documentation. */
3587 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
3588 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
3589 {
3590 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3591 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
3592 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3593 }
3594 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3595 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3596
3597 /*
3598 * Fill the shadow page table.
3599 */
3600 /* Get address and flags from the source PDE. */
3601 SHWPTE PteDstBase;
3602 SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
3603
3604 /* Loop thru the entries in the shadow PT. */
3605 const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
3606 Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
3607 GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr,
3608 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
3609 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3610 unsigned iPTDst = 0;
3611 while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3612 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3613 {
3614 if (pRam && GCPhys >= pRam->GCPhys)
3615 {
3616# ifndef PGM_WITH_A20
3617 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
3618# endif
3619 do
3620 {
3621 /* Make shadow PTE. */
3622# ifdef PGM_WITH_A20
3623 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
3624# else
3625 PPGMPAGE pPage = &pRam->aPages[iHCPage];
3626# endif
3627 SHWPTE PteDst;
3628
3629# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3630 /* Try to make the page writable if necessary. */
3631 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3632 && ( PGM_PAGE_IS_ZERO(pPage)
3633 || ( SHW_PTE_IS_RW(PteDstBase)
3634 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3635# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3636 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3637# endif
3638# ifdef VBOX_WITH_PAGE_SHARING
3639 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3640# endif
3641 && !PGM_PAGE_IS_BALLOONED(pPage))
3642 )
3643 )
3644 {
3645 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3646 AssertRCReturn(rc, rc);
3647 if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3648 break;
3649 }
3650# endif
3651
3652 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3653 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, SHW_PTE_GET_U(PteDstBase), &PteDst);
3654 else if (PGM_PAGE_IS_BALLOONED(pPage))
3655 SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3656 else
3657 SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase));
3658
3659 /* Only map writable pages writable. */
3660 if ( SHW_PTE_IS_P_RW(PteDst)
3661 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3662 {
3663 /* Still applies to shared pages. */
3664 Assert(!PGM_PAGE_IS_ZERO(pPage));
3665 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3666 Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
3667 }
3668
3669 if (SHW_PTE_IS_P(PteDst))
3670 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3671
3672 /* commit it (not atomic, new table) */
3673 pPTDst->a[iPTDst] = PteDst;
3674 Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3675 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3676 SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3677
3678 /* advance */
3679 GCPhys += GUEST_PAGE_SIZE;
3680 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3681# ifndef PGM_WITH_A20
3682 iHCPage++;
3683# endif
3684 iPTDst++;
3685 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3686 && GCPhys <= pRam->GCPhysLast);
3687
3688 /* Advance ram range list. */
3689 while (pRam && GCPhys > pRam->GCPhysLast)
3690 pRam = pRam->CTX_SUFF(pNext);
3691 }
3692 else if (pRam)
3693 {
3694 Log(("Invalid pages at %RGp\n", GCPhys));
3695 do
3696 {
3697 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3698 GCPhys += GUEST_PAGE_SIZE;
3699 iPTDst++;
3700 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3701 && GCPhys < pRam->GCPhys);
3702 PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3703 }
3704 else
3705 {
3706 Log(("Invalid pages at %RGp (2)\n", GCPhys));
3707 for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3708 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3709 }
3710 } /* while more PTEs */
3711 } /* 4KB / 4MB */
3712 }
3713 else
3714 AssertRelease(!SHW_PDE_IS_P(PdeDst));
3715
3716 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3717 if (RT_FAILURE(rc))
3718 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3719 return rc;
3720
3721#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
3722 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
3723 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
3724 && PGM_SHW_TYPE != PGM_TYPE_NONE
3725 NOREF(iPDSrc); NOREF(pPDSrc);
3726
3727 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3728
3729 /*
3730 * Validate input a little bit.
3731 */
3732 int rc = VINF_SUCCESS;
3733# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3734 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3735 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3736
3737 /* Fetch the pgm pool shadow descriptor. */
3738 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3739 Assert(pShwPde);
3740
3741# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3742 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3743 PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3744 PX86PDPAE pPDDst;
3745 PSHWPDE pPdeDst;
3746
3747 /* Fetch the pgm pool shadow descriptor. */
3748 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3749 AssertRCSuccessReturn(rc, rc);
3750 Assert(pShwPde);
3751
3752 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3753 pPdeDst = &pPDDst->a[iPDDst];
3754
3755# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3756 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3757 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3758 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3759 PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3760 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3761 AssertRCSuccessReturn(rc, rc);
3762 Assert(pPDDst);
3763 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3764
3765 /* Fetch the pgm pool shadow descriptor. */
3766 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3767 Assert(pShwPde);
3768
3769# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3770 const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3771 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3772 PEPTPD pPDDst;
3773 PEPTPDPT pPdptDst;
3774
3775 rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3776 if (rc != VINF_SUCCESS)
3777 {
3778 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3779 AssertRC(rc);
3780 return rc;
3781 }
3782 Assert(pPDDst);
3783 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3784
3785 /* Fetch the pgm pool shadow descriptor. */
3786 /** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
3787 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3788 Assert(pShwPde);
3789# endif
3790 SHWPDE PdeDst = *pPdeDst;
3791
3792 Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3793
3794# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3795 if (BTH_IS_NP_ACTIVE(pVM))
3796 {
3797 Assert(!VM_IS_NEM_ENABLED(pVM));
3798
3799 /* Check if we allocated a big page before for this 2 MB range. */
3800 PPGMPAGE pPage;
3801 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3802 if (RT_SUCCESS(rc))
3803 {
3804 RTHCPHYS HCPhys = NIL_RTHCPHYS;
3805 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3806 {
3807 if (PGM_A20_IS_ENABLED(pVCpu))
3808 {
3809 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3810 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3811 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3812 }
3813 else
3814 {
3815 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3816 pVM->pgm.s.cLargePagesDisabled++;
3817 }
3818 }
3819 else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3820 && PGM_A20_IS_ENABLED(pVCpu))
3821 {
3822 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3823 rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3824 if (RT_SUCCESS(rc))
3825 {
3826 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3827 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3828 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3829 }
3830 }
3831 else if ( PGMIsUsingLargePages(pVM)
3832 && PGM_A20_IS_ENABLED(pVCpu))
3833 {
3834 rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3835 if (RT_SUCCESS(rc))
3836 {
3837 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3838 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3839 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3840 }
3841 else
3842 LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3843 }
3844
3845 if (HCPhys != NIL_RTHCPHYS)
3846 {
3847# if PGM_SHW_TYPE == PGM_TYPE_EPT
3848 PdeDst.u = HCPhys | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_LEAF | EPT_E_IGNORE_PAT | EPT_E_MEMTYPE_WB
3849 | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
3850# else
3851 PdeDst.u = HCPhys | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PS
3852 | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */;
3853# endif
3854 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3855
3856 Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3857 /* Add a reference to the first page only. */
3858 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3859
3860 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3861 return VINF_SUCCESS;
3862 }
3863 }
3864 }
3865# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
3866
3867 /*
3868 * Allocate & map the page table.
3869 */
3870 PSHWPT pPTDst;
3871 PPGMPOOLPAGE pShwPage;
3872 RTGCPHYS GCPhys;
3873
3874 /* Virtual address = physical address */
3875 GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3876 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3877 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/,
3878 &pShwPage);
3879 if ( rc == VINF_SUCCESS
3880 || rc == VINF_PGM_CACHED_PAGE)
3881 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3882 else
3883 {
3884 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3885 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3886 }
3887
3888 if (rc == VINF_SUCCESS)
3889 {
3890 /* New page table; fully set it up. */
3891 Assert(pPTDst);
3892
3893 /* Mask away the page offset. */
3894 GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
3895
3896 for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3897 {
3898 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3899 | (iPTDst << GUEST_PAGE_SHIFT));
3900
3901 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3902 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3903 GCPtrCurPage,
3904 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3905 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3906
3907 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3908 break;
3909 }
3910 }
3911 else
3912 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3913
3914 /* Save the new PDE. */
3915# if PGM_SHW_TYPE == PGM_TYPE_EPT
3916 PdeDst.u = pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE
3917 | (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
3918# else
3919 PdeDst.u = pShwPage->Core.Key | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A
3920 | (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */);
3921# endif
3922 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3923
3924 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3925 if (RT_FAILURE(rc))
3926 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3927 return rc;
3928
3929#else
3930 NOREF(iPDSrc); NOREF(pPDSrc);
3931 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3932 return VERR_PGM_NOT_USED_IN_MODE;
3933#endif
3934}
3935
3936
3937
3938/**
3939 * Prefetch a page/set of pages.
3940 *
3941 * Typically used to sync commonly used pages before entering raw mode
3942 * after a CR3 reload.
3943 *
3944 * @returns VBox status code.
3945 * @param pVCpu The cross context virtual CPU structure.
3946 * @param GCPtrPage Page to invalidate.
3947 */
3948PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
3949{
3950#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3951 || PGM_GST_TYPE == PGM_TYPE_REAL \
3952 || PGM_GST_TYPE == PGM_TYPE_PROT \
3953 || PGM_GST_TYPE == PGM_TYPE_PAE \
3954 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3955 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3956 && PGM_SHW_TYPE != PGM_TYPE_NONE
3957 /*
3958 * Check that all Guest levels thru the PDE are present, getting the
3959 * PD and PDE in the processes.
3960 */
3961 int rc = VINF_SUCCESS;
3962# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3963# if PGM_GST_TYPE == PGM_TYPE_32BIT
3964 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3965 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3966# elif PGM_GST_TYPE == PGM_TYPE_PAE
3967 unsigned iPDSrc;
3968 X86PDPE PdpeSrc;
3969 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3970 if (!pPDSrc)
3971 return VINF_SUCCESS; /* not present */
3972# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3973 unsigned iPDSrc;
3974 PX86PML4E pPml4eSrc;
3975 X86PDPE PdpeSrc;
3976 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3977 if (!pPDSrc)
3978 return VINF_SUCCESS; /* not present */
3979# endif
3980 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3981# else
3982 PGSTPD pPDSrc = NULL;
3983 const unsigned iPDSrc = 0;
3984 GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
3985# endif
3986
3987 if ((PdeSrc.u & (X86_PDE_P | X86_PDE_A)) == (X86_PDE_P | X86_PDE_A))
3988 {
3989 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3990 PGM_LOCK_VOID(pVM);
3991
3992# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3993 const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3994# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3995 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3996 PX86PDPAE pPDDst;
3997 X86PDEPAE PdeDst;
3998# if PGM_GST_TYPE != PGM_TYPE_PAE
3999 X86PDPE PdpeSrc;
4000
4001 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
4002 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
4003# endif
4004 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
4005 if (rc != VINF_SUCCESS)
4006 {
4007 PGM_UNLOCK(pVM);
4008 AssertRC(rc);
4009 return rc;
4010 }
4011 Assert(pPDDst);
4012 PdeDst = pPDDst->a[iPDDst];
4013
4014# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4015 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4016 PX86PDPAE pPDDst;
4017 X86PDEPAE PdeDst;
4018
4019# if PGM_GST_TYPE == PGM_TYPE_PROT
4020 /* AMD-V nested paging */
4021 X86PML4E Pml4eSrc;
4022 X86PDPE PdpeSrc;
4023 PX86PML4E pPml4eSrc = &Pml4eSrc;
4024
4025 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
4026 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
4027 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
4028# endif
4029
4030 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
4031 if (rc != VINF_SUCCESS)
4032 {
4033 PGM_UNLOCK(pVM);
4034 AssertRC(rc);
4035 return rc;
4036 }
4037 Assert(pPDDst);
4038 PdeDst = pPDDst->a[iPDDst];
4039# endif
4040 if (!(PdeDst.u & X86_PDE_P))
4041 {
4042 /** @todo r=bird: This guy will set the A bit on the PDE,
4043 * probably harmless. */
4044 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
4045 }
4046 else
4047 {
4048 /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
4049 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
4050 * makes no sense to prefetch more than one page.
4051 */
4052 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
4053 if (RT_SUCCESS(rc))
4054 rc = VINF_SUCCESS;
4055 }
4056 PGM_UNLOCK(pVM);
4057 }
4058 return rc;
4059
4060#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4061 NOREF(pVCpu); NOREF(GCPtrPage);
4062 return VINF_SUCCESS; /* ignore */
4063#else
4064 AssertCompile(0);
4065#endif
4066}
4067
4068
4069
4070
4071/**
4072 * Syncs a page during a PGMVerifyAccess() call.
4073 *
4074 * @returns VBox status code (informational included).
4075 * @param pVCpu The cross context virtual CPU structure.
4076 * @param GCPtrPage The address of the page to sync.
4077 * @param fPage The effective guest page flags.
4078 * @param uErr The trap error code.
4079 * @remarks This will normally never be called on invalid guest page
4080 * translation entries.
4081 */
4082PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
4083{
4084 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4085
4086 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
4087 RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
4088
4089 Assert(!pVM->pgm.s.fNestedPaging);
4090#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
4091 || PGM_GST_TYPE == PGM_TYPE_REAL \
4092 || PGM_GST_TYPE == PGM_TYPE_PROT \
4093 || PGM_GST_TYPE == PGM_TYPE_PAE \
4094 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
4095 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
4096 && PGM_SHW_TYPE != PGM_TYPE_NONE
4097
4098 /*
4099 * Get guest PD and index.
4100 */
4101 /** @todo Performance: We've done all this a jiffy ago in the
4102 * PGMGstGetPage call. */
4103# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4104# if PGM_GST_TYPE == PGM_TYPE_32BIT
4105 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
4106 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4107
4108# elif PGM_GST_TYPE == PGM_TYPE_PAE
4109 unsigned iPDSrc = 0;
4110 X86PDPE PdpeSrc;
4111 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
4112 if (RT_UNLIKELY(!pPDSrc))
4113 {
4114 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
4115 return VINF_EM_RAW_GUEST_TRAP;
4116 }
4117
4118# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4119 unsigned iPDSrc = 0; /* shut up gcc */
4120 PX86PML4E pPml4eSrc = NULL; /* ditto */
4121 X86PDPE PdpeSrc;
4122 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
4123 if (RT_UNLIKELY(!pPDSrc))
4124 {
4125 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
4126 return VINF_EM_RAW_GUEST_TRAP;
4127 }
4128# endif
4129
4130# else /* !PGM_WITH_PAGING */
4131 PGSTPD pPDSrc = NULL;
4132 const unsigned iPDSrc = 0;
4133# endif /* !PGM_WITH_PAGING */
4134 int rc = VINF_SUCCESS;
4135
4136 PGM_LOCK_VOID(pVM);
4137
4138 /*
4139 * First check if the shadow pd is present.
4140 */
4141# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4142 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
4143
4144# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4145 PX86PDEPAE pPdeDst;
4146 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4147 PX86PDPAE pPDDst;
4148# if PGM_GST_TYPE != PGM_TYPE_PAE
4149 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
4150 X86PDPE PdpeSrc;
4151 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
4152# endif
4153 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
4154 if (rc != VINF_SUCCESS)
4155 {
4156 PGM_UNLOCK(pVM);
4157 AssertRC(rc);
4158 return rc;
4159 }
4160 Assert(pPDDst);
4161 pPdeDst = &pPDDst->a[iPDDst];
4162
4163# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4164 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4165 PX86PDPAE pPDDst;
4166 PX86PDEPAE pPdeDst;
4167
4168# if PGM_GST_TYPE == PGM_TYPE_PROT
4169 /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
4170 X86PML4E Pml4eSrc;
4171 X86PDPE PdpeSrc;
4172 PX86PML4E pPml4eSrc = &Pml4eSrc;
4173 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
4174 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
4175# endif
4176
4177 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
4178 if (rc != VINF_SUCCESS)
4179 {
4180 PGM_UNLOCK(pVM);
4181 AssertRC(rc);
4182 return rc;
4183 }
4184 Assert(pPDDst);
4185 pPdeDst = &pPDDst->a[iPDDst];
4186# endif
4187
4188 if (!(pPdeDst->u & X86_PDE_P))
4189 {
4190 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
4191 if (rc != VINF_SUCCESS)
4192 {
4193 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
4194 PGM_UNLOCK(pVM);
4195 AssertRC(rc);
4196 return rc;
4197 }
4198 }
4199
4200# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4201 /* Check for dirty bit fault */
4202 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
4203 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
4204 Log(("PGMVerifyAccess: success (dirty)\n"));
4205 else
4206# endif
4207 {
4208# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4209 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
4210# else
4211 GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
4212# endif
4213
4214 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
4215 if (uErr & X86_TRAP_PF_US)
4216 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
4217 else /* supervisor */
4218 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
4219
4220 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
4221 if (RT_SUCCESS(rc))
4222 {
4223 /* Page was successfully synced */
4224 Log2(("PGMVerifyAccess: success (sync)\n"));
4225 rc = VINF_SUCCESS;
4226 }
4227 else
4228 {
4229 Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
4230 rc = VINF_EM_RAW_GUEST_TRAP;
4231 }
4232 }
4233 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
4234 PGM_UNLOCK(pVM);
4235 return rc;
4236
4237#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
4238
4239 AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
4240 return VERR_PGM_NOT_USED_IN_MODE;
4241#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
4242}
4243
4244
4245/**
4246 * Syncs the paging hierarchy starting at CR3.
4247 *
4248 * @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
4249 * informational status codes.
4250 * @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
4251 * the VMM into guest context.
4252 * @param pVCpu The cross context virtual CPU structure.
4253 * @param cr0 Guest context CR0 register.
4254 * @param cr3 Guest context CR3 register. Not subjected to the A20
4255 * mask.
4256 * @param cr4 Guest context CR4 register.
4257 * @param fGlobal Including global page directories or not
4258 */
4259PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
4260{
4261 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4262 NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
4263
4264 LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
4265
4266#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
4267# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4268 PGM_LOCK_VOID(pVM);
4269 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4270 if (pPool->cDirtyPages)
4271 pgmPoolResetDirtyPages(pVM);
4272 PGM_UNLOCK(pVM);
4273# endif
4274#endif /* !NESTED && !EPT */
4275
4276#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4277 /*
4278 * Nested / EPT / None - No work.
4279 */
4280 return VINF_SUCCESS;
4281
4282#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4283 /*
4284 * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
4285 * out the shadow parts when the guest modifies its tables.
4286 */
4287 return VINF_SUCCESS;
4288
4289#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4290
4291 return VINF_SUCCESS;
4292#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4293}
4294
4295
4296
4297
4298#ifdef VBOX_STRICT
4299
4300/**
4301 * Checks that the shadow page table is in sync with the guest one.
4302 *
4303 * @returns The number of errors.
4304 * @param pVCpu The cross context virtual CPU structure.
4305 * @param cr3 Guest context CR3 register.
4306 * @param cr4 Guest context CR4 register.
4307 * @param GCPtr Where to start. Defaults to 0.
4308 * @param cb How much to check. Defaults to everything.
4309 */
4310PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
4311{
4312 NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
4313#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4314 return 0;
4315#else
4316 unsigned cErrors = 0;
4317 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4318 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
4319
4320# if PGM_GST_TYPE == PGM_TYPE_PAE
4321 /** @todo currently broken; crashes below somewhere */
4322 AssertFailed();
4323# endif
4324
4325# if PGM_GST_TYPE == PGM_TYPE_32BIT \
4326 || PGM_GST_TYPE == PGM_TYPE_PAE \
4327 || PGM_GST_TYPE == PGM_TYPE_AMD64
4328
4329 bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
4330 PPGMCPU pPGM = &pVCpu->pgm.s;
4331 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
4332 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
4333# ifndef IN_RING0
4334 RTHCPHYS HCPhys; /* general usage. */
4335# endif
4336 int rc;
4337
4338 /*
4339 * Check that the Guest CR3 and all its mappings are correct.
4340 */
4341 AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
4342 ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
4343 false);
4344# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
4345# if 0
4346# if PGM_GST_TYPE == PGM_TYPE_32BIT
4347 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
4348# else
4349 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
4350# endif
4351 AssertRCReturn(rc, 1);
4352 HCPhys = NIL_RTHCPHYS;
4353 rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
4354 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
4355# endif
4356# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
4357 pgmGstGet32bitPDPtr(pVCpu);
4358 RTGCPHYS GCPhys;
4359 rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
4360 AssertRCReturn(rc, 1);
4361 AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
4362# endif
4363# endif /* !IN_RING0 */
4364
4365 /*
4366 * Get and check the Shadow CR3.
4367 */
4368# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4369 unsigned cPDEs = X86_PG_ENTRIES;
4370 unsigned cIncrement = X86_PG_ENTRIES * GUEST_PAGE_SIZE;
4371# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4372# if PGM_GST_TYPE == PGM_TYPE_32BIT
4373 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
4374# else
4375 unsigned cPDEs = X86_PG_PAE_ENTRIES;
4376# endif
4377 unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4378# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4379 unsigned cPDEs = X86_PG_PAE_ENTRIES;
4380 unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4381# endif
4382 if (cb != ~(RTGCPTR)0)
4383 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
4384
4385/** @todo call the other two PGMAssert*() functions. */
4386
4387# if PGM_GST_TYPE == PGM_TYPE_AMD64
4388 unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4389
4390 for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
4391 {
4392 PPGMPOOLPAGE pShwPdpt = NULL;
4393 PX86PML4E pPml4eSrc;
4394 PX86PML4E pPml4eDst;
4395 RTGCPHYS GCPhysPdptSrc;
4396
4397 pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
4398 pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
4399
4400 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
4401 if (!(pPml4eDst->u & X86_PML4E_P))
4402 {
4403 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4404 continue;
4405 }
4406
4407 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
4408 GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
4409
4410 if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P))
4411 {
4412 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4413 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4414 cErrors++;
4415 continue;
4416 }
4417
4418 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
4419 {
4420 AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
4421 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4422 cErrors++;
4423 continue;
4424 }
4425
4426 if ( (pPml4eDst->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX))
4427 != (pPml4eSrc->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX)))
4428 {
4429 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4430 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4431 cErrors++;
4432 continue;
4433 }
4434# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4435 {
4436# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4437
4438# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
4439 /*
4440 * Check the PDPTEs too.
4441 */
4442 unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
4443
4444 for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
4445 {
4446 unsigned iPDSrc = 0; /* initialized to shut up gcc */
4447 PPGMPOOLPAGE pShwPde = NULL;
4448 PX86PDPE pPdpeDst;
4449 RTGCPHYS GCPhysPdeSrc;
4450 X86PDPE PdpeSrc;
4451 PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
4452# if PGM_GST_TYPE == PGM_TYPE_PAE
4453 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
4454 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
4455# else
4456 PX86PML4E pPml4eSrcIgn;
4457 PX86PDPT pPdptDst;
4458 PX86PDPAE pPDDst;
4459 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
4460
4461 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
4462 if (rc != VINF_SUCCESS)
4463 {
4464 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
4465 GCPtr += 512 * _2M;
4466 continue; /* next PDPTE */
4467 }
4468 Assert(pPDDst);
4469# endif
4470 Assert(iPDSrc == 0);
4471
4472 pPdpeDst = &pPdptDst->a[iPdpt];
4473
4474 if (!(pPdpeDst->u & X86_PDPE_P))
4475 {
4476 GCPtr += 512 * _2M;
4477 continue; /* next PDPTE */
4478 }
4479
4480 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
4481 GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
4482
4483 if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P))
4484 {
4485 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4486 GCPtr += 512 * _2M;
4487 cErrors++;
4488 continue;
4489 }
4490
4491 if (GCPhysPdeSrc != pShwPde->GCPhys)
4492 {
4493# if PGM_GST_TYPE == PGM_TYPE_AMD64
4494 AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4495# else
4496 AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4497# endif
4498 GCPtr += 512 * _2M;
4499 cErrors++;
4500 continue;
4501 }
4502
4503# if PGM_GST_TYPE == PGM_TYPE_AMD64
4504 if ( (pPdpeDst->u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX))
4505 != (PdpeSrc.u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX)))
4506 {
4507 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4508 GCPtr += 512 * _2M;
4509 cErrors++;
4510 continue;
4511 }
4512# endif
4513
4514# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4515 {
4516# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4517# if PGM_GST_TYPE == PGM_TYPE_32BIT
4518 GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4519# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4520 PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
4521# endif
4522# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
4523 /*
4524 * Iterate the shadow page directory.
4525 */
4526 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
4527 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
4528
4529 for (;
4530 iPDDst < cPDEs;
4531 iPDDst++, GCPtr += cIncrement)
4532 {
4533# if PGM_SHW_TYPE == PGM_TYPE_PAE
4534 const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4535# else
4536 const SHWPDE PdeDst = pPDDst->a[iPDDst];
4537# endif
4538 if ( (PdeDst.u & X86_PDE_P)
4539 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) )
4540 {
4541 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4542 PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4543 if (!pPoolPage)
4544 {
4545 AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4546 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4547 cErrors++;
4548 continue;
4549 }
4550 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4551
4552 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
4553 {
4554 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4555 GCPtr, (uint64_t)PdeDst.u));
4556 cErrors++;
4557 }
4558
4559 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
4560 {
4561 AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4562 GCPtr, (uint64_t)PdeDst.u));
4563 cErrors++;
4564 }
4565
4566 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4567 if (!(PdeSrc.u & X86_PDE_P))
4568 {
4569 AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4570 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4571 cErrors++;
4572 continue;
4573 }
4574
4575 if ( !(PdeSrc.u & X86_PDE_PS)
4576 || !fBigPagesSupported)
4577 {
4578 GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4579# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4580 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
4581# endif
4582 }
4583 else
4584 {
4585# if PGM_GST_TYPE == PGM_TYPE_32BIT
4586 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4587 {
4588 AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4589 GCPtr, (uint64_t)PdeSrc.u));
4590 cErrors++;
4591 continue;
4592 }
4593# endif
4594 GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4595# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4596 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4597# endif
4598 }
4599
4600 if ( pPoolPage->enmKind
4601 != (!(PdeSrc.u & X86_PDE_PS) || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4602 {
4603 AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4604 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4605 cErrors++;
4606 }
4607
4608 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4609 if (!pPhysPage)
4610 {
4611 AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4612 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4613 cErrors++;
4614 continue;
4615 }
4616
4617 if (GCPhysGst != pPoolPage->GCPhys)
4618 {
4619 AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4620 GCPhysGst, pPoolPage->GCPhys, GCPtr));
4621 cErrors++;
4622 continue;
4623 }
4624
4625 if ( !(PdeSrc.u & X86_PDE_PS)
4626 || !fBigPagesSupported)
4627 {
4628 /*
4629 * Page Table.
4630 */
4631 const GSTPT *pPTSrc;
4632 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)),
4633 &pPTSrc);
4634 if (RT_FAILURE(rc))
4635 {
4636 AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4637 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4638 cErrors++;
4639 continue;
4640 }
4641 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
4642 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
4643 {
4644 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4645 // (This problem will go away when/if we shadow multiple CR3s.)
4646 AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4647 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4648 cErrors++;
4649 continue;
4650 }
4651 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4652 {
4653 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4654 GCPtr, (uint64_t)PdeDst.u));
4655 cErrors++;
4656 continue;
4657 }
4658
4659 /* iterate the page table. */
4660# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4661 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4662 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4663# else
4664 const unsigned offPTSrc = 0;
4665# endif
4666 for (unsigned iPT = 0, off = 0;
4667 iPT < RT_ELEMENTS(pPTDst->a);
4668 iPT++, off += GUEST_PAGE_SIZE)
4669 {
4670 const SHWPTE PteDst = pPTDst->a[iPT];
4671
4672 /* skip not-present and dirty tracked entries. */
4673 if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4674 continue;
4675 Assert(SHW_PTE_IS_P(PteDst));
4676
4677 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4678 if (!(PteSrc.u & X86_PTE_P))
4679 {
4680# ifdef IN_RING3
4681 PGMAssertHandlerAndFlagsInSync(pVM);
4682 DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE
4683 | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3,
4684 0, 0, UINT64_MAX, 99, NULL);
4685# endif
4686 AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4687 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4688 (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4689 cErrors++;
4690 continue;
4691 }
4692
4693 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
4694# if 1 /** @todo sync accessed bit properly... */
4695 fIgnoreFlags |= X86_PTE_A;
4696# endif
4697
4698 /* match the physical addresses */
4699 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4700 GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4701
4702# ifdef IN_RING3
4703 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4704 if (RT_FAILURE(rc))
4705 {
4706# if 0
4707 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4708 {
4709 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4710 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4711 cErrors++;
4712 continue;
4713 }
4714# endif
4715 }
4716 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4717 {
4718 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4719 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4720 cErrors++;
4721 continue;
4722 }
4723# endif
4724
4725 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4726 if (!pPhysPage)
4727 {
4728# if 0
4729 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4730 {
4731 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4732 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4733 cErrors++;
4734 continue;
4735 }
4736# endif
4737 if (SHW_PTE_IS_RW(PteDst))
4738 {
4739 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4740 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4741 cErrors++;
4742 }
4743 fIgnoreFlags |= X86_PTE_RW;
4744 }
4745 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4746 {
4747 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4748 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4749 cErrors++;
4750 continue;
4751 }
4752
4753 /* flags */
4754 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
4755 {
4756 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4757 {
4758 if (SHW_PTE_IS_RW(PteDst))
4759 {
4760 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4761 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4762 cErrors++;
4763 continue;
4764 }
4765 fIgnoreFlags |= X86_PTE_RW;
4766 }
4767 else
4768 {
4769 if ( SHW_PTE_IS_P(PteDst)
4770# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
4771 && !PGM_PAGE_IS_MMIO(pPhysPage)
4772# endif
4773 )
4774 {
4775 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4776 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4777 cErrors++;
4778 continue;
4779 }
4780 fIgnoreFlags |= X86_PTE_P;
4781 }
4782 }
4783 else
4784 {
4785 if ((PteSrc.u & (X86_PTE_RW | X86_PTE_D)) == X86_PTE_RW)
4786 {
4787 if (SHW_PTE_IS_RW(PteDst))
4788 {
4789 AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4790 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4791 cErrors++;
4792 continue;
4793 }
4794 if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4795 {
4796 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4797 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4798 cErrors++;
4799 continue;
4800 }
4801 if (SHW_PTE_IS_D(PteDst))
4802 {
4803 AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4804 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4805 cErrors++;
4806 }
4807# if 0 /** @todo sync access bit properly... */
4808 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4809 {
4810 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4811 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4812 cErrors++;
4813 }
4814 fIgnoreFlags |= X86_PTE_RW;
4815# else
4816 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4817# endif
4818 }
4819 else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4820 {
4821 /* access bit emulation (not implemented). */
4822 if ((PteSrc.u & X86_PTE_A) || SHW_PTE_IS_P(PteDst))
4823 {
4824 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4825 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4826 cErrors++;
4827 continue;
4828 }
4829 if (!SHW_PTE_IS_A(PteDst))
4830 {
4831 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4832 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4833 cErrors++;
4834 }
4835 fIgnoreFlags |= X86_PTE_P;
4836 }
4837# ifdef DEBUG_sandervl
4838 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4839# endif
4840 }
4841
4842 if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4843 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4844 )
4845 {
4846 AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4847 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4848 fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4849 cErrors++;
4850 continue;
4851 }
4852 } /* foreach PTE */
4853 }
4854 else
4855 {
4856 /*
4857 * Big Page.
4858 */
4859 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4860 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
4861 {
4862 if (PdeDst.u & X86_PDE_RW)
4863 {
4864 AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4865 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4866 cErrors++;
4867 continue;
4868 }
4869 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4870 {
4871 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4872 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4873 cErrors++;
4874 continue;
4875 }
4876# if 0 /** @todo sync access bit properly... */
4877 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4878 {
4879 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4880 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4881 cErrors++;
4882 }
4883 fIgnoreFlags |= X86_PTE_RW;
4884# else
4885 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4886# endif
4887 }
4888 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4889 {
4890 /* access bit emulation (not implemented). */
4891 if ((PdeSrc.u & X86_PDE_A) || SHW_PDE_IS_P(PdeDst))
4892 {
4893 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4894 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4895 cErrors++;
4896 continue;
4897 }
4898 if (!SHW_PDE_IS_A(PdeDst))
4899 {
4900 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4901 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4902 cErrors++;
4903 }
4904 fIgnoreFlags |= X86_PTE_P;
4905 }
4906
4907 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4908 {
4909 AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4910 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4911 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4912 cErrors++;
4913 }
4914
4915 /* iterate the page table. */
4916 for (unsigned iPT = 0, off = 0;
4917 iPT < RT_ELEMENTS(pPTDst->a);
4918 iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE))
4919 {
4920 const SHWPTE PteDst = pPTDst->a[iPT];
4921
4922 if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4923 {
4924 AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4925 GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4926 cErrors++;
4927 }
4928
4929 /* skip not-present entries. */
4930 if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4931 continue;
4932
4933 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
4934
4935 /* match the physical addresses */
4936 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4937
4938# ifdef IN_RING3
4939 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4940 if (RT_FAILURE(rc))
4941 {
4942# if 0
4943 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4944 {
4945 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4946 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4947 cErrors++;
4948 }
4949# endif
4950 }
4951 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4952 {
4953 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4954 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4955 cErrors++;
4956 continue;
4957 }
4958# endif
4959 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4960 if (!pPhysPage)
4961 {
4962# if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4963 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4964 {
4965 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4966 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4967 cErrors++;
4968 continue;
4969 }
4970# endif
4971 if (SHW_PTE_IS_RW(PteDst))
4972 {
4973 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4974 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4975 cErrors++;
4976 }
4977 fIgnoreFlags |= X86_PTE_RW;
4978 }
4979 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4980 {
4981 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4982 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4983 cErrors++;
4984 continue;
4985 }
4986
4987 /* flags */
4988 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4989 {
4990 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4991 {
4992 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4993 {
4994 if ( SHW_PTE_IS_RW(PteDst)
4995 && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
4996 {
4997 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4998 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4999 cErrors++;
5000 continue;
5001 }
5002 fIgnoreFlags |= X86_PTE_RW;
5003 }
5004 }
5005 else
5006 {
5007 if ( SHW_PTE_IS_P(PteDst)
5008 && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage)
5009# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
5010 && !PGM_PAGE_IS_MMIO(pPhysPage)
5011# endif
5012 )
5013 {
5014 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
5015 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5016 cErrors++;
5017 continue;
5018 }
5019 fIgnoreFlags |= X86_PTE_P;
5020 }
5021 }
5022
5023 if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
5024 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
5025 )
5026 {
5027 AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
5028 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
5029 fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5030 cErrors++;
5031 continue;
5032 }
5033 } /* for each PTE */
5034 }
5035 }
5036 /* not present */
5037
5038 } /* for each PDE */
5039
5040 } /* for each PDPTE */
5041
5042 } /* for each PML4E */
5043
5044# ifdef DEBUG
5045 if (cErrors)
5046 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
5047# endif
5048# endif /* GST is in {32BIT, PAE, AMD64} */
5049 return cErrors;
5050#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
5051}
5052#endif /* VBOX_STRICT */
5053
5054
5055/**
5056 * Sets up the CR3 for shadow paging
5057 *
5058 * @returns Strict VBox status code.
5059 * @retval VINF_SUCCESS.
5060 *
5061 * @param pVCpu The cross context virtual CPU structure.
5062 * @param GCPhysCR3 The physical address in the CR3 register. (A20 mask
5063 * already applied.)
5064 */
5065PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
5066{
5067 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
5068 int rc = VINF_SUCCESS;
5069
5070 /* Update guest paging info. */
5071#if PGM_GST_TYPE == PGM_TYPE_32BIT \
5072 || PGM_GST_TYPE == PGM_TYPE_PAE \
5073 || PGM_GST_TYPE == PGM_TYPE_AMD64
5074
5075 LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
5076 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
5077
5078# if PGM_GST_TYPE == PGM_TYPE_PAE
5079 if ( !pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped)
5080 || pVCpu->pgm.s.GCPhysPaeCR3 != GCPhysCR3)
5081# endif
5082 {
5083 /*
5084 * Map the page CR3 points at.
5085 */
5086 RTHCPTR HCPtrGuestCR3;
5087 rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3);
5088 if (RT_SUCCESS(rc))
5089 {
5090# if PGM_GST_TYPE == PGM_TYPE_32BIT
5091# ifdef IN_RING3
5092 pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
5093 pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
5094# else
5095 pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
5096 pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
5097# endif
5098
5099# elif PGM_GST_TYPE == PGM_TYPE_PAE
5100# ifdef IN_RING3
5101 pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
5102 pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
5103# else
5104 pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
5105 pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
5106# endif
5107
5108 X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
5109#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5110 /*
5111 * When EPT is enabled by the nested-hypervisor and the nested-guest is in PAE mode,
5112 * the guest-CPU context would've already been updated with the 4 PAE PDPEs specified
5113 * in the virtual VMCS. The PDPEs can differ from those in guest memory referenced by
5114 * the translated nested-guest CR3. We -MUST- use the PDPEs provided in the virtual VMCS
5115 * rather than those in guest memory.
5116 *
5117 * See Intel spec. 26.3.2.4 "Loading Page-Directory-Pointer-Table Entries".
5118 */
5119 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
5120 CPUMGetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5121 else
5122#endif
5123 {
5124 /* Update CPUM with the PAE PDPEs referenced by CR3. */
5125 memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
5126 CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5127 }
5128
5129 /*
5130 * Map the 4 PAE PDPEs.
5131 */
5132 rc = PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
5133 if (RT_SUCCESS(rc))
5134 {
5135# ifdef IN_RING3
5136 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
5137 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5138# else
5139 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5140 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
5141# endif
5142 pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
5143 }
5144
5145# elif PGM_GST_TYPE == PGM_TYPE_AMD64
5146# ifdef IN_RING3
5147 pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
5148 pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
5149# else
5150 pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
5151 pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
5152# endif
5153# endif
5154 }
5155 else
5156 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
5157 }
5158#endif
5159
5160 /*
5161 * Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
5162 */
5163# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5164 || PGM_SHW_TYPE == PGM_TYPE_PAE \
5165 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
5166 && ( PGM_GST_TYPE != PGM_TYPE_REAL \
5167 && PGM_GST_TYPE != PGM_TYPE_PROT))
5168
5169 Assert(!pVM->pgm.s.fNestedPaging);
5170 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
5171
5172 /*
5173 * Update the shadow root page as well since that's not fixed.
5174 */
5175 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5176 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
5177 PPGMPOOLPAGE pNewShwPageCR3;
5178
5179 PGM_LOCK_VOID(pVM);
5180
5181# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5182 if (pPool->cDirtyPages)
5183 pgmPoolResetDirtyPages(pVM);
5184# endif
5185
5186 Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */
5187 int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE,
5188 PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/, &pNewShwPageCR3);
5189 AssertFatalRC(rc2);
5190
5191 pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
5192 pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
5193
5194 /* Set the current hypervisor CR3. */
5195 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
5196
5197 /* Clean up the old CR3 root. */
5198 if ( pOldShwPageCR3
5199 && pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
5200 {
5201 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
5202
5203 /* Mark the page as unlocked; allow flushing again. */
5204 pgmPoolUnlockPage(pPool, pOldShwPageCR3);
5205
5206 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
5207 }
5208 PGM_UNLOCK(pVM);
5209# else
5210 NOREF(GCPhysCR3);
5211# endif
5212
5213 return rc;
5214}
5215
5216/**
5217 * Unmaps the shadow CR3.
5218 *
5219 * @returns VBox status, no specials.
5220 * @param pVCpu The cross context virtual CPU structure.
5221 */
5222PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
5223{
5224 LogFlow(("UnmapCR3\n"));
5225
5226 int rc = VINF_SUCCESS;
5227 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
5228
5229 /*
5230 * Update guest paging info.
5231 */
5232#if PGM_GST_TYPE == PGM_TYPE_32BIT
5233 pVCpu->pgm.s.pGst32BitPdR3 = 0;
5234 pVCpu->pgm.s.pGst32BitPdR0 = 0;
5235
5236#elif PGM_GST_TYPE == PGM_TYPE_PAE
5237 pVCpu->pgm.s.pGstPaePdptR3 = 0;
5238 pVCpu->pgm.s.pGstPaePdptR0 = 0;
5239 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
5240 {
5241 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
5242 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
5243 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
5244 }
5245
5246#elif PGM_GST_TYPE == PGM_TYPE_AMD64
5247 pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
5248 pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
5249
5250#else /* prot/real mode stub */
5251 /* nothing to do */
5252#endif
5253
5254 /*
5255 * PAE PDPEs (and CR3) might have been mapped via PGMGstMapPaePdpesAtCr3()
5256 * prior to switching to PAE in pfnMapCr3(), so we need to clear them here.
5257 */
5258 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5259 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5260 pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
5261
5262 /*
5263 * Update shadow paging info.
5264 */
5265#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5266 || PGM_SHW_TYPE == PGM_TYPE_PAE \
5267 || PGM_SHW_TYPE == PGM_TYPE_AMD64))
5268# if PGM_GST_TYPE != PGM_TYPE_REAL
5269 Assert(!pVM->pgm.s.fNestedPaging);
5270# endif
5271 PGM_LOCK_VOID(pVM);
5272
5273 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
5274 {
5275 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5276
5277# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5278 if (pPool->cDirtyPages)
5279 pgmPoolResetDirtyPages(pVM);
5280# endif
5281
5282 /* Mark the page as unlocked; allow flushing again. */
5283 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
5284
5285 pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
5286 pVCpu->pgm.s.pShwPageCR3R3 = 0;
5287 pVCpu->pgm.s.pShwPageCR3R0 = 0;
5288 }
5289
5290 PGM_UNLOCK(pVM);
5291#endif
5292
5293 return rc;
5294}
5295
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette