VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 99813

Last change on this file since 99813 was 99788, checked in by vboxsync, 19 months ago

VMM/PGM: Nested VMX: bugref:10318 Don't need to get the physical page unless the guest is already using large pages. Other nits and assertions.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 232.9 KB
Line 
1/* $Id: PGMAllBth.h 99788 2023-05-15 06:01:35Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6 * PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7 * bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8 *
9 * @remarks This file is one big \#ifdef-orgy!
10 *
11 */
12
13/*
14 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
15 *
16 * This file is part of VirtualBox base platform packages, as
17 * available from https://www.virtualbox.org.
18 *
19 * This program is free software; you can redistribute it and/or
20 * modify it under the terms of the GNU General Public License
21 * as published by the Free Software Foundation, in version 3 of the
22 * License.
23 *
24 * This program is distributed in the hope that it will be useful, but
25 * WITHOUT ANY WARRANTY; without even the implied warranty of
26 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
27 * General Public License for more details.
28 *
29 * You should have received a copy of the GNU General Public License
30 * along with this program; if not, see <https://www.gnu.org/licenses>.
31 *
32 * SPDX-License-Identifier: GPL-3.0-only
33 */
34
35#ifdef _MSC_VER
36/** @todo we're generating unnecessary code in nested/ept shadow mode and for
37 * real/prot-guest+RC mode. */
38# pragma warning(disable: 4505)
39#endif
40
41
42/*********************************************************************************************************************************
43* Internal Functions *
44*********************************************************************************************************************************/
45RT_C_DECLS_BEGIN
46PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
47#ifndef IN_RING3
48PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken);
49PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
50 bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken);
51# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
52static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
53 unsigned iPte, PCPGMPTWALKGST pGstWalkAll);
54static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
55 uint32_t uErr, PPGMPTWALKGST pGstWalkAll);
56static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll);
57# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
58#endif
59PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
60static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
61static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
62static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
63#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
64static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
65#else
66static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
67#endif
68PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
69PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
70PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
71#ifdef VBOX_STRICT
72PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
73#endif
74PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
75PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
76
77#ifdef IN_RING3
78PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
79#endif
80RT_C_DECLS_END
81
82
83
84
85/*
86 * Filter out some illegal combinations of guest and shadow paging, so we can
87 * remove redundant checks inside functions.
88 */
89#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
90 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
91# error "Invalid combination; PAE guest implies PAE shadow"
92#endif
93
94#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
95 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 \
96 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
97# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
98#endif
99
100#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
101 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE \
102 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
103# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
104#endif
105
106#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
107 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
108# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
109#endif
110
111
112/**
113 * Enters the shadow+guest mode.
114 *
115 * @returns VBox status code.
116 * @param pVCpu The cross context virtual CPU structure.
117 * @param GCPhysCR3 The physical address from the CR3 register.
118 */
119PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
120{
121 /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
122 * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
123 */
124#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
125 || PGM_SHW_TYPE == PGM_TYPE_PAE \
126 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
127 && ( PGM_GST_TYPE == PGM_TYPE_REAL \
128 || PGM_GST_TYPE == PGM_TYPE_PROT))
129
130 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
131
132 Assert(!pVM->pgm.s.fNestedPaging);
133
134 PGM_LOCK_VOID(pVM);
135 /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
136 * but any calls to GC need a proper shadow page setup as well.
137 */
138 /* Free the previous root mapping if still active. */
139 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
140 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
141 if (pOldShwPageCR3)
142 {
143 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
144
145 /* Mark the page as unlocked; allow flushing again. */
146 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
147
148 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
149 pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
150 pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
151 }
152
153 /* construct a fake address. */
154 GCPhysCR3 = RT_BIT_64(63);
155 PPGMPOOLPAGE pNewShwPageCR3;
156 int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
157 NIL_PGMPOOL_IDX, UINT32_MAX, false /*fLockPage*/,
158 &pNewShwPageCR3);
159 AssertRCReturn(rc, rc);
160
161 pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
162 pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
163
164 /* Mark the page as locked; disallow flushing. */
165 pgmPoolLockPage(pPool, pNewShwPageCR3);
166
167 /* Set the current hypervisor CR3. */
168 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
169
170 PGM_UNLOCK(pVM);
171 return rc;
172#else
173 NOREF(pVCpu); NOREF(GCPhysCR3);
174 return VINF_SUCCESS;
175#endif
176}
177
178
179#ifndef IN_RING3
180
181# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
182/**
183 * Deal with a guest page fault.
184 *
185 * @returns Strict VBox status code.
186 * @retval VINF_EM_RAW_GUEST_TRAP
187 * @retval VINF_EM_RAW_EMULATE_INSTR
188 *
189 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
190 * @param pWalk The guest page table walk result.
191 * @param uErr The error code.
192 */
193PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr)
194{
195 /*
196 * Calc the error code for the guest trap.
197 */
198 uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
199 ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)
200 : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
201 if ( pWalk->fRsvdError
202 || pWalk->fBadPhysAddr)
203 {
204 uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P;
205 Assert(!pWalk->fNotPresent);
206 }
207 else if (!pWalk->fNotPresent)
208 uNewErr |= X86_TRAP_PF_P;
209 TRPMSetErrorCode(pVCpu, uNewErr);
210
211 LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel));
212 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
213 return VINF_EM_RAW_GUEST_TRAP;
214}
215# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
216
217
218#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
219/**
220 * Deal with a guest page fault.
221 *
222 * The caller has taken the PGM lock.
223 *
224 * @returns Strict VBox status code.
225 *
226 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
227 * @param uErr The error code.
228 * @param pCtx Pointer to the register context for the CPU.
229 * @param pvFault The fault address.
230 * @param pPage The guest page at @a pvFault.
231 * @param pWalk The guest page table walk result.
232 * @param pGstWalk The guest paging-mode specific walk information.
233 * @param pfLockTaken PGM lock taken here or not (out). This is true
234 * when we're called.
235 */
236static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
237 RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
238# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
239 , PPGMPTWALK pWalk
240 , PGSTPTWALK pGstWalk
241# endif
242 )
243{
244# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
245 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
246# endif
247 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
248 VBOXSTRICTRC rcStrict;
249
250 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
251 {
252 /*
253 * Physical page access handler.
254 */
255# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
256 const RTGCPHYS GCPhysFault = pWalk->GCPhys;
257# else
258 const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
259# endif
260 PPGMPHYSHANDLER pCur;
261 rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysFault, &pCur);
262 if (RT_SUCCESS(rcStrict))
263 {
264 PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
265
266# ifdef PGM_SYNC_N_PAGES
267 /*
268 * If the region is write protected and we got a page not present fault, then sync
269 * the pages. If the fault was caused by a read, then restart the instruction.
270 * In case of write access continue to the GC write handler.
271 *
272 * ASSUMES that there is only one handler per page or that they have similar write properties.
273 */
274 if ( !(uErr & X86_TRAP_PF_P)
275 && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
276 {
277# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
278 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
279# else
280 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
281# endif
282 if ( RT_FAILURE(rcStrict)
283 || !(uErr & X86_TRAP_PF_RW)
284 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
285 {
286 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
287 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
288 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
289 return rcStrict;
290 }
291 }
292# endif
293# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
294 /*
295 * If the access was not thru a #PF(RSVD|...) resync the page.
296 */
297 if ( !(uErr & X86_TRAP_PF_RSVD)
298 && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
299# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
300 && (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK))
301 == PGM_PTATTRS_W_MASK /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
302# endif
303 )
304 {
305# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
306 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
307# else
308 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
309# endif
310 if ( RT_FAILURE(rcStrict)
311 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
312 {
313 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
314 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
315 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
316 return rcStrict;
317 }
318 }
319# endif
320
321 AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
322 || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
323 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
324 pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
325 if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
326 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
327 else
328 {
329 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
330 if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
331 }
332
333 if (pCurType->pfnPfHandler)
334 {
335 STAM_PROFILE_START(&pCur->Stat, h);
336
337 if (pCurType->fKeepPgmLock)
338 {
339 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault,
340 !pCurType->fRing0DevInsIdx ? pCur->uUser
341 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser));
342
343 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
344 }
345 else
346 {
347 uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
348 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
349 PGM_UNLOCK(pVM);
350 *pfLockTaken = false;
351
352 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault, uUser);
353
354 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
355 }
356 }
357 else
358 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
359
360 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
361 return rcStrict;
362 }
363 AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
364 }
365
366 /*
367 * There is a handled area of the page, but this fault doesn't belong to it.
368 * We must emulate the instruction.
369 *
370 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
371 * we first check if this was a page-not-present fault for a page with only
372 * write access handlers. Restart the instruction if it wasn't a write access.
373 */
374 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled);
375
376 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
377 && !(uErr & X86_TRAP_PF_P))
378 {
379# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
380 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
381# else
382 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
383# endif
384 if ( RT_FAILURE(rcStrict)
385 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
386 || !(uErr & X86_TRAP_PF_RW))
387 {
388 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
389 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
390 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
391 return rcStrict;
392 }
393 }
394
395 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
396 * It's writing to an unhandled part of the LDT page several million times.
397 */
398 rcStrict = PGMInterpretInstruction(pVCpu, pvFault);
399 LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
400 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; });
401 return rcStrict;
402} /* if any kind of handler */
403# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
404
405
406/**
407 * \#PF Handler for raw-mode guest execution.
408 *
409 * @returns VBox status code (appropriate for trap handling and GC return).
410 *
411 * @param pVCpu The cross context virtual CPU structure.
412 * @param uErr The trap error code.
413 * @param pCtx Pointer to the register context for the CPU.
414 * @param pvFault The fault address.
415 * @param pfLockTaken PGM lock taken here or not (out)
416 */
417PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken)
418{
419 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
420
421 *pfLockTaken = false;
422
423# if ( PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \
424 || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
425 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
426 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
427 && PGM_SHW_TYPE != PGM_TYPE_NONE
428 int rc;
429
430# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
431 /*
432 * Walk the guest page translation tables and check if it's a guest fault.
433 */
434 PGMPTWALK Walk;
435 GSTPTWALK GstWalk;
436 rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk);
437 if (RT_FAILURE_NP(rc))
438 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
439
440 /* assert some GstWalk sanity. */
441# if PGM_GST_TYPE == PGM_TYPE_AMD64
442 /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. */
443# endif
444# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
445 /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */
446# endif
447 /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */
448 /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */
449 Assert(Walk.fSucceeded);
450 Assert(Walk.fEffective & PGM_PTATTRS_R_MASK);
451
452 if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
453 {
454 if ( ( (uErr & X86_TRAP_PF_RW)
455 && !(Walk.fEffective & PGM_PTATTRS_W_MASK)
456 && ( (uErr & X86_TRAP_PF_US)
457 || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
458 || ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK))
459 || ((uErr & X86_TRAP_PF_ID) && (Walk.fEffective & PGM_PTATTRS_NX_MASK))
460 )
461 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
462 }
463
464 /* Take the big lock now before we update flags. */
465 *pfLockTaken = true;
466 PGM_LOCK_VOID(pVM);
467
468 /*
469 * Set the accessed and dirty flags.
470 */
471 /** @todo Should probably use cmpxchg logic here as we're potentially racing
472 * other CPUs in SMP configs. (the lock isn't enough, since we take it
473 * after walking and the page tables could be stale already) */
474# if PGM_GST_TYPE == PGM_TYPE_AMD64
475 if (!(GstWalk.Pml4e.u & X86_PML4E_A))
476 {
477 GstWalk.Pml4e.u |= X86_PML4E_A;
478 GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A);
479 }
480 if (!(GstWalk.Pdpe.u & X86_PDPE_A))
481 {
482 GstWalk.Pdpe.u |= X86_PDPE_A;
483 GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A);
484 }
485# endif
486 if (Walk.fBigPage)
487 {
488 Assert(GstWalk.Pde.u & X86_PDE_PS);
489 if (uErr & X86_TRAP_PF_RW)
490 {
491 if ((GstWalk.Pde.u & (X86_PDE4M_A | X86_PDE4M_D)) != (X86_PDE4M_A | X86_PDE4M_D))
492 {
493 GstWalk.Pde.u |= X86_PDE4M_A | X86_PDE4M_D;
494 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A | X86_PDE4M_D);
495 }
496 }
497 else
498 {
499 if (!(GstWalk.Pde.u & X86_PDE4M_A))
500 {
501 GstWalk.Pde.u |= X86_PDE4M_A;
502 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A);
503 }
504 }
505 }
506 else
507 {
508 Assert(!(GstWalk.Pde.u & X86_PDE_PS));
509 if (!(GstWalk.Pde.u & X86_PDE_A))
510 {
511 GstWalk.Pde.u |= X86_PDE_A;
512 GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A);
513 }
514
515 if (uErr & X86_TRAP_PF_RW)
516 {
517# ifdef VBOX_WITH_STATISTICS
518 if (GstWalk.Pte.u & X86_PTE_D)
519 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty));
520 else
521 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage));
522# endif
523 if ((GstWalk.Pte.u & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
524 {
525 GstWalk.Pte.u |= X86_PTE_A | X86_PTE_D;
526 GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A | X86_PTE_D);
527 }
528 }
529 else
530 {
531 if (!(GstWalk.Pte.u & X86_PTE_A))
532 {
533 GstWalk.Pte.u |= X86_PTE_A;
534 GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A);
535 }
536 }
537 Assert(GstWalk.Pte.u == GstWalk.pPte->u);
538 }
539#if 0
540 /* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */
541 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u,
542 ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
543#endif
544
545# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
546 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */
547
548 /* Take the big lock now. */
549 *pfLockTaken = true;
550 PGM_LOCK_VOID(pVM);
551# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
552
553# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
554 /*
555 * If it is a reserved bit fault we know that it is an MMIO (access
556 * handler) related fault and can skip some 200 lines of code.
557 */
558 if (uErr & X86_TRAP_PF_RSVD)
559 {
560 Assert(uErr & X86_TRAP_PF_P);
561 PPGMPAGE pPage;
562# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
563 rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage);
564 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
565 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage,
566 pfLockTaken, &Walk, &GstWalk));
567 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
568# else
569 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
570 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
571 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
572 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
573# endif
574 AssertRC(rc);
575 PGM_INVL_PG(pVCpu, pvFault);
576 return rc; /* Restart with the corrected entry. */
577 }
578# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
579
580 /*
581 * Fetch the guest PDE, PDPE and PML4E.
582 */
583# if PGM_SHW_TYPE == PGM_TYPE_32BIT
584 const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
585 PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
586
587# elif PGM_SHW_TYPE == PGM_TYPE_PAE
588 const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
589 PX86PDPAE pPDDst;
590# if PGM_GST_TYPE == PGM_TYPE_PAE
591 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
592# else
593 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
594# endif
595 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
596
597# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
598 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
599 PX86PDPAE pPDDst;
600# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
601 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A,
602 X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst);
603# else
604 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
605# endif
606 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
607
608# elif PGM_SHW_TYPE == PGM_TYPE_EPT
609 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
610 PEPTPD pPDDst;
611 rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
612 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
613# endif
614 Assert(pPDDst);
615
616# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
617 /*
618 * Dirty page handling.
619 *
620 * If we successfully correct the write protection fault due to dirty bit
621 * tracking, then return immediately.
622 */
623 if (uErr & X86_TRAP_PF_RW) /* write fault? */
624 {
625 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
626 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
627 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
628 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
629 {
630 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0
631 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
632 ? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed
633 : &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
634 Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
635 return VINF_SUCCESS;
636 }
637#ifdef DEBUG_bird
638 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
639 AssertMsg(Walk.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
640#endif
641 }
642
643# if 0 /* rarely useful; leave for debugging. */
644 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
645# endif
646# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
647
648 /*
649 * A common case is the not-present error caused by lazy page table syncing.
650 *
651 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
652 * here so we can safely assume that the shadow PT is present when calling
653 * SyncPage later.
654 *
655 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
656 * of mapping conflict and defer to SyncCR3 in R3.
657 * (Again, we do NOT support access handlers for non-present guest pages.)
658 *
659 */
660# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
661 Assert(GstWalk.Pde.u & X86_PDE_P);
662# endif
663 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
664 && !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
665 {
666 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
667# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
668 LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
669 rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
670# else
671 LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
672 rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
673# endif
674 if (RT_SUCCESS(rc))
675 return rc;
676 Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
677 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
678 return VINF_PGM_SYNC_CR3;
679 }
680
681 /*
682 * Check if this fault address is flagged for special treatment,
683 * which means we'll have to figure out the physical address and
684 * check flags associated with it.
685 *
686 * ASSUME that we can limit any special access handling to pages
687 * in page tables which the guest believes to be present.
688 */
689# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
690 RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
691# else
692 RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
693# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
694 PPGMPAGE pPage;
695 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
696 if (RT_FAILURE(rc))
697 {
698 /*
699 * When the guest accesses invalid physical memory (e.g. probing
700 * of RAM or accessing a remapped MMIO range), then we'll fall
701 * back to the recompiler to emulate the instruction.
702 */
703 LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
704 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
705 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
706 return VINF_EM_RAW_EMULATE_INSTR;
707 }
708
709 /*
710 * Any handlers for this page?
711 */
712 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
713# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
714 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken,
715 &Walk, &GstWalk));
716# else
717 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
718# endif
719
720 /*
721 * We are here only if page is present in Guest page tables and
722 * trap is not handled by our handlers.
723 *
724 * Check it for page out-of-sync situation.
725 */
726 if (!(uErr & X86_TRAP_PF_P))
727 {
728 /*
729 * Page is not present in our page tables. Try to sync it!
730 */
731 if (uErr & X86_TRAP_PF_US)
732 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
733 else /* supervisor */
734 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
735
736 if (PGM_PAGE_IS_BALLOONED(pPage))
737 {
738 /* Emulate reads from ballooned pages as they are not present in
739 our shadow page tables. (Required for e.g. Solaris guests; soft
740 ecc, random nr generator.) */
741 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
742 LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
743 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
744 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
745 return rc;
746 }
747
748# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
749 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
750# else
751 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
752# endif
753 if (RT_SUCCESS(rc))
754 {
755 /* The page was successfully synced, return to the guest. */
756 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
757 return VINF_SUCCESS;
758 }
759 }
760 else /* uErr & X86_TRAP_PF_P: */
761 {
762 /*
763 * Write protected pages are made writable when the guest makes the
764 * first write to it. This happens for pages that are shared, write
765 * monitored or not yet allocated.
766 *
767 * We may also end up here when CR0.WP=0 in the guest.
768 *
769 * Also, a side effect of not flushing global PDEs are out of sync
770 * pages due to physical monitored regions, that are no longer valid.
771 * Assume for now it only applies to the read/write flag.
772 */
773 if (uErr & X86_TRAP_PF_RW)
774 {
775 /*
776 * Check if it is a read-only page.
777 */
778 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
779 {
780 Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
781 Assert(!PGM_PAGE_IS_ZERO(pPage));
782 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
783 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
784
785 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
786 if (rc != VINF_SUCCESS)
787 {
788 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
789 return rc;
790 }
791 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
792 return VINF_EM_NO_MEMORY;
793 }
794
795# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
796 /*
797 * Check to see if we need to emulate the instruction if CR0.WP=0.
798 */
799 if ( !(Walk.fEffective & PGM_PTATTRS_W_MASK)
800 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
801 && CPUMGetGuestCPL(pVCpu) < 3)
802 {
803 Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P));
804
805 /*
806 * The Netware WP0+RO+US hack.
807 *
808 * Netware sometimes(/always?) runs with WP0. It has been observed doing
809 * excessive write accesses to pages which are mapped with US=1 and RW=0
810 * while WP=0. This causes a lot of exits and extremely slow execution.
811 * To avoid trapping and emulating every write here, we change the shadow
812 * page table entry to map it as US=0 and RW=1 until user mode tries to
813 * access it again (see further below). We count these shadow page table
814 * changes so we can avoid having to clear the page pool every time the WP
815 * bit changes to 1 (see PGMCr0WpEnabled()).
816 */
817# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) && 1
818 if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
819 && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
820 && pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
821 {
822 Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage));
823 rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
824 if (rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3)
825 {
826 PGM_INVL_PG(pVCpu, pvFault);
827 pVCpu->pgm.s.cNetwareWp0Hacks++;
828 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; });
829 return rc;
830 }
831 AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
832 Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
833 }
834# endif
835
836 /* Interpret the access. */
837 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
838 Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK)));
839 if (RT_SUCCESS(rc))
840 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ);
841 else
842 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3);
843 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; });
844 return rc;
845 }
846# endif
847 /// @todo count the above case; else
848 if (uErr & X86_TRAP_PF_US)
849 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
850 else /* supervisor */
851 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
852
853 /*
854 * Sync the page.
855 *
856 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
857 * page is not present, which is not true in this case.
858 */
859# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
860 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
861# else
862 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
863# endif
864 if (RT_SUCCESS(rc))
865 {
866 /*
867 * Page was successfully synced, return to guest but invalidate
868 * the TLB first as the page is very likely to be in it.
869 */
870# if PGM_SHW_TYPE == PGM_TYPE_EPT
871 HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
872# else
873 PGM_INVL_PG(pVCpu, pvFault);
874# endif
875# ifdef VBOX_STRICT
876 PGMPTWALK GstPageWalk;
877 GstPageWalk.GCPhys = RTGCPHYS_MAX;
878 if (!pVM->pgm.s.fNestedPaging)
879 {
880 rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
881 AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) || ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective));
882 LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective));
883 }
884# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
885 uint64_t fPageShw = 0;
886 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
887 AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
888 ("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault));
889# endif
890# endif /* VBOX_STRICT */
891 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
892 return VINF_SUCCESS;
893 }
894 }
895# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
896 /*
897 * Check for Netware WP0+RO+US hack from above and undo it when user
898 * mode accesses the page again.
899 */
900 else if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
901 && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
902 && pVCpu->pgm.s.cNetwareWp0Hacks > 0
903 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
904 && CPUMGetGuestCPL(pVCpu) == 3
905 && pVM->cCpus == 1
906 )
907 {
908 Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
909 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
910 if (RT_SUCCESS(rc))
911 {
912 PGM_INVL_PG(pVCpu, pvFault);
913 pVCpu->pgm.s.cNetwareWp0Hacks--;
914 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; });
915 return VINF_SUCCESS;
916 }
917 }
918# endif /* PGM_WITH_PAGING */
919
920 /** @todo else: why are we here? */
921
922# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
923 /*
924 * Check for VMM page flags vs. Guest page flags consistency.
925 * Currently only for debug purposes.
926 */
927 if (RT_SUCCESS(rc))
928 {
929 /* Get guest page flags. */
930 PGMPTWALK GstPageWalk;
931 int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
932 if (RT_SUCCESS(rc2))
933 {
934 uint64_t fPageShw = 0;
935 rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
936
937#if 0
938 /*
939 * Compare page flags.
940 * Note: we have AVL, A, D bits desynced.
941 */
942 AssertMsg( (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
943 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
944 || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
945 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
946 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
947 && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW
948 && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US),
949 ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
950 pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
95101:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
95201:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
953
95401:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
95501:01:15.625516 00:08:43.268051 Location :
956e:\vbox\svn\trunk\srcPage flags mismatch!
957pvFault=fffff801b0d7b000
958 uErr=11 X86_TRAP_PF_ID | X86_TRAP_PF_P
959GCPhys=0000000019b52000
960fPageShw=0
961fPageGst=77b0000000000121
962rc=0
963#endif
964
965 }
966 else
967 AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
968 }
969 else
970 AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
971# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
972 }
973
974
975 /*
976 * If we get here it is because something failed above, i.e. most like guru
977 * meditiation time.
978 */
979 LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
980 __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pCtx->cs.Sel, pCtx->rip));
981 return rc;
982
983# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
984 NOREF(uErr); NOREF(pCtx); NOREF(pvFault);
985 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
986 return VERR_PGM_NOT_USED_IN_MODE;
987# endif
988}
989
990
991# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
992/**
993 * Deals with a nested-guest \#PF fault for a guest-physical page with a handler.
994 *
995 * @returns Strict VBox status code.
996 * @param pVCpu The cross context virtual CPU structure.
997 * @param uErr The error code.
998 * @param pCtx Pointer to the register context for the CPU.
999 * @param GCPhysNestedFault The nested-guest physical address of the fault.
1000 * @param pPage The guest page at @a GCPhysNestedFault.
1001 * @param GCPhysFault The guest-physical address of the fault.
1002 * @param pGstWalkAll The guest page walk result.
1003 * @param pfLockTaken Where to store whether the PGM is still held when
1004 * this function completes.
1005 *
1006 * @note The caller has taken the PGM lock.
1007 */
1008static VBOXSTRICTRC PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
1009 RTGCPHYS GCPhysNestedFault, PPGMPAGE pPage,
1010 RTGCPHYS GCPhysFault, PPGMPTWALKGST pGstWalkAll,
1011 bool *pfLockTaken)
1012{
1013# if PGM_GST_TYPE == PGM_TYPE_PROT \
1014 && PGM_SHW_TYPE == PGM_TYPE_EPT
1015
1016 /** @todo Assert uErr isn't X86_TRAP_PF_RSVD and remove release checks. */
1017 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysFault);
1018 AssertMsgReturn(PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage), ("%RGp %RGp uErr=%u\n", GCPhysNestedFault, GCPhysFault, uErr),
1019 VERR_PGM_HANDLER_IPE_1);
1020
1021 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1022 RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1023 RTGCPHYS const GCPhysPage = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1024
1025 /*
1026 * Physical page access handler.
1027 */
1028 PPGMPHYSHANDLER pCur;
1029 VBOXSTRICTRC rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pCur);
1030 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
1031
1032 PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
1033 Assert(pCurType);
1034
1035 /*
1036 * If the region is write protected and we got a page not present fault, then sync
1037 * the pages. If the fault was caused by a read, then restart the instruction.
1038 * In case of write access continue to the GC write handler.
1039 */
1040 if ( !(uErr & X86_TRAP_PF_P)
1041 && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1042 {
1043 Log7Func(("Syncing Monitored: GCPhysNestedPage=%RGp GCPhysPage=%RGp uErr=%#x\n", GCPhysNestedPage, GCPhysPage, uErr));
1044 rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
1045 Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1046 if ( RT_FAILURE(rcStrict)
1047 || !(uErr & X86_TRAP_PF_RW))
1048 {
1049 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1050 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1051 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1052 return rcStrict;
1053 }
1054 }
1055 else if ( !(uErr & X86_TRAP_PF_RSVD)
1056 && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE)
1057 {
1058 /*
1059 * If the access was NOT through an EPT misconfig (i.e. RSVD), sync the page.
1060 * This can happen for the VMX APIC-access page.
1061 */
1062 Log7Func(("Syncing MMIO: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1063 rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
1064 Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
1065 if (RT_FAILURE(rcStrict))
1066 {
1067 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1068 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
1069 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
1070 return rcStrict;
1071 }
1072 }
1073
1074 AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
1075 || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
1076 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
1077 GCPhysNestedFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
1078 if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
1079 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
1080 else
1081 {
1082 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
1083 if (uErr & X86_TRAP_PF_RSVD)
1084 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
1085 }
1086
1087 if (pCurType->pfnPfHandler)
1088 {
1089 STAM_PROFILE_START(&pCur->Stat, h);
1090 uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
1091 : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
1092
1093 if (pCurType->fKeepPgmLock)
1094 {
1095 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1096 STAM_PROFILE_STOP(&pCur->Stat, h);
1097 }
1098 else
1099 {
1100 PGM_UNLOCK(pVM);
1101 *pfLockTaken = false;
1102 rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
1103 STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
1104 }
1105 }
1106 else
1107 {
1108 AssertMsgFailed(("What's going on here!? Fault falls outside handler range!?\n"));
1109 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
1110 }
1111
1112 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
1113 return rcStrict;
1114
1115# else
1116 RT_NOREF8(pVCpu, uErr, pCtx, GCPhysNestedFault, pPage, GCPhysFault, pGstWalkAll, pfLockTaken);
1117 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1118 return VERR_PGM_NOT_USED_IN_MODE;
1119# endif
1120}
1121# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1122
1123
1124/**
1125 * Nested \#PF handler for nested-guest hardware-assisted execution using nested
1126 * paging.
1127 *
1128 * @returns VBox status code (appropriate for trap handling and GC return).
1129 * @param pVCpu The cross context virtual CPU structure.
1130 * @param uErr The fault error (X86_TRAP_PF_*).
1131 * @param pCtx Pointer to the register context for the CPU.
1132 * @param GCPhysNestedFault The nested-guest physical address of the fault.
1133 * @param fIsLinearAddrValid Whether translation of a nested-guest linear address
1134 * caused this fault. If @c false, GCPtrNestedFault
1135 * must be 0.
1136 * @param GCPtrNestedFault The nested-guest linear address of this fault.
1137 * @param pWalk The guest page table walk result.
1138 * @param pfLockTaken Where to store whether the PGM lock is still held
1139 * when this function completes.
1140 */
1141PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
1142 bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken)
1143{
1144 *pfLockTaken = false;
1145# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \
1146 && PGM_GST_TYPE == PGM_TYPE_PROT \
1147 && PGM_SHW_TYPE == PGM_TYPE_EPT
1148
1149 Assert(CPUMIsGuestVmxEptPagingEnabled(pVCpu));
1150 Assert(PGM_A20_IS_ENABLED(pVCpu));
1151
1152 /* We don't support mode-based execute control for EPT yet. */
1153 Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
1154 Assert(!(uErr & X86_TRAP_PF_US));
1155
1156 /* Take the big lock now. */
1157 *pfLockTaken = true;
1158 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1159 PGM_LOCK_VOID(pVM);
1160
1161 /*
1162 * Walk the guest EPT tables and check if it's an EPT violation or misconfiguration.
1163 */
1164 if (fIsLinearAddrValid)
1165 Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x GCPtrNestedFault=%RGv\n",
1166 pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr, GCPtrNestedFault));
1167 else
1168 Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x\n",
1169 pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr));
1170 PGMPTWALKGST GstWalkAll;
1171 int rc = pgmGstSlatWalk(pVCpu, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk, &GstWalkAll);
1172 if (RT_FAILURE(rc))
1173 return rc;
1174
1175 Assert(GstWalkAll.enmType == PGMPTWALKGSTTYPE_EPT);
1176 Assert(pWalk->fSucceeded);
1177 Assert(pWalk->fEffective & (PGM_PTATTRS_EPT_R_MASK | PGM_PTATTRS_EPT_W_MASK | PGM_PTATTRS_EPT_X_SUPER_MASK));
1178 Assert(pWalk->fIsSlat);
1179
1180# ifdef DEBUG_ramshankar
1181 /* Paranoia. */
1182 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_R_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_R_MASK));
1183 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_W_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_W_MASK));
1184 Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_NX_MASK) == !RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_X_SUPER_MASK));
1185# endif
1186
1187 Log7Func(("SLAT: GCPhysNestedFault=%RGp -> GCPhys=%#RGp\n", GCPhysNestedFault, pWalk->GCPhys));
1188
1189 /*
1190 * Check page-access permissions.
1191 */
1192 if ( ((uErr & X86_TRAP_PF_RW) && !(pWalk->fEffective & PGM_PTATTRS_W_MASK))
1193 || ((uErr & X86_TRAP_PF_ID) && (pWalk->fEffective & PGM_PTATTRS_NX_MASK)))
1194 {
1195 Log7Func(("Permission failed! GCPtrNested=%RGv GCPhysNested=%RGp uErr=%#x fEffective=%#RX64\n", GCPtrNestedFault,
1196 GCPhysNestedFault, uErr, pWalk->fEffective));
1197 pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION;
1198 return VERR_ACCESS_DENIED;
1199 }
1200
1201 PGM_A20_ASSERT_MASKED(pVCpu, pWalk->GCPhys);
1202 RTGCPHYS const GCPhysPage = pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1203 RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1204
1205 /*
1206 * If we were called via an EPT misconfig, it should've already resulted in a nested-guest VM-exit.
1207 */
1208 AssertMsgReturn(!(uErr & X86_TRAP_PF_RSVD),
1209 ("Unexpected EPT misconfig VM-exit. GCPhysPage=%RGp GCPhysNestedPage=%RGp\n", GCPhysPage, GCPhysNestedPage),
1210 VERR_PGM_MAPPING_IPE);
1211
1212 /*
1213 * Fetch and sync the nested-guest EPT page directory pointer.
1214 */
1215 PEPTPD pEptPd;
1216 rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL /*ppPdpt*/, &pEptPd, &GstWalkAll);
1217 AssertRCReturn(rc, rc);
1218 Assert(pEptPd);
1219
1220 /*
1221 * A common case is the not-present error caused by lazy page table syncing.
1222 *
1223 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
1224 * here so we can safely assume that the shadow PT is present when calling
1225 * NestedSyncPage later.
1226 *
1227 * NOTE: It's possible we will be syncing the VMX APIC-access page here.
1228 * In that case, we would sync the page but will NOT go ahead with emulating
1229 * the APIC-access VM-exit through IEM. However, once the page is mapped in
1230 * the shadow tables, subsequent APIC-access VM-exits for the nested-guest
1231 * will be triggered by hardware. Maybe calling the IEM #PF handler can be
1232 * considered as an optimization later.
1233 */
1234 unsigned const iPde = (GCPhysNestedPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1235 if ( !(uErr & X86_TRAP_PF_P)
1236 && !(pEptPd->a[iPde].u & EPT_PRESENT_MASK))
1237 {
1238 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
1239 Log7Func(("NestedSyncPT: Lazy. GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
1240 rc = PGM_BTH_NAME(NestedSyncPT)(pVCpu, GCPhysNestedPage, GCPhysPage, &GstWalkAll);
1241 if (RT_SUCCESS(rc))
1242 return rc;
1243 AssertMsgFailedReturn(("NestedSyncPT: %RGv failed! rc=%Rrc\n", GCPhysNestedPage, rc), VERR_PGM_MAPPING_IPE);
1244 }
1245
1246 /*
1247 * Check if this fault address is flagged for special treatment.
1248 * This handles faults on an MMIO or write-monitored page.
1249 *
1250 * If this happens to be the VMX APIC-access page, we don't treat is as MMIO
1251 * but rather sync it further below (as a regular guest page) which lets
1252 * hardware-assisted execution trigger the APIC-access VM-exits of the
1253 * nested-guest directly.
1254 */
1255 PPGMPAGE pPage;
1256 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1257 if (RT_FAILURE(rc))
1258 {
1259 /*
1260 * We failed to get the physical page which means it's a reserved/invalid
1261 * page address (not MMIO even). This can typically be observed with
1262 * Microsoft Hyper-V enabled Windows guests. We must fall back to emulating
1263 * the instruction, see @bugref{10318#c7}.
1264 */
1265 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
1266 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
1267 return VINF_EM_RAW_EMULATE_INSTR;
1268 }
1269 /* Check if this is an MMIO page and NOT the VMX APIC-access page. */
1270 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1271 {
1272 Log7Func(("MMIO: Calling NestedTrap0eHandlerDoAccessHandlers for GCPhys %RGp\n", GCPhysPage));
1273 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, GCPhysNestedFault,
1274 pPage, pWalk->GCPhys, &GstWalkAll,
1275 pfLockTaken));
1276 }
1277
1278 /*
1279 * We are here only if page is present in nested-guest page tables but the
1280 * trap is not handled by our handlers. Check for page out-of-sync situation.
1281 */
1282 if (!(uErr & X86_TRAP_PF_P))
1283 {
1284 Assert(!PGM_PAGE_IS_BALLOONED(pPage));
1285 Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1286 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
1287
1288 Log7Func(("SyncPage: Not-Present: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedFault, GCPhysPage));
1289 rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, PGM_SYNC_NR_PAGES, uErr, &GstWalkAll);
1290 if (RT_SUCCESS(rc))
1291 {
1292 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
1293 return VINF_SUCCESS;
1294 }
1295 }
1296 else if (uErr & X86_TRAP_PF_RW)
1297 {
1298 /*
1299 * Write protected pages are made writable when the guest makes the
1300 * first write to it. This happens for pages that are shared, write
1301 * monitored or not yet allocated.
1302 *
1303 * We may also end up here when CR0.WP=0 in the guest.
1304 *
1305 * Also, a side effect of not flushing global PDEs are out of sync
1306 * pages due to physical monitored regions, that are no longer valid.
1307 * Assume for now it only applies to the read/write flag.
1308 */
1309 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1310 {
1311 /* This is a read-only page. */
1312 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysPage));
1313 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
1314
1315 Log7Func(("Calling pgmPhysPageMakeWritable for GCPhysPage=%RGp\n", GCPhysPage));
1316 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1317 if (rc != VINF_SUCCESS)
1318 {
1319 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
1320 return rc;
1321 }
1322 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
1323 return VINF_EM_NO_MEMORY;
1324 }
1325
1326 Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */
1327 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1328
1329 /*
1330 * Sync the write-protected page.
1331 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1332 * page is not present, which is not true in this case.
1333 */
1334 Log7Func(("SyncPage: RW: cs:rip=%04x:%#RX64 GCPhysNestedPage=%RGp uErr=%#RX32 GCPhysPage=%RGp WalkGCPhys=%RGp\n",
1335 pCtx->cs.Sel, pCtx->rip, GCPhysNestedPage, (uint32_t)uErr, GCPhysPage, pWalk->GCPhys));
1336 rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /* cPages */, uErr, &GstWalkAll);
1337 if (RT_SUCCESS(rc))
1338 {
1339 HMInvalidatePhysPage(pVM, GCPhysPage);
1340 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
1341 return VINF_SUCCESS;
1342 }
1343 }
1344
1345 /*
1346 * If we get here it is because something failed above => guru meditation time.
1347 */
1348 LogRelFunc(("rc=%Rrc GCPhysNestedFault=%#RGp (%#RGp) uErr=%#RX32 cs:rip=%04x:%08RX64\n", rc, GCPhysNestedFault, GCPhysPage,
1349 (uint32_t)uErr, pCtx->cs.Sel, pCtx->rip));
1350 return VERR_PGM_MAPPING_IPE;
1351
1352# else /* !VBOX_WITH_NESTED_HWVIRT_VMX_EPT || PGM_GST_TYPE != PGM_TYPE_PROT || PGM_SHW_TYPE != PGM_TYPE_EPT */
1353 RT_NOREF7(pVCpu, uErr, pCtx, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk);
1354 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1355 return VERR_PGM_NOT_USED_IN_MODE;
1356# endif
1357}
1358
1359#endif /* !IN_RING3 */
1360
1361
1362/**
1363 * Emulation of the invlpg instruction.
1364 *
1365 *
1366 * @returns VBox status code.
1367 *
1368 * @param pVCpu The cross context virtual CPU structure.
1369 * @param GCPtrPage Page to invalidate.
1370 *
1371 * @remark ASSUMES that the guest is updating before invalidating. This order
1372 * isn't required by the CPU, so this is speculative and could cause
1373 * trouble.
1374 * @remark No TLB shootdown is done on any other VCPU as we assume that
1375 * invlpg emulation is the *only* reason for calling this function.
1376 * (The guest has to shoot down TLB entries on other CPUs itself)
1377 * Currently true, but keep in mind!
1378 *
1379 * @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1380 * Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1381 */
1382PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1383{
1384#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1385 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1386 && PGM_SHW_TYPE != PGM_TYPE_NONE
1387 int rc;
1388 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1389 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1390
1391 PGM_LOCK_ASSERT_OWNER(pVM);
1392
1393 LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1394
1395 /*
1396 * Get the shadow PD entry and skip out if this PD isn't present.
1397 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1398 */
1399# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1400 const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1401 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1402
1403 /* Fetch the pgm pool shadow descriptor. */
1404 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1405# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1406 if (!pShwPde)
1407 {
1408 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1409 return VINF_SUCCESS;
1410 }
1411# else
1412 Assert(pShwPde);
1413# endif
1414
1415# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1416 const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1417 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1418
1419 /* If the shadow PDPE isn't present, then skip the invalidate. */
1420# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1421 if (!pPdptDst || !(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1422# else
1423 if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1424# endif
1425 {
1426 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1427 PGM_INVL_PG(pVCpu, GCPtrPage);
1428 return VINF_SUCCESS;
1429 }
1430
1431 /* Fetch the pgm pool shadow descriptor. */
1432 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1433 AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1434
1435 PX86PDPAE pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1436 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1437 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1438
1439# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1440 /* PML4 */
1441 /*const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/
1442 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1443 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1444 PX86PDPAE pPDDst;
1445 PX86PDPT pPdptDst;
1446 PX86PML4E pPml4eDst;
1447 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1448 if (rc != VINF_SUCCESS)
1449 {
1450 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1451 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1452 PGM_INVL_PG(pVCpu, GCPtrPage);
1453 return VINF_SUCCESS;
1454 }
1455 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1456 Assert(pPDDst);
1457 Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P);
1458
1459 /* Fetch the pgm pool shadow descriptor. */
1460 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1461 Assert(pShwPde);
1462
1463# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1464
1465 const SHWPDE PdeDst = *pPdeDst;
1466 if (!(PdeDst.u & X86_PDE_P))
1467 {
1468 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1469 PGM_INVL_PG(pVCpu, GCPtrPage);
1470 return VINF_SUCCESS;
1471 }
1472
1473 /*
1474 * Get the guest PD entry and calc big page.
1475 */
1476# if PGM_GST_TYPE == PGM_TYPE_32BIT
1477 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1478 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1479 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1480# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1481 unsigned iPDSrc = 0;
1482# if PGM_GST_TYPE == PGM_TYPE_PAE
1483 X86PDPE PdpeSrcIgn;
1484 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1485# else /* AMD64 */
1486 PX86PML4E pPml4eSrcIgn;
1487 X86PDPE PdpeSrcIgn;
1488 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1489# endif
1490 GSTPDE PdeSrc;
1491
1492 if (pPDSrc)
1493 PdeSrc = pPDSrc->a[iPDSrc];
1494 else
1495 PdeSrc.u = 0;
1496# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1497 const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1498 const bool fIsBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1499 if (fWasBigPage != fIsBigPage)
1500 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1501
1502# ifdef IN_RING3
1503 /*
1504 * If a CR3 Sync is pending we may ignore the invalidate page operation
1505 * depending on the kind of sync and if it's a global page or not.
1506 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
1507 */
1508# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1509 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1510 || ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1511 && fIsBigPage
1512 && (PdeSrc.u & X86_PDE4M_G)
1513 )
1514 )
1515# else
1516 if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1517# endif
1518 {
1519 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1520 return VINF_SUCCESS;
1521 }
1522# endif /* IN_RING3 */
1523
1524 /*
1525 * Deal with the Guest PDE.
1526 */
1527 rc = VINF_SUCCESS;
1528 if (PdeSrc.u & X86_PDE_P)
1529 {
1530 Assert( (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1531 && ((PdeSrc.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW) || pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1532 if (!fIsBigPage)
1533 {
1534 /*
1535 * 4KB - page.
1536 */
1537 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1538 RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1539
1540# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1541 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1542 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1543# endif
1544 if (pShwPage->GCPhys == GCPhys)
1545 {
1546 /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1547 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1548
1549 PGSTPT pPTSrc;
1550 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1551 if (RT_SUCCESS(rc))
1552 {
1553 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1554 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1555 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1556 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1557 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1558 GCPtrPage, PteSrc.u & X86_PTE_P,
1559 (PteSrc.u & PdeSrc.u & X86_PTE_RW),
1560 (PteSrc.u & PdeSrc.u & X86_PTE_US),
1561 (uint64_t)PteSrc.u,
1562 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1563 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1564 }
1565 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1566 PGM_INVL_PG(pVCpu, GCPtrPage);
1567 }
1568 else
1569 {
1570 /*
1571 * The page table address changed.
1572 */
1573 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1574 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1575 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1576 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1577 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1578 PGM_INVL_VCPU_TLBS(pVCpu);
1579 }
1580 }
1581 else
1582 {
1583 /*
1584 * 2/4MB - page.
1585 */
1586 /* Before freeing the page, check if anything really changed. */
1587 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1588 RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1589# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1590 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1591 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1592# endif
1593 if ( pShwPage->GCPhys == GCPhys
1594 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1595 {
1596 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1597 /** @todo This test is wrong as it cannot check the G bit!
1598 * FIXME */
1599 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1600 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1601 && ( (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1602 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1603 {
1604 LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1605 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1606 return VINF_SUCCESS;
1607 }
1608 }
1609
1610 /*
1611 * Ok, the page table is present and it's been changed in the guest.
1612 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1613 * We could do this for some flushes in GC too, but we need an algorithm for
1614 * deciding which 4MB pages containing code likely to be executed very soon.
1615 */
1616 LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1617 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1618 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1619 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1620 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1621 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1622 }
1623 }
1624 else
1625 {
1626 /*
1627 * Page directory is not present, mark shadow PDE not present.
1628 */
1629 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1630 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1631 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1632 PGM_INVL_PG(pVCpu, GCPtrPage);
1633 }
1634 return rc;
1635
1636#else /* guest real and protected mode, nested + ept, none. */
1637 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1638 NOREF(pVCpu); NOREF(GCPtrPage);
1639 return VINF_SUCCESS;
1640#endif
1641}
1642
1643#if PGM_SHW_TYPE != PGM_TYPE_NONE
1644
1645/**
1646 * Update the tracking of shadowed pages.
1647 *
1648 * @param pVCpu The cross context virtual CPU structure.
1649 * @param pShwPage The shadow page.
1650 * @param HCPhys The physical page we is being dereferenced.
1651 * @param iPte Shadow PTE index
1652 * @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1653 */
1654DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1655 RTGCPHYS GCPhysPage)
1656{
1657 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1658
1659# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1660 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1661 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1662
1663 /* Use the hint we retrieved from the cached guest PT. */
1664 if (pShwPage->fDirty)
1665 {
1666 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1667
1668 Assert(pShwPage->cPresent);
1669 Assert(pPool->cPresent);
1670 pShwPage->cPresent--;
1671 pPool->cPresent--;
1672
1673 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1674 AssertRelease(pPhysPage);
1675 pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1676 return;
1677 }
1678# else
1679 NOREF(GCPhysPage);
1680# endif
1681
1682 STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a);
1683 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1684
1685 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1686 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1687 * 2. write protect all shadowed pages. I.e. implement caching.
1688 */
1689 /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1690
1691 /*
1692 * Find the guest address.
1693 */
1694 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1695 pRam;
1696 pRam = pRam->CTX_SUFF(pNext))
1697 {
1698 unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT;
1699 while (iPage-- > 0)
1700 {
1701 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1702 {
1703 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1704
1705 Assert(pShwPage->cPresent);
1706 Assert(pPool->cPresent);
1707 pShwPage->cPresent--;
1708 pPool->cPresent--;
1709
1710 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1711 STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a);
1712 return;
1713 }
1714 }
1715 }
1716
1717 for (;;)
1718 AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1719}
1720
1721
1722/**
1723 * Update the tracking of shadowed pages.
1724 *
1725 * @param pVCpu The cross context virtual CPU structure.
1726 * @param pShwPage The shadow page.
1727 * @param u16 The top 16-bit of the pPage->HCPhys.
1728 * @param pPage Pointer to the guest page. this will be modified.
1729 * @param iPTDst The index into the shadow table.
1730 */
1731DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16,
1732 PPGMPAGE pPage, const unsigned iPTDst)
1733{
1734 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1735
1736 /*
1737 * Just deal with the simple first time here.
1738 */
1739 if (!u16)
1740 {
1741 STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin);
1742 u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1743 /* Save the page table index. */
1744 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1745 }
1746 else
1747 u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1748
1749 /* write back */
1750 Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1751 PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1752
1753 /* update statistics. */
1754 pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1755 pShwPage->cPresent++;
1756 if (pShwPage->iFirstPresent > iPTDst)
1757 pShwPage->iFirstPresent = iPTDst;
1758}
1759
1760
1761/**
1762 * Modifies a shadow PTE to account for access handlers.
1763 *
1764 * @param pVM The cross context VM structure.
1765 * @param pVCpu The cross context virtual CPU structure.
1766 * @param pPage The page in question.
1767 * @param GCPhysPage The guest-physical address of the page.
1768 * @param fPteSrc The shadowed flags of the source PTE. Must include the
1769 * A (accessed) bit so it can be emulated correctly.
1770 * @param pPteDst The shadow PTE (output). This is temporary storage and
1771 * does not need to be set atomically.
1772 */
1773DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PVMCPUCC pVCpu, PCPGMPAGE pPage, RTGCPHYS GCPhysPage, uint64_t fPteSrc,
1774 PSHWPTE pPteDst)
1775{
1776 RT_NOREF_PV(pVM); RT_NOREF_PV(fPteSrc); RT_NOREF_PV(pVCpu); RT_NOREF_PV(GCPhysPage);
1777
1778 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1779 * Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1780 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1781 {
1782 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1783# if PGM_SHW_TYPE == PGM_TYPE_EPT
1784 pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) | EPT_E_READ | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
1785# else
1786 if (fPteSrc & X86_PTE_A)
1787 {
1788 SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage));
1789 SHW_PTE_SET_RO(*pPteDst);
1790 }
1791 else
1792 SHW_PTE_SET(*pPteDst, 0);
1793# endif
1794 }
1795# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1796# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
1797 else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1798 && ( BTH_IS_NP_ACTIVE(pVM)
1799 || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1800# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1801 && pVM->pgm.s.fLessThan52PhysicalAddressBits
1802# endif
1803 )
1804 {
1805 LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1806# if PGM_SHW_TYPE == PGM_TYPE_EPT
1807 /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1808 pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg
1809 /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1810 | EPT_E_WRITE
1811 /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1812 | EPT_E_MEMTYPE_INVALID_3;
1813# else
1814 /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1815 SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P);
1816# endif
1817 }
1818# endif
1819# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1820 else
1821 {
1822 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1823 SHW_PTE_SET(*pPteDst, 0);
1824 }
1825 /** @todo count these kinds of entries. */
1826}
1827
1828
1829/**
1830 * Creates a 4K shadow page for a guest page.
1831 *
1832 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1833 * physical address. The PdeSrc argument only the flags are used. No page
1834 * structured will be mapped in this function.
1835 *
1836 * @param pVCpu The cross context virtual CPU structure.
1837 * @param pPteDst Destination page table entry.
1838 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1839 * Can safely assume that only the flags are being used.
1840 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1841 * @param pShwPage Pointer to the shadow page.
1842 * @param iPTDst The index into the shadow table.
1843 *
1844 * @remark Not used for 2/4MB pages!
1845 */
1846# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
1847static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1848 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1849# else
1850static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1851 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1852# endif
1853{
1854 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1855 RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1856
1857# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1858 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1859 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1860
1861 if (pShwPage->fDirty)
1862 {
1863 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1864 PGSTPT pGstPT;
1865
1866 /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1867 pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1868 GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1869 pGstPT->a[iPTDst].u = PteSrc.u;
1870 }
1871# else
1872 Assert(!pShwPage->fDirty);
1873# endif
1874
1875# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1876 if ( (PteSrc.u & X86_PTE_P)
1877 && GST_IS_PTE_VALID(pVCpu, PteSrc))
1878# endif
1879 {
1880# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1881 RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1882# endif
1883 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1884
1885 /*
1886 * Find the ram range.
1887 */
1888 PPGMPAGE pPage;
1889 int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1890 if (RT_SUCCESS(rc))
1891 {
1892 /* Ignore ballooned pages.
1893 Don't return errors or use a fatal assert here as part of a
1894 shadow sync range might included ballooned pages. */
1895 if (PGM_PAGE_IS_BALLOONED(pPage))
1896 {
1897 Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */
1898 return;
1899 }
1900
1901# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1902 /* Make the page writable if necessary. */
1903 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1904 && ( PGM_PAGE_IS_ZERO(pPage)
1905# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1906 || ( (PteSrc.u & X86_PTE_RW)
1907# else
1908 || ( 1
1909# endif
1910 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1911# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1912 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1913# endif
1914# ifdef VBOX_WITH_PAGE_SHARING
1915 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1916# endif
1917 )
1918 )
1919 )
1920 {
1921 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1922 AssertRC(rc);
1923 }
1924# endif
1925
1926 /*
1927 * Make page table entry.
1928 */
1929 SHWPTE PteDst;
1930# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1931 uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1932# else
1933 uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D;
1934# endif
1935 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
1936 {
1937# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1938 /*
1939 * If the page or page directory entry is not marked accessed,
1940 * we mark the page not present.
1941 */
1942 if (!(PteSrc.u & X86_PTE_A) || !(PdeSrc.u & X86_PDE_A))
1943 {
1944 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1945 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage));
1946 SHW_PTE_SET(PteDst, 0);
1947 }
1948 /*
1949 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1950 * when the page is modified.
1951 */
1952 else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW))
1953 {
1954 AssertCompile(X86_PTE_RW == X86_PDE_RW);
1955 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage));
1956 SHW_PTE_SET(PteDst,
1957 fGstShwPteFlags
1958 | PGM_PAGE_GET_HCPHYS(pPage)
1959 | PGM_PTFLAGS_TRACK_DIRTY);
1960 SHW_PTE_SET_RO(PteDst);
1961 }
1962 else
1963# endif
1964 {
1965 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped));
1966# if PGM_SHW_TYPE == PGM_TYPE_EPT
1967 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage)
1968 | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
1969# else
1970 SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage));
1971# endif
1972 }
1973
1974 /*
1975 * Make sure only allocated pages are mapped writable.
1976 */
1977 if ( SHW_PTE_IS_P_RW(PteDst)
1978 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1979 {
1980 /* Still applies to shared pages. */
1981 Assert(!PGM_PAGE_IS_ZERO(pPage));
1982 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1983 Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1984 }
1985 }
1986 else
1987 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhysPage, fGstShwPteFlags, &PteDst);
1988
1989 /*
1990 * Keep user track up to date.
1991 */
1992 if (SHW_PTE_IS_P(PteDst))
1993 {
1994 if (!SHW_PTE_IS_P(*pPteDst))
1995 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1996 else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1997 {
1998 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst)));
1999 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2000 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2001 }
2002 }
2003 else if (SHW_PTE_IS_P(*pPteDst))
2004 {
2005 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
2006 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2007 }
2008
2009 /*
2010 * Update statistics and commit the entry.
2011 */
2012# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2013 if (!(PteSrc.u & X86_PTE_G))
2014 pShwPage->fSeenNonGlobal = true;
2015# endif
2016 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2017 return;
2018 }
2019
2020/** @todo count these three different kinds. */
2021 Log2(("SyncPageWorker: invalid address in Pte\n"));
2022 }
2023# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2024 else if (!(PteSrc.u & X86_PTE_P))
2025 Log2(("SyncPageWorker: page not present in Pte\n"));
2026 else
2027 Log2(("SyncPageWorker: invalid Pte\n"));
2028# endif
2029
2030 /*
2031 * The page is not present or the PTE is bad. Replace the shadow PTE by
2032 * an empty entry, making sure to keep the user tracking up to date.
2033 */
2034 if (SHW_PTE_IS_P(*pPteDst))
2035 {
2036 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
2037 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2038 }
2039 SHW_PTE_ATOMIC_SET(*pPteDst, 0);
2040}
2041
2042
2043/**
2044 * Syncs a guest OS page.
2045 *
2046 * There are no conflicts at this point, neither is there any need for
2047 * page table allocations.
2048 *
2049 * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
2050 * When called in AMD64 guest mode, the guest PML4E shall be valid.
2051 *
2052 * @returns VBox status code.
2053 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
2054 * @param pVCpu The cross context virtual CPU structure.
2055 * @param PdeSrc Page directory entry of the guest.
2056 * @param GCPtrPage Guest context page address.
2057 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2058 * @param uErr Fault error (X86_TRAP_PF_*).
2059 */
2060static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
2061{
2062 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2063 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2064 LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
2065 RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
2066
2067 PGM_LOCK_ASSERT_OWNER(pVM);
2068
2069# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2070 || PGM_GST_TYPE == PGM_TYPE_PAE \
2071 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2072 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
2073
2074 /*
2075 * Assert preconditions.
2076 */
2077 Assert(PdeSrc.u & X86_PDE_P);
2078 Assert(cPages);
2079# if 0 /* rarely useful; leave for debugging. */
2080 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
2081# endif
2082
2083 /*
2084 * Get the shadow PDE, find the shadow page table in the pool.
2085 */
2086# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2087 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2088 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2089
2090 /* Fetch the pgm pool shadow descriptor. */
2091 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2092 Assert(pShwPde);
2093
2094# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2095 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2096 PPGMPOOLPAGE pShwPde = NULL;
2097 PX86PDPAE pPDDst;
2098
2099 /* Fetch the pgm pool shadow descriptor. */
2100 int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2101 AssertRCSuccessReturn(rc2, rc2);
2102 Assert(pShwPde);
2103
2104 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2105 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2106
2107# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2108 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2109 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2110 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2111 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2112
2113 int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2114 AssertRCSuccessReturn(rc2, rc2);
2115 Assert(pPDDst && pPdptDst);
2116 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2117# endif
2118 SHWPDE PdeDst = *pPdeDst;
2119
2120 /*
2121 * - In the guest SMP case we could have blocked while another VCPU reused
2122 * this page table.
2123 * - With W7-64 we may also take this path when the A bit is cleared on
2124 * higher level tables (PDPE/PML4E). The guest does not invalidate the
2125 * relevant TLB entries. If we're write monitoring any page mapped by
2126 * the modified entry, we may end up here with a "stale" TLB entry.
2127 */
2128 if (!(PdeDst.u & X86_PDE_P))
2129 {
2130 Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2131 AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW),
2132 ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2133 if (uErr & X86_TRAP_PF_P)
2134 PGM_INVL_PG(pVCpu, GCPtrPage);
2135 return VINF_SUCCESS; /* force the instruction to be executed again. */
2136 }
2137
2138 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2139 Assert(pShwPage);
2140
2141# if PGM_GST_TYPE == PGM_TYPE_AMD64
2142 /* Fetch the pgm pool shadow descriptor. */
2143 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2144 Assert(pShwPde);
2145# endif
2146
2147 /*
2148 * Check that the page is present and that the shadow PDE isn't out of sync.
2149 */
2150 const bool fBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
2151 const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
2152 RTGCPHYS GCPhys;
2153 if (!fBigPage)
2154 {
2155 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2156# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2157 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2158 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
2159# endif
2160 }
2161 else
2162 {
2163 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2164# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2165 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2166 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2167# endif
2168 }
2169 /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2170 if ( fPdeValid
2171 && pShwPage->GCPhys == GCPhys
2172 && (PdeSrc.u & X86_PDE_P)
2173 && (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
2174 && ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW))
2175# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2176 && ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) || !GST_IS_NX_ACTIVE(pVCpu))
2177# endif
2178 )
2179 {
2180 /*
2181 * Check that the PDE is marked accessed already.
2182 * Since we set the accessed bit *before* getting here on a #PF, this
2183 * check is only meant for dealing with non-#PF'ing paths.
2184 */
2185 if (PdeSrc.u & X86_PDE_A)
2186 {
2187 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2188 if (!fBigPage)
2189 {
2190 /*
2191 * 4KB Page - Map the guest page table.
2192 */
2193 PGSTPT pPTSrc;
2194 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2195 if (RT_SUCCESS(rc))
2196 {
2197# ifdef PGM_SYNC_N_PAGES
2198 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2199 if ( cPages > 1
2200 && !(uErr & X86_TRAP_PF_P)
2201 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2202 {
2203 /*
2204 * This code path is currently only taken when the caller is PGMTrap0eHandler
2205 * for non-present pages!
2206 *
2207 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2208 * deal with locality.
2209 */
2210 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2211# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2212 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2213 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2214# else
2215 const unsigned offPTSrc = 0;
2216# endif
2217 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2218 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2219 iPTDst = 0;
2220 else
2221 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2222
2223 for (; iPTDst < iPTDstEnd; iPTDst++)
2224 {
2225 const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2226
2227 if ( (pPteSrc->u & X86_PTE_P)
2228 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2229 {
2230 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT))
2231 | ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT);
2232 NOREF(GCPtrCurPage);
2233 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2234 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2235 GCPtrCurPage, pPteSrc->u & X86_PTE_P,
2236 !!(pPteSrc->u & PdeSrc.u & X86_PTE_RW),
2237 !!(pPteSrc->u & PdeSrc.u & X86_PTE_US),
2238 (uint64_t)pPteSrc->u,
2239 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2240 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2241 }
2242 }
2243 }
2244 else
2245# endif /* PGM_SYNC_N_PAGES */
2246 {
2247 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2248 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2249 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2250 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2251 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2252 GCPtrPage, PteSrc.u & X86_PTE_P,
2253 !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
2254 !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
2255 (uint64_t)PteSrc.u,
2256 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2257 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2258 }
2259 }
2260 else /* MMIO or invalid page: emulated in #PF handler. */
2261 {
2262 LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2263 Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2264 }
2265 }
2266 else
2267 {
2268 /*
2269 * 4/2MB page - lazy syncing shadow 4K pages.
2270 * (There are many causes of getting here, it's no longer only CSAM.)
2271 */
2272 /* Calculate the GC physical address of this 4KB shadow page. */
2273 GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2274 /* Find ram range. */
2275 PPGMPAGE pPage;
2276 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2277 if (RT_SUCCESS(rc))
2278 {
2279 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2280
2281# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2282 /* Try to make the page writable if necessary. */
2283 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2284 && ( PGM_PAGE_IS_ZERO(pPage)
2285 || ( (PdeSrc.u & X86_PDE_RW)
2286 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2287# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2288 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2289# endif
2290# ifdef VBOX_WITH_PAGE_SHARING
2291 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2292# endif
2293 )
2294 )
2295 )
2296 {
2297 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2298 AssertRC(rc);
2299 }
2300# endif
2301
2302 /*
2303 * Make shadow PTE entry.
2304 */
2305 SHWPTE PteDst;
2306 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2307 SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
2308 else
2309 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2310
2311 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2312 if ( SHW_PTE_IS_P(PteDst)
2313 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2314 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2315
2316 /* Make sure only allocated pages are mapped writable. */
2317 if ( SHW_PTE_IS_P_RW(PteDst)
2318 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2319 {
2320 /* Still applies to shared pages. */
2321 Assert(!PGM_PAGE_IS_ZERO(pPage));
2322 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2323 Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2324 }
2325
2326 SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2327
2328 /*
2329 * If the page is not flagged as dirty and is writable, then make it read-only
2330 * at PD level, so we can set the dirty bit when the page is modified.
2331 *
2332 * ASSUMES that page access handlers are implemented on page table entry level.
2333 * Thus we will first catch the dirty access and set PDE.D and restart. If
2334 * there is an access handler, we'll trap again and let it work on the problem.
2335 */
2336 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2337 * As for invlpg, it simply frees the whole shadow PT.
2338 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2339 if ((PdeSrc.u & (X86_PDE4M_D | X86_PDE_RW)) == X86_PDE_RW)
2340 {
2341 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2342 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2343 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2344 }
2345 else
2346 {
2347 PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY | X86_PDE_RW);
2348 PdeDst.u |= PdeSrc.u & X86_PDE_RW;
2349 }
2350 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2351 Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2352 GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US),
2353 (uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2354 }
2355 else
2356 {
2357 LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2358 /** @todo must wipe the shadow page table entry in this
2359 * case. */
2360 }
2361 }
2362 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2363 return VINF_SUCCESS;
2364 }
2365
2366 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs));
2367 }
2368 else if (fPdeValid)
2369 {
2370 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2371 Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2372 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2373 }
2374 else
2375 {
2376/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2377 Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2378 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2379 }
2380
2381 /*
2382 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2383 * Yea, I'm lazy.
2384 */
2385 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2386 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
2387
2388 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2389 PGM_INVL_VCPU_TLBS(pVCpu);
2390 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2391
2392
2393# elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2394 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2395 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
2396 NOREF(PdeSrc);
2397
2398# ifdef PGM_SYNC_N_PAGES
2399 /*
2400 * Get the shadow PDE, find the shadow page table in the pool.
2401 */
2402# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2403 X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2404
2405# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2406 X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2407
2408# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2409 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2410 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2411 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2412 X86PDEPAE PdeDst;
2413 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2414
2415 int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2416 AssertRCSuccessReturn(rc, rc);
2417 Assert(pPDDst && pPdptDst);
2418 PdeDst = pPDDst->a[iPDDst];
2419
2420# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2421 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2422 PEPTPD pPDDst;
2423 EPTPDE PdeDst;
2424
2425 int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2426 if (rc != VINF_SUCCESS)
2427 {
2428 AssertRC(rc);
2429 return rc;
2430 }
2431 Assert(pPDDst);
2432 PdeDst = pPDDst->a[iPDDst];
2433# endif
2434 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2435 if (!SHW_PDE_IS_P(PdeDst))
2436 {
2437 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2438 Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2439 return VINF_SUCCESS; /* force the instruction to be executed again. */
2440 }
2441
2442 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2443 if (SHW_PDE_IS_BIG(PdeDst))
2444 {
2445 Assert(pVM->pgm.s.fNestedPaging);
2446 Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2447 return VINF_SUCCESS;
2448 }
2449
2450 /* Mask away the page offset. */
2451 GCPtrPage &= ~((RTGCPTR)0xfff);
2452
2453 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2454 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2455
2456 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2457 if ( cPages > 1
2458 && !(uErr & X86_TRAP_PF_P)
2459 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2460 {
2461 /*
2462 * This code path is currently only taken when the caller is PGMTrap0eHandler
2463 * for non-present pages!
2464 *
2465 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2466 * deal with locality.
2467 */
2468 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2469 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2470 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2471 iPTDst = 0;
2472 else
2473 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2474 for (; iPTDst < iPTDstEnd; iPTDst++)
2475 {
2476 if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2477 {
2478 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2479 | (iPTDst << GUEST_PAGE_SHIFT));
2480
2481 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2482 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2483 GCPtrCurPage,
2484 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2485 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2486
2487 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2488 break;
2489 }
2490 else
2491 Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n",
2492 (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2493 }
2494 }
2495 else
2496# endif /* PGM_SYNC_N_PAGES */
2497 {
2498 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2499 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2500 | (iPTDst << GUEST_PAGE_SHIFT));
2501
2502 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2503
2504 Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2505 GCPtrPage,
2506 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2507 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2508 }
2509 return VINF_SUCCESS;
2510
2511# else
2512 NOREF(PdeSrc);
2513 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2514 return VERR_PGM_NOT_USED_IN_MODE;
2515# endif
2516}
2517
2518#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
2519
2520#if !defined(IN_RING3) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
2521
2522/**
2523 * Sync a shadow page for a nested-guest page.
2524 *
2525 * @param pVCpu The cross context virtual CPU structure.
2526 * @param pPte The shadow page table entry.
2527 * @param GCPhysPage The guest-physical address of the page.
2528 * @param pShwPage The shadow page of the page table.
2529 * @param iPte The index of the page table entry.
2530 * @param pGstWalkAll The guest page table walk result.
2531 *
2532 * @note Not to be used for 2/4MB pages!
2533 */
2534static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
2535 unsigned iPte, PCPGMPTWALKGST pGstWalkAll)
2536{
2537 /*
2538 * Do not make assumptions about anything other than the final PTE entry in the
2539 * guest page table walk result. For instance, while mapping 2M PDEs as 4K pages,
2540 * the PDE might still be having its leaf bit set.
2541 *
2542 * In the future, we could consider introducing a generic SLAT macro like PSLATPTE
2543 * and using that instead of passing the full SLAT translation result.
2544 */
2545 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2546 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
2547 Assert(!pShwPage->fDirty);
2548 Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
2549 AssertMsg(!(pGstWalkAll->u.Ept.Pte.u & EPT_E_LEAF), ("Large page unexpected: %RX64\n", pGstWalkAll->u.Ept.Pte.u));
2550 AssertMsg((pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK) == GCPhysPage,
2551 ("PTE address mismatch. GCPhysPage=%RGp Pte=%RX64\n", GCPhysPage, pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK));
2552
2553 /*
2554 * Find the ram range.
2555 */
2556 PPGMPAGE pPage;
2557 int rc = pgmPhysGetPageEx(pVCpu->CTX_SUFF(pVM), GCPhysPage, &pPage);
2558 if (RT_SUCCESS(rc))
2559 { /* likely */ }
2560 else
2561 {
2562 /*
2563 * This is a RAM hole/invalid/reserved address (not MMIO).
2564 * Nested Microsoft Hyper-V maps addresses like 0xf0220000 as RW WB memory.
2565 * Shadow a not-present page similar to MMIO, see @bugref{10318#c7}.
2566 */
2567 Assert(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS);
2568 if (SHW_PTE_IS_P(*pPte))
2569 {
2570 Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
2571 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2572 }
2573 Log7Func(("RAM hole/reserved %RGp -> ShwPte=0\n", GCPhysPage));
2574 SHW_PTE_ATOMIC_SET(*pPte, 0);
2575 return;
2576 }
2577
2578 Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2579
2580 /*
2581 * Make page table entry.
2582 */
2583 SHWPTE Pte;
2584 uint64_t const fGstShwPteFlags = pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptShadowedPteMask;
2585 if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
2586 {
2587# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2588 /* Page wasn't allocated, write protect it. */
2589 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2590 && ( PGM_PAGE_IS_ZERO(pPage)
2591 || ( (pGstWalkAll->u.Ept.Pte.u & EPT_E_WRITE)
2592 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2593# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2594 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2595# endif
2596# ifdef VBOX_WITH_PAGE_SHARING
2597 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2598# endif
2599 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_BALLOONED
2600 )
2601 )
2602 )
2603 {
2604 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE);
2605 Log7Func(("zero page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2606 }
2607 else
2608# endif
2609 {
2610 /** @todo access bit. */
2611 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
2612 Log7Func(("regular page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2613 }
2614 }
2615 else if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2616 {
2617 /** @todo access bit. */
2618 Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE);
2619 Log7Func(("monitored page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
2620 }
2621 else
2622 {
2623 /** @todo Do MMIO optimizations here too? */
2624 Log7Func(("mmio/all page (%R[pgmpage]) at %RGp -> 0\n", pPage, GCPhysPage));
2625 Pte.u = 0;
2626 }
2627
2628 /* Make sure only allocated pages are mapped writable. */
2629 Assert(!SHW_PTE_IS_P_RW(Pte) || PGM_PAGE_IS_ALLOCATED(pPage));
2630
2631 /*
2632 * Keep user track up to date.
2633 */
2634 if (SHW_PTE_IS_P(Pte))
2635 {
2636 if (!SHW_PTE_IS_P(*pPte))
2637 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2638 else if (SHW_PTE_GET_HCPHYS(*pPte) != SHW_PTE_GET_HCPHYS(Pte))
2639 {
2640 Log2(("SyncPageWorker: deref! *pPte=%RX64 Pte=%RX64\n", SHW_PTE_LOG64(*pPte), SHW_PTE_LOG64(Pte)));
2641 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2642 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
2643 }
2644 }
2645 else if (SHW_PTE_IS_P(*pPte))
2646 {
2647 Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
2648 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
2649 }
2650
2651 /*
2652 * Commit the entry.
2653 */
2654 SHW_PTE_ATOMIC_SET2(*pPte, Pte);
2655 return;
2656}
2657
2658
2659/**
2660 * Syncs a nested-guest page.
2661 *
2662 * There are no conflicts at this point, neither is there any need for
2663 * page table allocations.
2664 *
2665 * @returns VBox status code.
2666 * @param pVCpu The cross context virtual CPU structure.
2667 * @param GCPhysNestedPage The nested-guest physical address of the page being
2668 * synced.
2669 * @param GCPhysPage The guest-physical address of the page being synced.
2670 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2671 * @param uErr The page fault error (X86_TRAP_PF_XXX).
2672 * @param pGstWalkAll The guest page table walk result.
2673 */
2674static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
2675 uint32_t uErr, PPGMPTWALKGST pGstWalkAll)
2676{
2677 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2678 Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2679 Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2680
2681 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2682 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2683 Log7Func(("GCPhysNestedPage=%RGv GCPhysPage=%RGp cPages=%u uErr=%#x\n", GCPhysNestedPage, GCPhysPage, cPages, uErr));
2684 RT_NOREF_PV(uErr); RT_NOREF_PV(cPages);
2685
2686 PGM_LOCK_ASSERT_OWNER(pVM);
2687
2688 /*
2689 * Get the shadow PDE, find the shadow page table in the pool.
2690 */
2691 unsigned const iPde = ((GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK);
2692 PEPTPD pPd;
2693 int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL, &pPd, pGstWalkAll);
2694 if (RT_SUCCESS(rc))
2695 { /* likely */ }
2696 else
2697 {
2698 Log(("Failed to fetch EPT PD for %RGp (%RGp) rc=%Rrc\n", GCPhysNestedPage, GCPhysPage, rc));
2699 return rc;
2700 }
2701 Assert(pPd);
2702 EPTPDE Pde = pPd->a[iPde];
2703
2704 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2705 if (!SHW_PDE_IS_P(Pde))
2706 {
2707 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)Pde.u));
2708 Log7Func(("CPU%d: SyncPage: Pde at %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2709 return VINF_SUCCESS; /* force the instruction to be executed again. */
2710 }
2711
2712 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2713 if (SHW_PDE_IS_BIG(Pde))
2714 {
2715 Log7Func(("CPU%d: SyncPage: %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
2716 return VINF_SUCCESS;
2717 }
2718
2719 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, Pde.u & EPT_PDE_PG_MASK);
2720 PEPTPT pPt = (PEPTPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2721
2722 /*
2723 * If we've shadowed a guest EPT PDE that maps a 2M page using a 4K table,
2724 * then sync the 4K sub-page in the 2M range.
2725 */
2726 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2727 {
2728 Assert(!SHW_PDE_IS_BIG(Pde));
2729
2730 Assert(pGstWalkAll->u.Ept.Pte.u == 0);
2731 Assert((Pde.u & EPT_PRESENT_MASK) == (pGstWalkAll->u.Ept.Pde.u & EPT_PRESENT_MASK));
2732 Assert(pShwPage->GCPhys == (pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK));
2733
2734#if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
2735 PPGMPAGE pPage;
2736 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); AssertRC(rc);
2737 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
2738 Assert(pShwPage->enmKind == PGMPOOLKIND_EPT_PT_FOR_EPT_2MB);
2739#endif
2740 uint64_t const fGstPteFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask & ~EPT_E_LEAF;
2741 pGstWalkAll->u.Ept.Pte.u = GCPhysPage | fGstPteFlags;
2742
2743 unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2744 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
2745 Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2746
2747 /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
2748 pGstWalkAll->u.Ept.Pte.u = 0;
2749 return VINF_SUCCESS;
2750 }
2751
2752 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2753# ifdef PGM_SYNC_N_PAGES
2754 if ( cPages > 1
2755 && !(uErr & X86_TRAP_PF_P)
2756 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2757 {
2758 /*
2759 * This code path is currently only taken for non-present pages!
2760 *
2761 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2762 * deal with locality.
2763 */
2764 unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2765 unsigned const iPteEnd = RT_MIN(iPte + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPt->a));
2766 if (iPte < PGM_SYNC_NR_PAGES / 2)
2767 iPte = 0;
2768 else
2769 iPte -= PGM_SYNC_NR_PAGES / 2;
2770 for (; iPte < iPteEnd; iPte++)
2771 {
2772 if (!SHW_PTE_IS_P(pPt->a[iPte]))
2773 {
2774 PGMPTWALKGST GstWalkPt;
2775 PGMPTWALK WalkPt;
2776 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2777 GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
2778 rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt,
2779 &GstWalkPt);
2780 if (RT_SUCCESS(rc))
2781 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], WalkPt.GCPhys, pShwPage, iPte, &GstWalkPt);
2782 else
2783 {
2784 /*
2785 * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
2786 * Ensure the shadow tables entry is not-present.
2787 */
2788 /** @todo Potential room for optimization (explained in NestedSyncPT). */
2789 AssertMsg(!pPt->a[iPte].u, ("%RX64\n", pPt->a[iPte].u));
2790 }
2791 Log7Func(("Many: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2792 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2793 break;
2794 }
2795 else
2796 {
2797# ifdef VBOX_STRICT
2798 /* Paranoia - Verify address of the page is what it should be. */
2799 PGMPTWALKGST GstWalkPt;
2800 PGMPTWALK WalkPt;
2801 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
2802 GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
2803 rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt, &GstWalkPt);
2804 AssertRC(rc);
2805 PPGMPAGE pPage;
2806 rc = pgmPhysGetPageEx(pVM, WalkPt.GCPhys, &pPage);
2807 AssertRC(rc);
2808 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]),
2809 ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp HCPhys=%RHp Shw=%RHp\n",
2810 GCPhysNestedPage, WalkPt.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
2811# endif
2812 Log7Func(("Many3: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2813 }
2814 }
2815 }
2816 else
2817# endif /* PGM_SYNC_N_PAGES */
2818 {
2819 unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2820 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
2821 Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
2822 }
2823
2824 return VINF_SUCCESS;
2825}
2826
2827
2828/**
2829 * Sync a shadow page table for a nested-guest page table.
2830 *
2831 * The shadow page table is not present in the shadow PDE.
2832 *
2833 * Handles mapping conflicts.
2834 *
2835 * A precondition for this method is that the shadow PDE is not present. The
2836 * caller must take the PGM lock before checking this and continue to hold it
2837 * when calling this method.
2838 *
2839 * @returns VBox status code.
2840 * @param pVCpu The cross context virtual CPU structure.
2841 * @param GCPhysNestedPage The nested-guest physical page address of the page
2842 * being synced.
2843 * @param GCPhysPage The guest-physical address of the page being synced.
2844 * @param pGstWalkAll The guest page table walk result.
2845 */
2846static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll)
2847{
2848 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
2849 Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
2850 Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
2851
2852 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2853 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2854
2855 Log7Func(("GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
2856
2857 PGM_LOCK_ASSERT_OWNER(pVM);
2858 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2859
2860 PEPTPD pPd;
2861 PEPTPDPT pPdpt;
2862 unsigned const iPde = (GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK;
2863 int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, &pPdpt, &pPd, pGstWalkAll);
2864 if (RT_SUCCESS(rc))
2865 { /* likely */ }
2866 else
2867 {
2868 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2869 AssertRC(rc);
2870 return rc;
2871 }
2872 Assert(pPd);
2873 PSHWPDE pPde = &pPd->a[iPde];
2874
2875 unsigned const iPdpt = (GCPhysNestedPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2876 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdpt->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2877 Assert(pShwPde->enmKind == PGMPOOLKIND_EPT_PD_FOR_EPT_PD);
2878
2879 SHWPDE Pde = *pPde;
2880 Assert(!SHW_PDE_IS_P(Pde)); /* We're only supposed to call SyncPT on PDE!P and conflicts. */
2881
2882# ifdef PGM_WITH_LARGE_PAGES
2883 Assert(BTH_IS_NP_ACTIVE(pVM));
2884
2885 /*
2886 * Check if the guest is mapping a 2M page.
2887 */
2888 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
2889 {
2890 PPGMPAGE pPage;
2891 rc = pgmPhysGetPageEx(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK, &pPage);
2892 AssertRCReturn(rc, rc);
2893
2894 /* A20 is always enabled in VMX root and non-root operation. */
2895 Assert(PGM_A20_IS_ENABLED(pVCpu));
2896
2897 /*
2898 * Check if we have or can get a 2M backing page here.
2899 */
2900 RTHCPHYS HCPhys = NIL_RTHCPHYS;
2901 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
2902 {
2903 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
2904 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2905 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2906 }
2907 else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2908 {
2909 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
2910 rc = pgmPhysRecheckLargePage(pVM, GCPhysPage, pPage);
2911 if (RT_SUCCESS(rc))
2912 {
2913 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2914 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2915 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2916 }
2917 }
2918 else if (PGMIsUsingLargePages(pVM))
2919 {
2920 rc = pgmPhysAllocLargePage(pVM, GCPhysPage);
2921 if (RT_SUCCESS(rc))
2922 {
2923 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2924 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2925 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2926 }
2927 }
2928
2929 /*
2930 * If we have a 2M backing page, we can map the guest's 2M page right away.
2931 */
2932 uint64_t const fShwBigPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask;
2933 if (HCPhys != NIL_RTHCPHYS)
2934 {
2935 Pde.u = HCPhys | fShwBigPdeFlags;
2936 Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzBigPdeMask));
2937 Assert(Pde.u & EPT_E_LEAF);
2938 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
2939
2940 /* Add a reference to the first page only. */
2941 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPde);
2942
2943 Assert(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED);
2944
2945 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2946 Log7Func(("GstPde=%RGp ShwPde=%RX64 [2M]\n", pGstWalkAll->u.Ept.Pde.u, Pde.u));
2947 return VINF_SUCCESS;
2948 }
2949
2950 /*
2951 * We didn't get a perfect 2M fit. Split the 2M page into 4K pages.
2952 * The page ought not to be marked as a big (2M) page at this point.
2953 */
2954 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
2955
2956 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2957 PGMPOOLACCESS enmAccess;
2958 {
2959 Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_USER_EXECUTE)); /* Mode-based execute control for EPT not supported. */
2960 bool const fNoExecute = !(pGstWalkAll->u.Ept.Pde.u & EPT_E_EXECUTE);
2961 if (pGstWalkAll->u.Ept.Pde.u & EPT_E_WRITE)
2962 enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2963 else
2964 enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2965 }
2966
2967 /*
2968 * Allocate & map a 4K shadow table to cover the 2M guest page.
2969 */
2970 PPGMPOOLPAGE pShwPage;
2971 RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK;
2972 rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_2MB, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2973 pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
2974 if ( rc == VINF_SUCCESS
2975 || rc == VINF_PGM_CACHED_PAGE)
2976 { /* likely */ }
2977 else
2978 {
2979 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2980 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2981 }
2982
2983 PSHWPT pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2984 Assert(pPt);
2985 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
2986 if (rc == VINF_SUCCESS)
2987 {
2988 /* The 4K PTEs shall inherit the flags of the 2M PDE page sans the leaf bit. */
2989 uint64_t const fShwPteFlags = fShwBigPdeFlags & ~EPT_E_LEAF;
2990
2991 /* Sync each 4K pages in the 2M range. */
2992 for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++)
2993 {
2994 RTGCPHYS const GCPhysSubPage = GCPhysPt | (iPte << GUEST_PAGE_SHIFT);
2995 pGstWalkAll->u.Ept.Pte.u = GCPhysSubPage | fShwPteFlags;
2996 Assert(!(pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptMbzPteMask));
2997 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysSubPage, pShwPage, iPte, pGstWalkAll);
2998 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [2M->4K]\n", pGstWalkAll->u.Ept.Pte, pPt->a[iPte].u, iPte));
2999 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3000 break;
3001 }
3002
3003 /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
3004 pGstWalkAll->u.Ept.Pte.u = 0;
3005 }
3006 else
3007 {
3008 Assert(rc == VINF_PGM_CACHED_PAGE);
3009# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3010 /* Paranoia - Verify address of each of the subpages are what they should be. */
3011 RTGCPHYS GCPhysSubPage = GCPhysPt;
3012 for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++, GCPhysSubPage += GUEST_PAGE_SIZE)
3013 {
3014 PPGMPAGE pSubPage;
3015 rc = pgmPhysGetPageEx(pVM, GCPhysSubPage, &pSubPage);
3016 AssertRC(rc);
3017 AssertMsg( PGM_PAGE_GET_HCPHYS(pSubPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte])
3018 || !SHW_PTE_IS_P(pPt->a[iPte]),
3019 ("PGM 2M page and shadow PTE conflict. GCPhysSubPage=%RGp Page=%RHp Shw=%RHp\n",
3020 GCPhysSubPage, PGM_PAGE_GET_HCPHYS(pSubPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3021 }
3022# endif
3023 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3024 }
3025
3026 /* Save the new PDE. */
3027 uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3028 Pde.u = pShwPage->Core.Key | fShwPdeFlags;
3029 Assert(!(Pde.u & EPT_E_LEAF));
3030 Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3031 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3032 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3033 Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3034 return rc;
3035 }
3036# endif /* PGM_WITH_LARGE_PAGES */
3037
3038 /*
3039 * Allocate & map the shadow page table.
3040 */
3041 PSHWPT pPt;
3042 PPGMPOOLPAGE pShwPage;
3043
3044 RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE_PG_MASK;
3045 rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_PT, PGMPOOLACCESS_DONTCARE,
3046 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
3047 if ( rc == VINF_SUCCESS
3048 || rc == VINF_PGM_CACHED_PAGE)
3049 { /* likely */ }
3050 else
3051 {
3052 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3053 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3054 }
3055
3056 pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3057 Assert(pPt);
3058 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
3059
3060 if (rc == VINF_SUCCESS)
3061 {
3062 /* Sync the page we've already translated through SLAT. */
3063 const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3064 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
3065 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3066
3067 /* Sync the rest of page table (expensive but might be cheaper than nested-guest VM-exits in hardware). */
3068 for (unsigned iPteCur = 0; iPteCur < RT_ELEMENTS(pPt->a); iPteCur++)
3069 {
3070 if (iPteCur != iPte)
3071 {
3072 PGMPTWALKGST GstWalkPt;
3073 PGMPTWALK WalkPt;
3074 GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
3075 GCPhysNestedPage |= (iPteCur << GUEST_PAGE_SHIFT);
3076 int const rc2 = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/,
3077 &WalkPt, &GstWalkPt);
3078 if (RT_SUCCESS(rc2))
3079 {
3080 PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPteCur], WalkPt.GCPhys, pShwPage, iPteCur, &GstWalkPt);
3081 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", GstWalkPt.u.Ept.Pte.u, pPt->a[iPteCur].u, iPteCur));
3082 }
3083 else
3084 {
3085 /*
3086 * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
3087 * Ensure the shadow tables entry is not-present.
3088 */
3089 /** @todo We currently don't configure these to cause EPT misconfigs but rather trap
3090 * them using EPT violations and walk the guest EPT tables to determine
3091 * whether they are EPT misconfigs VM-exits for the nested-hypervisor. We
3092 * could optimize this by using a specific combination of reserved bits
3093 * which we could immediately identify as EPT misconfigs of the
3094 * nested-hypervisor without having to walk its EPT tables. However, tracking
3095 * non-present entries might be tricky...
3096 */
3097 AssertMsg(!pPt->a[iPteCur].u, ("%RX64\n", pPt->a[iPteCur].u));
3098 }
3099 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3100 break;
3101 }
3102 }
3103 }
3104 else
3105 {
3106 Assert(rc == VINF_PGM_CACHED_PAGE);
3107# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
3108 /* Paranoia - Verify address of the page is what it should be. */
3109 PPGMPAGE pPage;
3110 rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
3111 AssertRC(rc);
3112 const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3113 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]) || !SHW_PTE_IS_P(pPt->a[iPte]),
3114 ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp Page=%RHp Shw=%RHp\n",
3115 GCPhysNestedPage, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
3116 Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [cache]\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
3117# endif
3118 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3119 }
3120
3121 /* Save the new PDE. */
3122 uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
3123 Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF));
3124 Assert(!(pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
3125 Pde.u = pShwPage->Core.Key | fShwPdeFlags;
3126 SHW_PDE_ATOMIC_SET2(*pPde, Pde);
3127 Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
3128
3129 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3130 return rc;
3131}
3132
3133#endif /* !IN_RING3 && VBOX_WITH_NESTED_HWVIRT_VMX_EPT && PGM_SHW_TYPE == PGM_TYPE_EPT*/
3134#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
3135
3136/**
3137 * Handle dirty bit tracking faults.
3138 *
3139 * @returns VBox status code.
3140 * @param pVCpu The cross context virtual CPU structure.
3141 * @param uErr Page fault error code.
3142 * @param pPdeSrc Guest page directory entry.
3143 * @param pPdeDst Shadow page directory entry.
3144 * @param GCPtrPage Guest context page address.
3145 */
3146static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
3147 RTGCPTR GCPtrPage)
3148{
3149 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3150 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3151 NOREF(uErr);
3152
3153 PGM_LOCK_ASSERT_OWNER(pVM);
3154
3155 /*
3156 * Handle big page.
3157 */
3158 if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
3159 {
3160 if ((pPdeDst->u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3161 {
3162 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3163 Assert(pPdeSrc->u & X86_PDE_RW);
3164
3165 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
3166 * fault again and take this path to only invalidate the entry (see below). */
3167 SHWPDE PdeDst = *pPdeDst;
3168 PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY;
3169 PdeDst.u |= X86_PDE_RW | X86_PDE_A;
3170 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3171 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
3172 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3173 }
3174
3175# ifdef IN_RING0
3176 /* Check for stale TLB entry; only applies to the SMP guest case. */
3177 if ( pVM->cCpus > 1
3178 && (pPdeDst->u & (X86_PDE_P | X86_PDE_RW | X86_PDE_A)) == (X86_PDE_P | X86_PDE_RW | X86_PDE_A))
3179 {
3180 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3181 if (pShwPage)
3182 {
3183 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3184 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3185 if (SHW_PTE_IS_P_RW(*pPteDst))
3186 {
3187 /* Stale TLB entry. */
3188 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3189 PGM_INVL_PG(pVCpu, GCPtrPage);
3190 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3191 }
3192 }
3193 }
3194# endif /* IN_RING0 */
3195 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3196 }
3197
3198 /*
3199 * Map the guest page table.
3200 */
3201 PGSTPT pPTSrc;
3202 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
3203 AssertRCReturn(rc, rc);
3204
3205 if (SHW_PDE_IS_P(*pPdeDst))
3206 {
3207 GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
3208 const GSTPTE PteSrc = *pPteSrc;
3209
3210 /*
3211 * Map shadow page table.
3212 */
3213 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
3214 if (pShwPage)
3215 {
3216 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3217 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
3218 if (SHW_PTE_IS_P(*pPteDst)) /** @todo Optimize accessed bit emulation? */
3219 {
3220 if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
3221 {
3222 PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
3223 SHWPTE PteDst = *pPteDst;
3224
3225 LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
3226 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
3227
3228 Assert(PteSrc.u & X86_PTE_RW);
3229
3230 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB
3231 * entry will not harm; write access will simply fault again and
3232 * take this path to only invalidate the entry.
3233 */
3234 if (RT_LIKELY(pPage))
3235 {
3236 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3237 {
3238 //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
3239 Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
3240 /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
3241 SHW_PTE_SET_RO(PteDst);
3242 }
3243 else
3244 {
3245 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
3246 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
3247 {
3248 rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
3249 AssertRC(rc);
3250 }
3251 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
3252 SHW_PTE_SET_RW(PteDst);
3253 else
3254 {
3255 /* Still applies to shared pages. */
3256 Assert(!PGM_PAGE_IS_ZERO(pPage));
3257 SHW_PTE_SET_RO(PteDst);
3258 }
3259 }
3260 }
3261 else
3262 SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
3263
3264 SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
3265 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
3266 PGM_INVL_PG(pVCpu, GCPtrPage);
3267 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3268 }
3269
3270# ifdef IN_RING0
3271 /* Check for stale TLB entry; only applies to the SMP guest case. */
3272 if ( pVM->cCpus > 1
3273 && SHW_PTE_IS_RW(*pPteDst)
3274 && SHW_PTE_IS_A(*pPteDst))
3275 {
3276 /* Stale TLB entry. */
3277 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
3278 PGM_INVL_PG(pVCpu, GCPtrPage);
3279 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
3280 }
3281# endif
3282 }
3283 }
3284 else
3285 AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
3286 }
3287
3288 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
3289}
3290
3291#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
3292
3293/**
3294 * Sync a shadow page table.
3295 *
3296 * The shadow page table is not present in the shadow PDE.
3297 *
3298 * Handles mapping conflicts.
3299 *
3300 * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
3301 * conflict), and Trap0eHandler.
3302 *
3303 * A precondition for this method is that the shadow PDE is not present. The
3304 * caller must take the PGM lock before checking this and continue to hold it
3305 * when calling this method.
3306 *
3307 * @returns VBox status code.
3308 * @param pVCpu The cross context virtual CPU structure.
3309 * @param iPDSrc Page directory index.
3310 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
3311 * Assume this is a temporary mapping.
3312 * @param GCPtrPage GC Pointer of the page that caused the fault
3313 */
3314static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
3315{
3316 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3317 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3318
3319#if 0 /* rarely useful; leave for debugging. */
3320 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
3321#endif
3322 LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
3323
3324 PGM_LOCK_ASSERT_OWNER(pVM);
3325
3326#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3327 || PGM_GST_TYPE == PGM_TYPE_PAE \
3328 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
3329 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3330 && PGM_SHW_TYPE != PGM_TYPE_NONE
3331 int rc = VINF_SUCCESS;
3332
3333 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3334
3335 /*
3336 * Some input validation first.
3337 */
3338 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
3339
3340 /*
3341 * Get the relevant shadow PDE entry.
3342 */
3343# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3344 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
3345 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3346
3347 /* Fetch the pgm pool shadow descriptor. */
3348 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3349 Assert(pShwPde);
3350
3351# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3352 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3353 PPGMPOOLPAGE pShwPde = NULL;
3354 PX86PDPAE pPDDst;
3355 PSHWPDE pPdeDst;
3356
3357 /* Fetch the pgm pool shadow descriptor. */
3358 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3359 AssertRCSuccessReturn(rc, rc);
3360 Assert(pShwPde);
3361
3362 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3363 pPdeDst = &pPDDst->a[iPDDst];
3364
3365# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3366 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3367 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3368 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3369 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
3370 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3371 AssertRCSuccessReturn(rc, rc);
3372 Assert(pPDDst);
3373 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3374
3375# endif
3376 SHWPDE PdeDst = *pPdeDst;
3377
3378# if PGM_GST_TYPE == PGM_TYPE_AMD64
3379 /* Fetch the pgm pool shadow descriptor. */
3380 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3381 Assert(pShwPde);
3382# endif
3383
3384 Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/
3385
3386 /*
3387 * Sync the page directory entry.
3388 */
3389 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3390 const bool fPageTable = !(PdeSrc.u & X86_PDE_PS) || !GST_IS_PSE_ACTIVE(pVCpu);
3391 if ( (PdeSrc.u & X86_PDE_P)
3392 && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
3393 {
3394 /*
3395 * Allocate & map the page table.
3396 */
3397 PSHWPT pPTDst;
3398 PPGMPOOLPAGE pShwPage;
3399 RTGCPHYS GCPhys;
3400 if (fPageTable)
3401 {
3402 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
3403# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3404 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3405 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
3406# endif
3407 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
3408 pShwPde->idx, iPDDst, false /*fLockPage*/,
3409 &pShwPage);
3410 }
3411 else
3412 {
3413 PGMPOOLACCESS enmAccess;
3414# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
3415 const bool fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu);
3416# else
3417 const bool fNoExecute = false;
3418# endif
3419
3420 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3421# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3422 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
3423 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
3424# endif
3425 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
3426 if (PdeSrc.u & X86_PDE_US)
3427 {
3428 if (PdeSrc.u & X86_PDE_RW)
3429 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
3430 else
3431 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
3432 }
3433 else
3434 {
3435 if (PdeSrc.u & X86_PDE_RW)
3436 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
3437 else
3438 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
3439 }
3440 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
3441 pShwPde->idx, iPDDst, false /*fLockPage*/,
3442 &pShwPage);
3443 }
3444 if (rc == VINF_SUCCESS)
3445 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3446 else if (rc == VINF_PGM_CACHED_PAGE)
3447 {
3448 /*
3449 * The PT was cached, just hook it up.
3450 */
3451 if (fPageTable)
3452 PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3453 else
3454 {
3455 PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3456 /* (see explanation and assumptions further down.) */
3457 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
3458 {
3459 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3460 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
3461 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3462 }
3463 }
3464 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3465 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3466 return VINF_SUCCESS;
3467 }
3468 else
3469 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3470 /** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
3471 * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
3472 * irrelevant at this point. */
3473 PdeDst.u &= X86_PDE_AVL_MASK;
3474 PdeDst.u |= pShwPage->Core.Key;
3475
3476 /*
3477 * Page directory has been accessed (this is a fault situation, remember).
3478 */
3479 /** @todo
3480 * Well, when the caller is PrefetchPage or InvalidatePage is isn't a
3481 * fault situation. What's more, the Trap0eHandler has already set the
3482 * accessed bit. So, it's actually just VerifyAccessSyncPage which
3483 * might need setting the accessed flag.
3484 *
3485 * The best idea is to leave this change to the caller and add an
3486 * assertion that it's set already. */
3487 pPDSrc->a[iPDSrc].u |= X86_PDE_A;
3488 if (fPageTable)
3489 {
3490 /*
3491 * Page table - 4KB.
3492 *
3493 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
3494 */
3495 Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
3496 GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u));
3497 PGSTPT pPTSrc;
3498 rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
3499 if (RT_SUCCESS(rc))
3500 {
3501 /*
3502 * Start by syncing the page directory entry so CSAM's TLB trick works.
3503 */
3504 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
3505 | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3506 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3507 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3508
3509 /*
3510 * Directory/page user or supervisor privilege: (same goes for read/write)
3511 *
3512 * Directory Page Combined
3513 * U/S U/S U/S
3514 * 0 0 0
3515 * 0 1 0
3516 * 1 0 0
3517 * 1 1 1
3518 *
3519 * Simple AND operation. Table listed for completeness.
3520 *
3521 */
3522 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K));
3523# ifdef PGM_SYNC_N_PAGES
3524 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3525 unsigned iPTDst = iPTBase;
3526 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
3527 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
3528 iPTDst = 0;
3529 else
3530 iPTDst -= PGM_SYNC_NR_PAGES / 2;
3531# else /* !PGM_SYNC_N_PAGES */
3532 unsigned iPTDst = 0;
3533 const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
3534# endif /* !PGM_SYNC_N_PAGES */
3535 RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
3536 | ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT);
3537# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3538 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3539 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
3540# else
3541 const unsigned offPTSrc = 0;
3542# endif
3543 for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE)
3544 {
3545 const unsigned iPTSrc = iPTDst + offPTSrc;
3546 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
3547 if (PteSrc.u & X86_PTE_P)
3548 {
3549 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
3550 Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
3551 GCPtrCur,
3552 PteSrc.u & X86_PTE_P,
3553 !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
3554 !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
3555 (uint64_t)PteSrc.u,
3556 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
3557 (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
3558 }
3559 /* else: the page table was cleared by the pool */
3560 } /* for PTEs */
3561 }
3562 }
3563 else
3564 {
3565 /*
3566 * Big page - 2/4MB.
3567 *
3568 * We'll walk the ram range list in parallel and optimize lookups.
3569 * We will only sync one shadow page table at a time.
3570 */
3571 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M));
3572
3573 /**
3574 * @todo It might be more efficient to sync only a part of the 4MB
3575 * page (similar to what we do for 4KB PDs).
3576 */
3577
3578 /*
3579 * Start by syncing the page directory entry.
3580 */
3581 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
3582 | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3583
3584 /*
3585 * If the page is not flagged as dirty and is writable, then make it read-only
3586 * at PD level, so we can set the dirty bit when the page is modified.
3587 *
3588 * ASSUMES that page access handlers are implemented on page table entry level.
3589 * Thus we will first catch the dirty access and set PDE.D and restart. If
3590 * there is an access handler, we'll trap again and let it work on the problem.
3591 */
3592 /** @todo move the above stuff to a section in the PGM documentation. */
3593 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
3594 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
3595 {
3596 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
3597 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
3598 PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
3599 }
3600 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3601 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3602
3603 /*
3604 * Fill the shadow page table.
3605 */
3606 /* Get address and flags from the source PDE. */
3607 SHWPTE PteDstBase;
3608 SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
3609
3610 /* Loop thru the entries in the shadow PT. */
3611 const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
3612 Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
3613 GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr,
3614 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
3615 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3616 unsigned iPTDst = 0;
3617 while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3618 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3619 {
3620 if (pRam && GCPhys >= pRam->GCPhys)
3621 {
3622# ifndef PGM_WITH_A20
3623 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
3624# endif
3625 do
3626 {
3627 /* Make shadow PTE. */
3628# ifdef PGM_WITH_A20
3629 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
3630# else
3631 PPGMPAGE pPage = &pRam->aPages[iHCPage];
3632# endif
3633 SHWPTE PteDst;
3634
3635# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3636 /* Try to make the page writable if necessary. */
3637 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3638 && ( PGM_PAGE_IS_ZERO(pPage)
3639 || ( SHW_PTE_IS_RW(PteDstBase)
3640 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3641# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3642 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3643# endif
3644# ifdef VBOX_WITH_PAGE_SHARING
3645 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3646# endif
3647 && !PGM_PAGE_IS_BALLOONED(pPage))
3648 )
3649 )
3650 {
3651 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3652 AssertRCReturn(rc, rc);
3653 if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3654 break;
3655 }
3656# endif
3657
3658 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
3659 PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, SHW_PTE_GET_U(PteDstBase), &PteDst);
3660 else if (PGM_PAGE_IS_BALLOONED(pPage))
3661 SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3662 else
3663 SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase));
3664
3665 /* Only map writable pages writable. */
3666 if ( SHW_PTE_IS_P_RW(PteDst)
3667 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3668 {
3669 /* Still applies to shared pages. */
3670 Assert(!PGM_PAGE_IS_ZERO(pPage));
3671 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3672 Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
3673 }
3674
3675 if (SHW_PTE_IS_P(PteDst))
3676 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3677
3678 /* commit it (not atomic, new table) */
3679 pPTDst->a[iPTDst] = PteDst;
3680 Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3681 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3682 SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3683
3684 /* advance */
3685 GCPhys += GUEST_PAGE_SIZE;
3686 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3687# ifndef PGM_WITH_A20
3688 iHCPage++;
3689# endif
3690 iPTDst++;
3691 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3692 && GCPhys <= pRam->GCPhysLast);
3693
3694 /* Advance ram range list. */
3695 while (pRam && GCPhys > pRam->GCPhysLast)
3696 pRam = pRam->CTX_SUFF(pNext);
3697 }
3698 else if (pRam)
3699 {
3700 Log(("Invalid pages at %RGp\n", GCPhys));
3701 do
3702 {
3703 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3704 GCPhys += GUEST_PAGE_SIZE;
3705 iPTDst++;
3706 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3707 && GCPhys < pRam->GCPhys);
3708 PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3709 }
3710 else
3711 {
3712 Log(("Invalid pages at %RGp (2)\n", GCPhys));
3713 for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3714 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3715 }
3716 } /* while more PTEs */
3717 } /* 4KB / 4MB */
3718 }
3719 else
3720 AssertRelease(!SHW_PDE_IS_P(PdeDst));
3721
3722 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3723 if (RT_FAILURE(rc))
3724 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3725 return rc;
3726
3727#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
3728 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
3729 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
3730 && PGM_SHW_TYPE != PGM_TYPE_NONE
3731 NOREF(iPDSrc); NOREF(pPDSrc);
3732
3733 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3734
3735 /*
3736 * Validate input a little bit.
3737 */
3738 int rc = VINF_SUCCESS;
3739# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3740 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3741 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3742
3743 /* Fetch the pgm pool shadow descriptor. */
3744 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3745 Assert(pShwPde);
3746
3747# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3748 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3749 PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3750 PX86PDPAE pPDDst;
3751 PSHWPDE pPdeDst;
3752
3753 /* Fetch the pgm pool shadow descriptor. */
3754 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3755 AssertRCSuccessReturn(rc, rc);
3756 Assert(pShwPde);
3757
3758 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3759 pPdeDst = &pPDDst->a[iPDDst];
3760
3761# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3762 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3763 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3764 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3765 PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3766 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3767 AssertRCSuccessReturn(rc, rc);
3768 Assert(pPDDst);
3769 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3770
3771 /* Fetch the pgm pool shadow descriptor. */
3772 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3773 Assert(pShwPde);
3774
3775# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3776 const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3777 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3778 PEPTPD pPDDst;
3779 PEPTPDPT pPdptDst;
3780
3781 rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3782 if (rc != VINF_SUCCESS)
3783 {
3784 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3785 AssertRC(rc);
3786 return rc;
3787 }
3788 Assert(pPDDst);
3789 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3790
3791 /* Fetch the pgm pool shadow descriptor. */
3792 /** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
3793 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3794 Assert(pShwPde);
3795# endif
3796 SHWPDE PdeDst = *pPdeDst;
3797
3798 Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3799
3800# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3801 if (BTH_IS_NP_ACTIVE(pVM))
3802 {
3803 Assert(!VM_IS_NEM_ENABLED(pVM));
3804
3805 /* Check if we allocated a big page before for this 2 MB range. */
3806 PPGMPAGE pPage;
3807 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3808 if (RT_SUCCESS(rc))
3809 {
3810 RTHCPHYS HCPhys = NIL_RTHCPHYS;
3811 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3812 {
3813 if (PGM_A20_IS_ENABLED(pVCpu))
3814 {
3815 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3816 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3817 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3818 }
3819 else
3820 {
3821 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3822 pVM->pgm.s.cLargePagesDisabled++;
3823 }
3824 }
3825 else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3826 && PGM_A20_IS_ENABLED(pVCpu))
3827 {
3828 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3829 rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3830 if (RT_SUCCESS(rc))
3831 {
3832 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3833 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3834 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3835 }
3836 }
3837 else if ( PGMIsUsingLargePages(pVM)
3838 && PGM_A20_IS_ENABLED(pVCpu))
3839 {
3840 rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3841 if (RT_SUCCESS(rc))
3842 {
3843 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3844 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3845 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3846 }
3847 else
3848 LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3849 }
3850
3851 if (HCPhys != NIL_RTHCPHYS)
3852 {
3853# if PGM_SHW_TYPE == PGM_TYPE_EPT
3854 PdeDst.u = HCPhys | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_LEAF | EPT_E_IGNORE_PAT | EPT_E_MEMTYPE_WB
3855 | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
3856# else
3857 PdeDst.u = HCPhys | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PS
3858 | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */;
3859# endif
3860 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3861
3862 Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3863 /* Add a reference to the first page only. */
3864 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3865
3866 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3867 return VINF_SUCCESS;
3868 }
3869 }
3870 }
3871# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
3872
3873 /*
3874 * Allocate & map the page table.
3875 */
3876 PSHWPT pPTDst;
3877 PPGMPOOLPAGE pShwPage;
3878 RTGCPHYS GCPhys;
3879
3880 /* Virtual address = physical address */
3881 GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3882 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3883 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/,
3884 &pShwPage);
3885 if ( rc == VINF_SUCCESS
3886 || rc == VINF_PGM_CACHED_PAGE)
3887 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3888 else
3889 {
3890 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3891 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3892 }
3893
3894 if (rc == VINF_SUCCESS)
3895 {
3896 /* New page table; fully set it up. */
3897 Assert(pPTDst);
3898
3899 /* Mask away the page offset. */
3900 GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
3901
3902 for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3903 {
3904 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3905 | (iPTDst << GUEST_PAGE_SHIFT));
3906
3907 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3908 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3909 GCPtrCurPage,
3910 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3911 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3912
3913 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3914 break;
3915 }
3916 }
3917 else
3918 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3919
3920 /* Save the new PDE. */
3921# if PGM_SHW_TYPE == PGM_TYPE_EPT
3922 PdeDst.u = pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE
3923 | (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
3924# else
3925 PdeDst.u = pShwPage->Core.Key | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A
3926 | (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */);
3927# endif
3928 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3929
3930 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3931 if (RT_FAILURE(rc))
3932 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3933 return rc;
3934
3935#else
3936 NOREF(iPDSrc); NOREF(pPDSrc);
3937 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3938 return VERR_PGM_NOT_USED_IN_MODE;
3939#endif
3940}
3941
3942
3943
3944/**
3945 * Prefetch a page/set of pages.
3946 *
3947 * Typically used to sync commonly used pages before entering raw mode
3948 * after a CR3 reload.
3949 *
3950 * @returns VBox status code.
3951 * @param pVCpu The cross context virtual CPU structure.
3952 * @param GCPtrPage Page to invalidate.
3953 */
3954PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
3955{
3956#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3957 || PGM_GST_TYPE == PGM_TYPE_REAL \
3958 || PGM_GST_TYPE == PGM_TYPE_PROT \
3959 || PGM_GST_TYPE == PGM_TYPE_PAE \
3960 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3961 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3962 && PGM_SHW_TYPE != PGM_TYPE_NONE
3963 /*
3964 * Check that all Guest levels thru the PDE are present, getting the
3965 * PD and PDE in the processes.
3966 */
3967 int rc = VINF_SUCCESS;
3968# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3969# if PGM_GST_TYPE == PGM_TYPE_32BIT
3970 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3971 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3972# elif PGM_GST_TYPE == PGM_TYPE_PAE
3973 unsigned iPDSrc;
3974 X86PDPE PdpeSrc;
3975 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3976 if (!pPDSrc)
3977 return VINF_SUCCESS; /* not present */
3978# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3979 unsigned iPDSrc;
3980 PX86PML4E pPml4eSrc;
3981 X86PDPE PdpeSrc;
3982 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3983 if (!pPDSrc)
3984 return VINF_SUCCESS; /* not present */
3985# endif
3986 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3987# else
3988 PGSTPD pPDSrc = NULL;
3989 const unsigned iPDSrc = 0;
3990 GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
3991# endif
3992
3993 if ((PdeSrc.u & (X86_PDE_P | X86_PDE_A)) == (X86_PDE_P | X86_PDE_A))
3994 {
3995 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3996 PGM_LOCK_VOID(pVM);
3997
3998# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3999 const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
4000# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4001 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4002 PX86PDPAE pPDDst;
4003 X86PDEPAE PdeDst;
4004# if PGM_GST_TYPE != PGM_TYPE_PAE
4005 X86PDPE PdpeSrc;
4006
4007 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
4008 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
4009# endif
4010 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
4011 if (rc != VINF_SUCCESS)
4012 {
4013 PGM_UNLOCK(pVM);
4014 AssertRC(rc);
4015 return rc;
4016 }
4017 Assert(pPDDst);
4018 PdeDst = pPDDst->a[iPDDst];
4019
4020# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4021 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4022 PX86PDPAE pPDDst;
4023 X86PDEPAE PdeDst;
4024
4025# if PGM_GST_TYPE == PGM_TYPE_PROT
4026 /* AMD-V nested paging */
4027 X86PML4E Pml4eSrc;
4028 X86PDPE PdpeSrc;
4029 PX86PML4E pPml4eSrc = &Pml4eSrc;
4030
4031 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
4032 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
4033 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
4034# endif
4035
4036 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
4037 if (rc != VINF_SUCCESS)
4038 {
4039 PGM_UNLOCK(pVM);
4040 AssertRC(rc);
4041 return rc;
4042 }
4043 Assert(pPDDst);
4044 PdeDst = pPDDst->a[iPDDst];
4045# endif
4046 if (!(PdeDst.u & X86_PDE_P))
4047 {
4048 /** @todo r=bird: This guy will set the A bit on the PDE,
4049 * probably harmless. */
4050 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
4051 }
4052 else
4053 {
4054 /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
4055 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
4056 * makes no sense to prefetch more than one page.
4057 */
4058 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
4059 if (RT_SUCCESS(rc))
4060 rc = VINF_SUCCESS;
4061 }
4062 PGM_UNLOCK(pVM);
4063 }
4064 return rc;
4065
4066#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4067 NOREF(pVCpu); NOREF(GCPtrPage);
4068 return VINF_SUCCESS; /* ignore */
4069#else
4070 AssertCompile(0);
4071#endif
4072}
4073
4074
4075
4076
4077/**
4078 * Syncs a page during a PGMVerifyAccess() call.
4079 *
4080 * @returns VBox status code (informational included).
4081 * @param pVCpu The cross context virtual CPU structure.
4082 * @param GCPtrPage The address of the page to sync.
4083 * @param fPage The effective guest page flags.
4084 * @param uErr The trap error code.
4085 * @remarks This will normally never be called on invalid guest page
4086 * translation entries.
4087 */
4088PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
4089{
4090 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4091
4092 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
4093 RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
4094
4095 Assert(!pVM->pgm.s.fNestedPaging);
4096#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
4097 || PGM_GST_TYPE == PGM_TYPE_REAL \
4098 || PGM_GST_TYPE == PGM_TYPE_PROT \
4099 || PGM_GST_TYPE == PGM_TYPE_PAE \
4100 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
4101 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
4102 && PGM_SHW_TYPE != PGM_TYPE_NONE
4103
4104 /*
4105 * Get guest PD and index.
4106 */
4107 /** @todo Performance: We've done all this a jiffy ago in the
4108 * PGMGstGetPage call. */
4109# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4110# if PGM_GST_TYPE == PGM_TYPE_32BIT
4111 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
4112 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4113
4114# elif PGM_GST_TYPE == PGM_TYPE_PAE
4115 unsigned iPDSrc = 0;
4116 X86PDPE PdpeSrc;
4117 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
4118 if (RT_UNLIKELY(!pPDSrc))
4119 {
4120 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
4121 return VINF_EM_RAW_GUEST_TRAP;
4122 }
4123
4124# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4125 unsigned iPDSrc = 0; /* shut up gcc */
4126 PX86PML4E pPml4eSrc = NULL; /* ditto */
4127 X86PDPE PdpeSrc;
4128 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
4129 if (RT_UNLIKELY(!pPDSrc))
4130 {
4131 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
4132 return VINF_EM_RAW_GUEST_TRAP;
4133 }
4134# endif
4135
4136# else /* !PGM_WITH_PAGING */
4137 PGSTPD pPDSrc = NULL;
4138 const unsigned iPDSrc = 0;
4139# endif /* !PGM_WITH_PAGING */
4140 int rc = VINF_SUCCESS;
4141
4142 PGM_LOCK_VOID(pVM);
4143
4144 /*
4145 * First check if the shadow pd is present.
4146 */
4147# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4148 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
4149
4150# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4151 PX86PDEPAE pPdeDst;
4152 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4153 PX86PDPAE pPDDst;
4154# if PGM_GST_TYPE != PGM_TYPE_PAE
4155 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
4156 X86PDPE PdpeSrc;
4157 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
4158# endif
4159 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
4160 if (rc != VINF_SUCCESS)
4161 {
4162 PGM_UNLOCK(pVM);
4163 AssertRC(rc);
4164 return rc;
4165 }
4166 Assert(pPDDst);
4167 pPdeDst = &pPDDst->a[iPDDst];
4168
4169# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4170 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
4171 PX86PDPAE pPDDst;
4172 PX86PDEPAE pPdeDst;
4173
4174# if PGM_GST_TYPE == PGM_TYPE_PROT
4175 /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
4176 X86PML4E Pml4eSrc;
4177 X86PDPE PdpeSrc;
4178 PX86PML4E pPml4eSrc = &Pml4eSrc;
4179 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
4180 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
4181# endif
4182
4183 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
4184 if (rc != VINF_SUCCESS)
4185 {
4186 PGM_UNLOCK(pVM);
4187 AssertRC(rc);
4188 return rc;
4189 }
4190 Assert(pPDDst);
4191 pPdeDst = &pPDDst->a[iPDDst];
4192# endif
4193
4194 if (!(pPdeDst->u & X86_PDE_P))
4195 {
4196 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
4197 if (rc != VINF_SUCCESS)
4198 {
4199 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
4200 PGM_UNLOCK(pVM);
4201 AssertRC(rc);
4202 return rc;
4203 }
4204 }
4205
4206# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4207 /* Check for dirty bit fault */
4208 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
4209 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
4210 Log(("PGMVerifyAccess: success (dirty)\n"));
4211 else
4212# endif
4213 {
4214# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4215 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
4216# else
4217 GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
4218# endif
4219
4220 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
4221 if (uErr & X86_TRAP_PF_US)
4222 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
4223 else /* supervisor */
4224 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
4225
4226 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
4227 if (RT_SUCCESS(rc))
4228 {
4229 /* Page was successfully synced */
4230 Log2(("PGMVerifyAccess: success (sync)\n"));
4231 rc = VINF_SUCCESS;
4232 }
4233 else
4234 {
4235 Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
4236 rc = VINF_EM_RAW_GUEST_TRAP;
4237 }
4238 }
4239 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
4240 PGM_UNLOCK(pVM);
4241 return rc;
4242
4243#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
4244
4245 AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
4246 return VERR_PGM_NOT_USED_IN_MODE;
4247#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
4248}
4249
4250
4251/**
4252 * Syncs the paging hierarchy starting at CR3.
4253 *
4254 * @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
4255 * informational status codes.
4256 * @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
4257 * the VMM into guest context.
4258 * @param pVCpu The cross context virtual CPU structure.
4259 * @param cr0 Guest context CR0 register.
4260 * @param cr3 Guest context CR3 register. Not subjected to the A20
4261 * mask.
4262 * @param cr4 Guest context CR4 register.
4263 * @param fGlobal Including global page directories or not
4264 */
4265PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
4266{
4267 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4268 NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
4269
4270 LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
4271
4272#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
4273# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4274 PGM_LOCK_VOID(pVM);
4275 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4276 if (pPool->cDirtyPages)
4277 pgmPoolResetDirtyPages(pVM);
4278 PGM_UNLOCK(pVM);
4279# endif
4280#endif /* !NESTED && !EPT */
4281
4282#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4283 /*
4284 * Nested / EPT / None - No work.
4285 */
4286 return VINF_SUCCESS;
4287
4288#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4289 /*
4290 * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
4291 * out the shadow parts when the guest modifies its tables.
4292 */
4293 return VINF_SUCCESS;
4294
4295#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4296
4297 return VINF_SUCCESS;
4298#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
4299}
4300
4301
4302
4303
4304#ifdef VBOX_STRICT
4305
4306/**
4307 * Checks that the shadow page table is in sync with the guest one.
4308 *
4309 * @returns The number of errors.
4310 * @param pVCpu The cross context virtual CPU structure.
4311 * @param cr3 Guest context CR3 register.
4312 * @param cr4 Guest context CR4 register.
4313 * @param GCPtr Where to start. Defaults to 0.
4314 * @param cb How much to check. Defaults to everything.
4315 */
4316PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
4317{
4318 NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
4319#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
4320 return 0;
4321#else
4322 unsigned cErrors = 0;
4323 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4324 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
4325
4326# if PGM_GST_TYPE == PGM_TYPE_PAE
4327 /** @todo currently broken; crashes below somewhere */
4328 AssertFailed();
4329# endif
4330
4331# if PGM_GST_TYPE == PGM_TYPE_32BIT \
4332 || PGM_GST_TYPE == PGM_TYPE_PAE \
4333 || PGM_GST_TYPE == PGM_TYPE_AMD64
4334
4335 bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
4336 PPGMCPU pPGM = &pVCpu->pgm.s;
4337 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
4338 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
4339# ifndef IN_RING0
4340 RTHCPHYS HCPhys; /* general usage. */
4341# endif
4342 int rc;
4343
4344 /*
4345 * Check that the Guest CR3 and all its mappings are correct.
4346 */
4347 AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
4348 ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
4349 false);
4350# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
4351# if 0
4352# if PGM_GST_TYPE == PGM_TYPE_32BIT
4353 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
4354# else
4355 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
4356# endif
4357 AssertRCReturn(rc, 1);
4358 HCPhys = NIL_RTHCPHYS;
4359 rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
4360 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
4361# endif
4362# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
4363 pgmGstGet32bitPDPtr(pVCpu);
4364 RTGCPHYS GCPhys;
4365 rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
4366 AssertRCReturn(rc, 1);
4367 AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
4368# endif
4369# endif /* !IN_RING0 */
4370
4371 /*
4372 * Get and check the Shadow CR3.
4373 */
4374# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4375 unsigned cPDEs = X86_PG_ENTRIES;
4376 unsigned cIncrement = X86_PG_ENTRIES * GUEST_PAGE_SIZE;
4377# elif PGM_SHW_TYPE == PGM_TYPE_PAE
4378# if PGM_GST_TYPE == PGM_TYPE_32BIT
4379 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
4380# else
4381 unsigned cPDEs = X86_PG_PAE_ENTRIES;
4382# endif
4383 unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4384# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
4385 unsigned cPDEs = X86_PG_PAE_ENTRIES;
4386 unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
4387# endif
4388 if (cb != ~(RTGCPTR)0)
4389 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
4390
4391/** @todo call the other two PGMAssert*() functions. */
4392
4393# if PGM_GST_TYPE == PGM_TYPE_AMD64
4394 unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4395
4396 for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
4397 {
4398 PPGMPOOLPAGE pShwPdpt = NULL;
4399 PX86PML4E pPml4eSrc;
4400 PX86PML4E pPml4eDst;
4401 RTGCPHYS GCPhysPdptSrc;
4402
4403 pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
4404 pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
4405
4406 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
4407 if (!(pPml4eDst->u & X86_PML4E_P))
4408 {
4409 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4410 continue;
4411 }
4412
4413 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
4414 GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
4415
4416 if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P))
4417 {
4418 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4419 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4420 cErrors++;
4421 continue;
4422 }
4423
4424 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
4425 {
4426 AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
4427 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4428 cErrors++;
4429 continue;
4430 }
4431
4432 if ( (pPml4eDst->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX))
4433 != (pPml4eSrc->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX)))
4434 {
4435 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4436 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4437 cErrors++;
4438 continue;
4439 }
4440# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4441 {
4442# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4443
4444# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
4445 /*
4446 * Check the PDPTEs too.
4447 */
4448 unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
4449
4450 for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
4451 {
4452 unsigned iPDSrc = 0; /* initialized to shut up gcc */
4453 PPGMPOOLPAGE pShwPde = NULL;
4454 PX86PDPE pPdpeDst;
4455 RTGCPHYS GCPhysPdeSrc;
4456 X86PDPE PdpeSrc;
4457 PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
4458# if PGM_GST_TYPE == PGM_TYPE_PAE
4459 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
4460 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
4461# else
4462 PX86PML4E pPml4eSrcIgn;
4463 PX86PDPT pPdptDst;
4464 PX86PDPAE pPDDst;
4465 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
4466
4467 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
4468 if (rc != VINF_SUCCESS)
4469 {
4470 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
4471 GCPtr += 512 * _2M;
4472 continue; /* next PDPTE */
4473 }
4474 Assert(pPDDst);
4475# endif
4476 Assert(iPDSrc == 0);
4477
4478 pPdpeDst = &pPdptDst->a[iPdpt];
4479
4480 if (!(pPdpeDst->u & X86_PDPE_P))
4481 {
4482 GCPtr += 512 * _2M;
4483 continue; /* next PDPTE */
4484 }
4485
4486 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
4487 GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
4488
4489 if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P))
4490 {
4491 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4492 GCPtr += 512 * _2M;
4493 cErrors++;
4494 continue;
4495 }
4496
4497 if (GCPhysPdeSrc != pShwPde->GCPhys)
4498 {
4499# if PGM_GST_TYPE == PGM_TYPE_AMD64
4500 AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4501# else
4502 AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4503# endif
4504 GCPtr += 512 * _2M;
4505 cErrors++;
4506 continue;
4507 }
4508
4509# if PGM_GST_TYPE == PGM_TYPE_AMD64
4510 if ( (pPdpeDst->u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX))
4511 != (PdpeSrc.u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX)))
4512 {
4513 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4514 GCPtr += 512 * _2M;
4515 cErrors++;
4516 continue;
4517 }
4518# endif
4519
4520# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4521 {
4522# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4523# if PGM_GST_TYPE == PGM_TYPE_32BIT
4524 GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4525# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4526 PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
4527# endif
4528# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
4529 /*
4530 * Iterate the shadow page directory.
4531 */
4532 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
4533 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
4534
4535 for (;
4536 iPDDst < cPDEs;
4537 iPDDst++, GCPtr += cIncrement)
4538 {
4539# if PGM_SHW_TYPE == PGM_TYPE_PAE
4540 const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4541# else
4542 const SHWPDE PdeDst = pPDDst->a[iPDDst];
4543# endif
4544 if ( (PdeDst.u & X86_PDE_P)
4545 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) )
4546 {
4547 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4548 PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4549 if (!pPoolPage)
4550 {
4551 AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4552 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4553 cErrors++;
4554 continue;
4555 }
4556 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4557
4558 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
4559 {
4560 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4561 GCPtr, (uint64_t)PdeDst.u));
4562 cErrors++;
4563 }
4564
4565 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
4566 {
4567 AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4568 GCPtr, (uint64_t)PdeDst.u));
4569 cErrors++;
4570 }
4571
4572 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4573 if (!(PdeSrc.u & X86_PDE_P))
4574 {
4575 AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4576 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4577 cErrors++;
4578 continue;
4579 }
4580
4581 if ( !(PdeSrc.u & X86_PDE_PS)
4582 || !fBigPagesSupported)
4583 {
4584 GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4585# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4586 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
4587# endif
4588 }
4589 else
4590 {
4591# if PGM_GST_TYPE == PGM_TYPE_32BIT
4592 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4593 {
4594 AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4595 GCPtr, (uint64_t)PdeSrc.u));
4596 cErrors++;
4597 continue;
4598 }
4599# endif
4600 GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4601# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4602 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4603# endif
4604 }
4605
4606 if ( pPoolPage->enmKind
4607 != (!(PdeSrc.u & X86_PDE_PS) || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4608 {
4609 AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4610 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4611 cErrors++;
4612 }
4613
4614 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4615 if (!pPhysPage)
4616 {
4617 AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4618 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4619 cErrors++;
4620 continue;
4621 }
4622
4623 if (GCPhysGst != pPoolPage->GCPhys)
4624 {
4625 AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4626 GCPhysGst, pPoolPage->GCPhys, GCPtr));
4627 cErrors++;
4628 continue;
4629 }
4630
4631 if ( !(PdeSrc.u & X86_PDE_PS)
4632 || !fBigPagesSupported)
4633 {
4634 /*
4635 * Page Table.
4636 */
4637 const GSTPT *pPTSrc;
4638 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)),
4639 &pPTSrc);
4640 if (RT_FAILURE(rc))
4641 {
4642 AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4643 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4644 cErrors++;
4645 continue;
4646 }
4647 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
4648 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
4649 {
4650 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4651 // (This problem will go away when/if we shadow multiple CR3s.)
4652 AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4653 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4654 cErrors++;
4655 continue;
4656 }
4657 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4658 {
4659 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4660 GCPtr, (uint64_t)PdeDst.u));
4661 cErrors++;
4662 continue;
4663 }
4664
4665 /* iterate the page table. */
4666# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4667 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4668 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4669# else
4670 const unsigned offPTSrc = 0;
4671# endif
4672 for (unsigned iPT = 0, off = 0;
4673 iPT < RT_ELEMENTS(pPTDst->a);
4674 iPT++, off += GUEST_PAGE_SIZE)
4675 {
4676 const SHWPTE PteDst = pPTDst->a[iPT];
4677
4678 /* skip not-present and dirty tracked entries. */
4679 if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4680 continue;
4681 Assert(SHW_PTE_IS_P(PteDst));
4682
4683 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4684 if (!(PteSrc.u & X86_PTE_P))
4685 {
4686# ifdef IN_RING3
4687 PGMAssertHandlerAndFlagsInSync(pVM);
4688 DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE
4689 | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3,
4690 0, 0, UINT64_MAX, 99, NULL);
4691# endif
4692 AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4693 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4694 (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4695 cErrors++;
4696 continue;
4697 }
4698
4699 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
4700# if 1 /** @todo sync accessed bit properly... */
4701 fIgnoreFlags |= X86_PTE_A;
4702# endif
4703
4704 /* match the physical addresses */
4705 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4706 GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4707
4708# ifdef IN_RING3
4709 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4710 if (RT_FAILURE(rc))
4711 {
4712# if 0
4713 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4714 {
4715 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4716 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4717 cErrors++;
4718 continue;
4719 }
4720# endif
4721 }
4722 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4723 {
4724 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4725 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4726 cErrors++;
4727 continue;
4728 }
4729# endif
4730
4731 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4732 if (!pPhysPage)
4733 {
4734# if 0
4735 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4736 {
4737 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4738 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4739 cErrors++;
4740 continue;
4741 }
4742# endif
4743 if (SHW_PTE_IS_RW(PteDst))
4744 {
4745 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4746 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4747 cErrors++;
4748 }
4749 fIgnoreFlags |= X86_PTE_RW;
4750 }
4751 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4752 {
4753 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4754 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4755 cErrors++;
4756 continue;
4757 }
4758
4759 /* flags */
4760 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
4761 {
4762 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4763 {
4764 if (SHW_PTE_IS_RW(PteDst))
4765 {
4766 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4767 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4768 cErrors++;
4769 continue;
4770 }
4771 fIgnoreFlags |= X86_PTE_RW;
4772 }
4773 else
4774 {
4775 if ( SHW_PTE_IS_P(PteDst)
4776# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
4777 && !PGM_PAGE_IS_MMIO(pPhysPage)
4778# endif
4779 )
4780 {
4781 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4782 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4783 cErrors++;
4784 continue;
4785 }
4786 fIgnoreFlags |= X86_PTE_P;
4787 }
4788 }
4789 else
4790 {
4791 if ((PteSrc.u & (X86_PTE_RW | X86_PTE_D)) == X86_PTE_RW)
4792 {
4793 if (SHW_PTE_IS_RW(PteDst))
4794 {
4795 AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4796 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4797 cErrors++;
4798 continue;
4799 }
4800 if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4801 {
4802 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4803 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4804 cErrors++;
4805 continue;
4806 }
4807 if (SHW_PTE_IS_D(PteDst))
4808 {
4809 AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4810 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4811 cErrors++;
4812 }
4813# if 0 /** @todo sync access bit properly... */
4814 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4815 {
4816 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4817 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4818 cErrors++;
4819 }
4820 fIgnoreFlags |= X86_PTE_RW;
4821# else
4822 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4823# endif
4824 }
4825 else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4826 {
4827 /* access bit emulation (not implemented). */
4828 if ((PteSrc.u & X86_PTE_A) || SHW_PTE_IS_P(PteDst))
4829 {
4830 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4831 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4832 cErrors++;
4833 continue;
4834 }
4835 if (!SHW_PTE_IS_A(PteDst))
4836 {
4837 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4838 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4839 cErrors++;
4840 }
4841 fIgnoreFlags |= X86_PTE_P;
4842 }
4843# ifdef DEBUG_sandervl
4844 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4845# endif
4846 }
4847
4848 if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4849 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4850 )
4851 {
4852 AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4853 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4854 fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4855 cErrors++;
4856 continue;
4857 }
4858 } /* foreach PTE */
4859 }
4860 else
4861 {
4862 /*
4863 * Big Page.
4864 */
4865 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4866 if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
4867 {
4868 if (PdeDst.u & X86_PDE_RW)
4869 {
4870 AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4871 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4872 cErrors++;
4873 continue;
4874 }
4875 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4876 {
4877 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4878 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4879 cErrors++;
4880 continue;
4881 }
4882# if 0 /** @todo sync access bit properly... */
4883 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4884 {
4885 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4886 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4887 cErrors++;
4888 }
4889 fIgnoreFlags |= X86_PTE_RW;
4890# else
4891 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4892# endif
4893 }
4894 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4895 {
4896 /* access bit emulation (not implemented). */
4897 if ((PdeSrc.u & X86_PDE_A) || SHW_PDE_IS_P(PdeDst))
4898 {
4899 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4900 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4901 cErrors++;
4902 continue;
4903 }
4904 if (!SHW_PDE_IS_A(PdeDst))
4905 {
4906 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4907 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4908 cErrors++;
4909 }
4910 fIgnoreFlags |= X86_PTE_P;
4911 }
4912
4913 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4914 {
4915 AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4916 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4917 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4918 cErrors++;
4919 }
4920
4921 /* iterate the page table. */
4922 for (unsigned iPT = 0, off = 0;
4923 iPT < RT_ELEMENTS(pPTDst->a);
4924 iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE))
4925 {
4926 const SHWPTE PteDst = pPTDst->a[iPT];
4927
4928 if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4929 {
4930 AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4931 GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4932 cErrors++;
4933 }
4934
4935 /* skip not-present entries. */
4936 if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4937 continue;
4938
4939 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
4940
4941 /* match the physical addresses */
4942 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4943
4944# ifdef IN_RING3
4945 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4946 if (RT_FAILURE(rc))
4947 {
4948# if 0
4949 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4950 {
4951 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4952 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4953 cErrors++;
4954 }
4955# endif
4956 }
4957 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4958 {
4959 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4960 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4961 cErrors++;
4962 continue;
4963 }
4964# endif
4965 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4966 if (!pPhysPage)
4967 {
4968# if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4969 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4970 {
4971 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4972 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4973 cErrors++;
4974 continue;
4975 }
4976# endif
4977 if (SHW_PTE_IS_RW(PteDst))
4978 {
4979 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4980 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4981 cErrors++;
4982 }
4983 fIgnoreFlags |= X86_PTE_RW;
4984 }
4985 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4986 {
4987 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4988 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4989 cErrors++;
4990 continue;
4991 }
4992
4993 /* flags */
4994 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4995 {
4996 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4997 {
4998 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4999 {
5000 if ( SHW_PTE_IS_RW(PteDst)
5001 && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
5002 {
5003 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
5004 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5005 cErrors++;
5006 continue;
5007 }
5008 fIgnoreFlags |= X86_PTE_RW;
5009 }
5010 }
5011 else
5012 {
5013 if ( SHW_PTE_IS_P(PteDst)
5014 && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage)
5015# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
5016 && !PGM_PAGE_IS_MMIO(pPhysPage)
5017# endif
5018 )
5019 {
5020 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
5021 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5022 cErrors++;
5023 continue;
5024 }
5025 fIgnoreFlags |= X86_PTE_P;
5026 }
5027 }
5028
5029 if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
5030 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
5031 )
5032 {
5033 AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
5034 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
5035 fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
5036 cErrors++;
5037 continue;
5038 }
5039 } /* for each PTE */
5040 }
5041 }
5042 /* not present */
5043
5044 } /* for each PDE */
5045
5046 } /* for each PDPTE */
5047
5048 } /* for each PML4E */
5049
5050# ifdef DEBUG
5051 if (cErrors)
5052 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
5053# endif
5054# endif /* GST is in {32BIT, PAE, AMD64} */
5055 return cErrors;
5056#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
5057}
5058#endif /* VBOX_STRICT */
5059
5060
5061/**
5062 * Sets up the CR3 for shadow paging
5063 *
5064 * @returns Strict VBox status code.
5065 * @retval VINF_SUCCESS.
5066 *
5067 * @param pVCpu The cross context virtual CPU structure.
5068 * @param GCPhysCR3 The physical address in the CR3 register. (A20 mask
5069 * already applied.)
5070 */
5071PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
5072{
5073 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
5074 int rc = VINF_SUCCESS;
5075
5076 /* Update guest paging info. */
5077#if PGM_GST_TYPE == PGM_TYPE_32BIT \
5078 || PGM_GST_TYPE == PGM_TYPE_PAE \
5079 || PGM_GST_TYPE == PGM_TYPE_AMD64
5080
5081 LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
5082 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
5083
5084# if PGM_GST_TYPE == PGM_TYPE_PAE
5085 if ( !pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped)
5086 || pVCpu->pgm.s.GCPhysPaeCR3 != GCPhysCR3)
5087# endif
5088 {
5089 /*
5090 * Map the page CR3 points at.
5091 */
5092 RTHCPTR HCPtrGuestCR3;
5093 rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3);
5094 if (RT_SUCCESS(rc))
5095 {
5096# if PGM_GST_TYPE == PGM_TYPE_32BIT
5097# ifdef IN_RING3
5098 pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
5099 pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
5100# else
5101 pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
5102 pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
5103# endif
5104
5105# elif PGM_GST_TYPE == PGM_TYPE_PAE
5106# ifdef IN_RING3
5107 pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
5108 pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
5109# else
5110 pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
5111 pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
5112# endif
5113
5114 X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
5115#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5116 /*
5117 * When EPT is enabled by the nested-hypervisor and the nested-guest is in PAE mode,
5118 * the guest-CPU context would've already been updated with the 4 PAE PDPEs specified
5119 * in the virtual VMCS. The PDPEs can differ from those in guest memory referenced by
5120 * the translated nested-guest CR3. We -MUST- use the PDPEs provided in the virtual VMCS
5121 * rather than those in guest memory.
5122 *
5123 * See Intel spec. 26.3.2.4 "Loading Page-Directory-Pointer-Table Entries".
5124 */
5125 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
5126 CPUMGetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5127 else
5128#endif
5129 {
5130 /* Update CPUM with the PAE PDPEs referenced by CR3. */
5131 memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
5132 CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
5133 }
5134
5135 /*
5136 * Map the 4 PAE PDPEs.
5137 */
5138 rc = PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
5139 if (RT_SUCCESS(rc))
5140 {
5141# ifdef IN_RING3
5142 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
5143 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5144# else
5145 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5146 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
5147# endif
5148 pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
5149 }
5150
5151# elif PGM_GST_TYPE == PGM_TYPE_AMD64
5152# ifdef IN_RING3
5153 pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
5154 pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
5155# else
5156 pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
5157 pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
5158# endif
5159# endif
5160 }
5161 else
5162 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
5163 }
5164#endif
5165
5166 /*
5167 * Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
5168 */
5169# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5170 || PGM_SHW_TYPE == PGM_TYPE_PAE \
5171 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
5172 && ( PGM_GST_TYPE != PGM_TYPE_REAL \
5173 && PGM_GST_TYPE != PGM_TYPE_PROT))
5174
5175 Assert(!pVM->pgm.s.fNestedPaging);
5176 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
5177
5178 /*
5179 * Update the shadow root page as well since that's not fixed.
5180 */
5181 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5182 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
5183 PPGMPOOLPAGE pNewShwPageCR3;
5184
5185 PGM_LOCK_VOID(pVM);
5186
5187# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5188 if (pPool->cDirtyPages)
5189 pgmPoolResetDirtyPages(pVM);
5190# endif
5191
5192 Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */
5193 int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE,
5194 PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/, &pNewShwPageCR3);
5195 AssertFatalRC(rc2);
5196
5197 pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
5198 pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
5199
5200 /* Set the current hypervisor CR3. */
5201 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
5202
5203 /* Clean up the old CR3 root. */
5204 if ( pOldShwPageCR3
5205 && pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
5206 {
5207 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
5208
5209 /* Mark the page as unlocked; allow flushing again. */
5210 pgmPoolUnlockPage(pPool, pOldShwPageCR3);
5211
5212 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
5213 }
5214 PGM_UNLOCK(pVM);
5215# else
5216 NOREF(GCPhysCR3);
5217# endif
5218
5219 return rc;
5220}
5221
5222/**
5223 * Unmaps the shadow CR3.
5224 *
5225 * @returns VBox status, no specials.
5226 * @param pVCpu The cross context virtual CPU structure.
5227 */
5228PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
5229{
5230 LogFlow(("UnmapCR3\n"));
5231
5232 int rc = VINF_SUCCESS;
5233 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
5234
5235 /*
5236 * Update guest paging info.
5237 */
5238#if PGM_GST_TYPE == PGM_TYPE_32BIT
5239 pVCpu->pgm.s.pGst32BitPdR3 = 0;
5240 pVCpu->pgm.s.pGst32BitPdR0 = 0;
5241
5242#elif PGM_GST_TYPE == PGM_TYPE_PAE
5243 pVCpu->pgm.s.pGstPaePdptR3 = 0;
5244 pVCpu->pgm.s.pGstPaePdptR0 = 0;
5245 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
5246 {
5247 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
5248 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
5249 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
5250 }
5251
5252#elif PGM_GST_TYPE == PGM_TYPE_AMD64
5253 pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
5254 pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
5255
5256#else /* prot/real mode stub */
5257 /* nothing to do */
5258#endif
5259
5260 /*
5261 * PAE PDPEs (and CR3) might have been mapped via PGMGstMapPaePdpesAtCr3()
5262 * prior to switching to PAE in pfnMapCr3(), so we need to clear them here.
5263 */
5264 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
5265 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
5266 pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
5267
5268 /*
5269 * Update shadow paging info.
5270 */
5271#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
5272 || PGM_SHW_TYPE == PGM_TYPE_PAE \
5273 || PGM_SHW_TYPE == PGM_TYPE_AMD64))
5274# if PGM_GST_TYPE != PGM_TYPE_REAL
5275 Assert(!pVM->pgm.s.fNestedPaging);
5276# endif
5277 PGM_LOCK_VOID(pVM);
5278
5279 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
5280 {
5281 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5282
5283# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5284 if (pPool->cDirtyPages)
5285 pgmPoolResetDirtyPages(pVM);
5286# endif
5287
5288 /* Mark the page as unlocked; allow flushing again. */
5289 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
5290
5291 pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
5292 pVCpu->pgm.s.pShwPageCR3R3 = 0;
5293 pVCpu->pgm.s.pShwPageCR3R0 = 0;
5294 }
5295
5296 PGM_UNLOCK(pVM);
5297#endif
5298
5299 return rc;
5300}
5301
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette