VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 86462

Last change on this file since 86462 was 86462, checked in by vboxsync, 4 years ago

VMM/PGM: Don't check for, or even define, PGM_PLXFLAGS_MAPPING when PGM_WITHOUT_MAPPINGS is active (the default now).

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 199.1 KB
Line 
1/* $Id: PGMAllBth.h 86462 2020-10-06 16:43:10Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6 * PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7 * bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8 *
9 * @remarks This file is one big \#ifdef-orgy!
10 *
11 */
12
13/*
14 * Copyright (C) 2006-2020 Oracle Corporation
15 *
16 * This file is part of VirtualBox Open Source Edition (OSE), as
17 * available from http://www.virtualbox.org. This file is free software;
18 * you can redistribute it and/or modify it under the terms of the GNU
19 * General Public License (GPL) as published by the Free Software
20 * Foundation, in version 2 as it comes in the "COPYING" file of the
21 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
22 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
23 */
24
25#ifdef _MSC_VER
26/** @todo we're generating unnecessary code in nested/ept shadow mode and for
27 * real/prot-guest+RC mode. */
28# pragma warning(disable: 4505)
29#endif
30
31
32/*********************************************************************************************************************************
33* Internal Functions *
34*********************************************************************************************************************************/
35RT_C_DECLS_BEGIN
36PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
37#ifndef IN_RING3
38PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
39#endif
40PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
41static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
42static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
43static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
44#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
45static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
46#else
47static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
48#endif
49PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
50PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
51PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
52#ifdef VBOX_STRICT
53PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
54#endif
55PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
56PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
57
58#ifdef IN_RING3
59PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
60#endif
61RT_C_DECLS_END
62
63
64
65
66/*
67 * Filter out some illegal combinations of guest and shadow paging, so we can
68 * remove redundant checks inside functions.
69 */
70#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
71 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
72# error "Invalid combination; PAE guest implies PAE shadow"
73#endif
74
75#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
76 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 \
77 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
78# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
79#endif
80
81#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
82 && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE \
83 || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
84# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
85#endif
86
87#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
88 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
89# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
90#endif
91
92
93/**
94 * Enters the shadow+guest mode.
95 *
96 * @returns VBox status code.
97 * @param pVCpu The cross context virtual CPU structure.
98 * @param GCPhysCR3 The physical address from the CR3 register.
99 */
100PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
101{
102 /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
103 * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
104 */
105#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
106 || PGM_SHW_TYPE == PGM_TYPE_PAE \
107 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
108 && ( PGM_GST_TYPE == PGM_TYPE_REAL \
109 || PGM_GST_TYPE == PGM_TYPE_PROT))
110
111 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
112
113 Assert((HMIsNestedPagingActive(pVM) || VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
114 Assert(!pVM->pgm.s.fNestedPaging);
115
116 pgmLock(pVM);
117 /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
118 * but any calls to GC need a proper shadow page setup as well.
119 */
120 /* Free the previous root mapping if still active. */
121 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
122 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
123 if (pOldShwPageCR3)
124 {
125 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
126
127 /* Mark the page as unlocked; allow flushing again. */
128 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
129
130# ifndef PGM_WITHOUT_MAPPINGS
131 /* Remove the hypervisor mappings from the shadow page table. */
132 pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
133# endif
134
135 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
136 pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
137 pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
138 }
139
140 /* construct a fake address. */
141 GCPhysCR3 = RT_BIT_64(63);
142 PPGMPOOLPAGE pNewShwPageCR3;
143 int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
144 NIL_PGMPOOL_IDX, UINT32_MAX, false /*fLockPage*/,
145 &pNewShwPageCR3);
146 AssertRCReturn(rc, rc);
147
148 pVCpu->pgm.s.pShwPageCR3R3 = (R3PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR3(pVM, pNewShwPageCR3);
149 pVCpu->pgm.s.pShwPageCR3R0 = (R0PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR0(pVM, pNewShwPageCR3);
150
151 /* Mark the page as locked; disallow flushing. */
152 pgmPoolLockPage(pPool, pNewShwPageCR3);
153
154 /* Set the current hypervisor CR3. */
155 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
156
157# ifndef PGM_WITHOUT_MAPPINGS
158 /* Apply all hypervisor mappings to the new CR3. */
159 rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
160# endif
161
162 pgmUnlock(pVM);
163 return rc;
164#else
165 NOREF(pVCpu); NOREF(GCPhysCR3);
166 return VINF_SUCCESS;
167#endif
168}
169
170
171#ifndef IN_RING3
172
173# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
174/**
175 * Deal with a guest page fault.
176 *
177 * @returns Strict VBox status code.
178 * @retval VINF_EM_RAW_GUEST_TRAP
179 * @retval VINF_EM_RAW_EMULATE_INSTR
180 *
181 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
182 * @param pGstWalk The guest page table walk result.
183 * @param uErr The error code.
184 */
185PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
186{
187# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE)
188 /*
189 * Check for write conflicts with our hypervisor mapping.
190 *
191 * If the guest happens to access a non-present page, where our hypervisor
192 * is currently mapped, then we'll create a #PF storm in the guest.
193 */
194 if ( (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW)
195 && pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM))
196 && MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
197 {
198 /* Force a CR3 sync to check for conflicts and emulate the instruction. */
199 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
200 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
201 return VINF_EM_RAW_EMULATE_INSTR;
202 }
203# endif
204
205 /*
206 * Calc the error code for the guest trap.
207 */
208 uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
209 ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)
210 : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
211 if ( pGstWalk->Core.fRsvdError
212 || pGstWalk->Core.fBadPhysAddr)
213 {
214 uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P;
215 Assert(!pGstWalk->Core.fNotPresent);
216 }
217 else if (!pGstWalk->Core.fNotPresent)
218 uNewErr |= X86_TRAP_PF_P;
219 TRPMSetErrorCode(pVCpu, uNewErr);
220
221 LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
222 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
223 return VINF_EM_RAW_GUEST_TRAP;
224}
225# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
226
227
228#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
229/**
230 * Deal with a guest page fault.
231 *
232 * The caller has taken the PGM lock.
233 *
234 * @returns Strict VBox status code.
235 *
236 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
237 * @param uErr The error code.
238 * @param pRegFrame The register frame.
239 * @param pvFault The fault address.
240 * @param pPage The guest page at @a pvFault.
241 * @param pGstWalk The guest page table walk result.
242 * @param pfLockTaken PGM lock taken here or not (out). This is true
243 * when we're called.
244 */
245static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
246 RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
247# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
248 , PGSTPTWALK pGstWalk
249# endif
250 )
251{
252# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
253 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
254# endif
255 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
256 VBOXSTRICTRC rcStrict;
257
258 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
259 {
260 /*
261 * Physical page access handler.
262 */
263# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
264 const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
265# else
266 const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
267# endif
268 PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
269 if (pCur)
270 {
271 PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
272
273# ifdef PGM_SYNC_N_PAGES
274 /*
275 * If the region is write protected and we got a page not present fault, then sync
276 * the pages. If the fault was caused by a read, then restart the instruction.
277 * In case of write access continue to the GC write handler.
278 *
279 * ASSUMES that there is only one handler per page or that they have similar write properties.
280 */
281 if ( !(uErr & X86_TRAP_PF_P)
282 && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
283 {
284# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
285 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
286# else
287 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
288# endif
289 if ( RT_FAILURE(rcStrict)
290 || !(uErr & X86_TRAP_PF_RW)
291 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
292 {
293 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
294 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
295 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
296 return rcStrict;
297 }
298 }
299# endif
300# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
301 /*
302 * If the access was not thru a #PF(RSVD|...) resync the page.
303 */
304 if ( !(uErr & X86_TRAP_PF_RSVD)
305 && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
306# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
307 && pGstWalk->Core.fEffectiveRW
308 && !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
309# endif
310 )
311 {
312# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
313 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
314# else
315 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
316# endif
317 if ( RT_FAILURE(rcStrict)
318 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
319 {
320 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
321 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
322 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
323 return rcStrict;
324 }
325 }
326# endif
327
328 AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
329 || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
330 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
331 pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
332 if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
333 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
334 else
335 {
336 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
337 if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
338 }
339
340 if (pCurType->CTX_SUFF(pfnPfHandler))
341 {
342 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
343 void *pvUser = pCur->CTX_SUFF(pvUser);
344
345 STAM_PROFILE_START(&pCur->Stat, h);
346 if (pCur->hType != pPool->hAccessHandlerType)
347 {
348 pgmUnlock(pVM);
349 *pfLockTaken = false;
350 }
351
352 rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
353
354# ifdef VBOX_WITH_STATISTICS
355 pgmLock(pVM);
356 pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
357 if (pCur)
358 STAM_PROFILE_STOP(&pCur->Stat, h);
359 pgmUnlock(pVM);
360# endif
361 }
362 else
363 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
364
365 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
366 return rcStrict;
367 }
368 }
369
370 /*
371 * There is a handled area of the page, but this fault doesn't belong to it.
372 * We must emulate the instruction.
373 *
374 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
375 * we first check if this was a page-not-present fault for a page with only
376 * write access handlers. Restart the instruction if it wasn't a write access.
377 */
378 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
379
380 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
381 && !(uErr & X86_TRAP_PF_P))
382 {
383# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
384 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
385# else
386 rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
387# endif
388 if ( RT_FAILURE(rcStrict)
389 || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
390 || !(uErr & X86_TRAP_PF_RW))
391 {
392 AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
393 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
394 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
395 return rcStrict;
396 }
397 }
398
399 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
400 * It's writing to an unhandled part of the LDT page several million times.
401 */
402 rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
403 LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
404 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
405 return rcStrict;
406} /* if any kind of handler */
407# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
408
409
410/**
411 * \#PF Handler for raw-mode guest execution.
412 *
413 * @returns VBox status code (appropriate for trap handling and GC return).
414 *
415 * @param pVCpu The cross context virtual CPU structure.
416 * @param uErr The trap error code.
417 * @param pRegFrame Trap register frame.
418 * @param pvFault The fault address.
419 * @param pfLockTaken PGM lock taken here or not (out)
420 */
421PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
422{
423 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
424
425 *pfLockTaken = false;
426
427# if ( PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \
428 || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
429 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
430 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
431 && PGM_SHW_TYPE != PGM_TYPE_NONE
432 int rc;
433
434# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
435 /*
436 * Walk the guest page translation tables and check if it's a guest fault.
437 */
438 GSTPTWALK GstWalk;
439 rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
440 if (RT_FAILURE_NP(rc))
441 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
442
443 /* assert some GstWalk sanity. */
444# if PGM_GST_TYPE == PGM_TYPE_AMD64
445 /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. */
446# endif
447# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
448 /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */
449# endif
450 /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */
451 /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */
452 Assert(GstWalk.Core.fSucceeded);
453
454 if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
455 {
456 if ( ( (uErr & X86_TRAP_PF_RW)
457 && !GstWalk.Core.fEffectiveRW
458 && ( (uErr & X86_TRAP_PF_US)
459 || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
460 || ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
461 || ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
462 )
463 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
464 }
465
466 /*
467 * Set the accessed and dirty flags.
468 */
469# if PGM_GST_TYPE == PGM_TYPE_AMD64
470 GstWalk.Pml4e.u |= X86_PML4E_A;
471 GstWalk.pPml4e->u |= X86_PML4E_A;
472 GstWalk.Pdpe.u |= X86_PDPE_A;
473 GstWalk.pPdpe->u |= X86_PDPE_A;
474# endif
475 if (GstWalk.Core.fBigPage)
476 {
477 Assert(GstWalk.Pde.b.u1Size);
478 if (uErr & X86_TRAP_PF_RW)
479 {
480 GstWalk.Pde.u |= X86_PDE4M_A | X86_PDE4M_D;
481 GstWalk.pPde->u |= X86_PDE4M_A | X86_PDE4M_D;
482 }
483 else
484 {
485 GstWalk.Pde.u |= X86_PDE4M_A;
486 GstWalk.pPde->u |= X86_PDE4M_A;
487 }
488 }
489 else
490 {
491 Assert(!GstWalk.Pde.b.u1Size);
492 GstWalk.Pde.u |= X86_PDE_A;
493 GstWalk.pPde->u |= X86_PDE_A;
494 if (uErr & X86_TRAP_PF_RW)
495 {
496# ifdef VBOX_WITH_STATISTICS
497 if (!GstWalk.Pte.n.u1Dirty)
498 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
499 else
500 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
501# endif
502 GstWalk.Pte.u |= X86_PTE_A | X86_PTE_D;
503 GstWalk.pPte->u |= X86_PTE_A | X86_PTE_D;
504 }
505 else
506 {
507 GstWalk.Pte.u |= X86_PTE_A;
508 GstWalk.pPte->u |= X86_PTE_A;
509 }
510 Assert(GstWalk.Pte.u == GstWalk.pPte->u);
511 }
512 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u,
513 ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
514# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
515 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */
516# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
517
518 /* Take the big lock now. */
519 *pfLockTaken = true;
520 pgmLock(pVM);
521
522# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
523 /*
524 * If it is a reserved bit fault we know that it is an MMIO (access
525 * handler) related fault and can skip some 200 lines of code.
526 */
527 if (uErr & X86_TRAP_PF_RSVD)
528 {
529 Assert(uErr & X86_TRAP_PF_P);
530 PPGMPAGE pPage;
531# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
532 rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
533 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
534 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
535 pfLockTaken, &GstWalk));
536 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
537# else
538 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
539 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
540 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
541 pfLockTaken));
542 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
543# endif
544 AssertRC(rc);
545 PGM_INVL_PG(pVCpu, pvFault);
546 return rc; /* Restart with the corrected entry. */
547 }
548# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
549
550 /*
551 * Fetch the guest PDE, PDPE and PML4E.
552 */
553# if PGM_SHW_TYPE == PGM_TYPE_32BIT
554 const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
555 PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
556
557# elif PGM_SHW_TYPE == PGM_TYPE_PAE
558 const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
559 PX86PDPAE pPDDst;
560# if PGM_GST_TYPE == PGM_TYPE_PAE
561 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
562# else
563 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
564# endif
565 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
566
567# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
568 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
569 PX86PDPAE pPDDst;
570# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
571 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A,
572 X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst);
573# else
574 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
575# endif
576 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
577
578# elif PGM_SHW_TYPE == PGM_TYPE_EPT
579 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
580 PEPTPD pPDDst;
581 rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
582 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
583# endif
584 Assert(pPDDst);
585
586# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
587 /*
588 * Dirty page handling.
589 *
590 * If we successfully correct the write protection fault due to dirty bit
591 * tracking, then return immediately.
592 */
593 if (uErr & X86_TRAP_PF_RW) /* write fault? */
594 {
595 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
596 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
597 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
598 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
599 {
600 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
601 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
602 ? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
603 : &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
604 Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
605 return VINF_SUCCESS;
606 }
607#ifdef DEBUG_bird
608 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
609 AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
610#endif
611 }
612
613# if 0 /* rarely useful; leave for debugging. */
614 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
615# endif
616# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
617
618 /*
619 * A common case is the not-present error caused by lazy page table syncing.
620 *
621 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
622 * here so we can safely assume that the shadow PT is present when calling
623 * SyncPage later.
624 *
625 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
626 * of mapping conflict and defer to SyncCR3 in R3.
627 * (Again, we do NOT support access handlers for non-present guest pages.)
628 *
629 */
630# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
631 Assert(GstWalk.Pde.n.u1Present);
632# endif
633 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
634 && !pPDDst->a[iPDDst].n.u1Present)
635 {
636 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
637# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
638 LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
639 rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
640# else
641 LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
642 rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
643# endif
644 if (RT_SUCCESS(rc))
645 return rc;
646 Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
647 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
648 return VINF_PGM_SYNC_CR3;
649 }
650
651# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
652 /*
653 * Check if this address is within any of our mappings.
654 *
655 * This is *very* fast and it's gonna save us a bit of effort below and prevent
656 * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
657 * (BTW, it's impossible to have physical access handlers in a mapping.)
658 */
659 if (pgmMapAreMappingsEnabled(pVM))
660 {
661 PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
662 for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
663 {
664 if (pvFault < pMapping->GCPtr)
665 break;
666 if (pvFault - pMapping->GCPtr < pMapping->cb)
667 {
668 /*
669 * The first thing we check is if we've got an undetected conflict.
670 */
671 if (pgmMapAreMappingsFloating(pVM))
672 {
673 unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
674 while (iPT-- > 0)
675 if (GstWalk.pPde[iPT].n.u1Present)
676 {
677 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
678 Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
679 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
680 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
681 return VINF_PGM_SYNC_CR3;
682 }
683 }
684
685 /*
686 * Pretend we're not here and let the guest handle the trap.
687 */
688 TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
689 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
690 LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
691 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
692 return VINF_EM_RAW_GUEST_TRAP;
693 }
694 }
695 } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
696# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
697
698 /*
699 * Check if this fault address is flagged for special treatment,
700 * which means we'll have to figure out the physical address and
701 * check flags associated with it.
702 *
703 * ASSUME that we can limit any special access handling to pages
704 * in page tables which the guest believes to be present.
705 */
706# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
707 RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
708# else
709 RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK);
710# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
711 PPGMPAGE pPage;
712 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
713 if (RT_FAILURE(rc))
714 {
715 /*
716 * When the guest accesses invalid physical memory (e.g. probing
717 * of RAM or accessing a remapped MMIO range), then we'll fall
718 * back to the recompiler to emulate the instruction.
719 */
720 LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
721 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
722 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
723 return VINF_EM_RAW_EMULATE_INSTR;
724 }
725
726 /*
727 * Any handlers for this page?
728 */
729 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
730# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
731 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
732 &GstWalk));
733# else
734 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
735# endif
736
737 /*
738 * We are here only if page is present in Guest page tables and
739 * trap is not handled by our handlers.
740 *
741 * Check it for page out-of-sync situation.
742 */
743 if (!(uErr & X86_TRAP_PF_P))
744 {
745 /*
746 * Page is not present in our page tables. Try to sync it!
747 */
748 if (uErr & X86_TRAP_PF_US)
749 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
750 else /* supervisor */
751 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
752
753 if (PGM_PAGE_IS_BALLOONED(pPage))
754 {
755 /* Emulate reads from ballooned pages as they are not present in
756 our shadow page tables. (Required for e.g. Solaris guests; soft
757 ecc, random nr generator.) */
758 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
759 LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
760 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
761 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
762 return rc;
763 }
764
765# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
766 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
767# else
768 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
769# endif
770 if (RT_SUCCESS(rc))
771 {
772 /* The page was successfully synced, return to the guest. */
773 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
774 return VINF_SUCCESS;
775 }
776 }
777 else /* uErr & X86_TRAP_PF_P: */
778 {
779 /*
780 * Write protected pages are made writable when the guest makes the
781 * first write to it. This happens for pages that are shared, write
782 * monitored or not yet allocated.
783 *
784 * We may also end up here when CR0.WP=0 in the guest.
785 *
786 * Also, a side effect of not flushing global PDEs are out of sync
787 * pages due to physical monitored regions, that are no longer valid.
788 * Assume for now it only applies to the read/write flag.
789 */
790 if (uErr & X86_TRAP_PF_RW)
791 {
792 /*
793 * Check if it is a read-only page.
794 */
795 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
796 {
797 Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
798 Assert(!PGM_PAGE_IS_ZERO(pPage));
799 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
800 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
801
802 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
803 if (rc != VINF_SUCCESS)
804 {
805 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
806 return rc;
807 }
808 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
809 return VINF_EM_NO_MEMORY;
810 }
811
812# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
813 /*
814 * Check to see if we need to emulate the instruction if CR0.WP=0.
815 */
816 if ( !GstWalk.Core.fEffectiveRW
817 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
818 && CPUMGetGuestCPL(pVCpu) < 3)
819 {
820 Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P));
821
822 /*
823 * The Netware WP0+RO+US hack.
824 *
825 * Netware sometimes(/always?) runs with WP0. It has been observed doing
826 * excessive write accesses to pages which are mapped with US=1 and RW=0
827 * while WP=0. This causes a lot of exits and extremely slow execution.
828 * To avoid trapping and emulating every write here, we change the shadow
829 * page table entry to map it as US=0 and RW=1 until user mode tries to
830 * access it again (see further below). We count these shadow page table
831 * changes so we can avoid having to clear the page pool every time the WP
832 * bit changes to 1 (see PGMCr0WpEnabled()).
833 */
834# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) && 1
835 if ( GstWalk.Core.fEffectiveUS
836 && !GstWalk.Core.fEffectiveRW
837 && (GstWalk.Core.fBigPage || GstWalk.Pde.n.u1Write)
838 && pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
839 {
840 Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, GstWalk.Core.fBigPage));
841 rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, GstWalk.Core.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
842 if (rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3)
843 {
844 PGM_INVL_PG(pVCpu, pvFault);
845 pVCpu->pgm.s.cNetwareWp0Hacks++;
846 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsHack; });
847 return rc;
848 }
849 AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
850 Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
851 }
852# endif
853
854 /* Interpret the access. */
855 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
856 Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), GstWalk.Core.fBigPage, GstWalk.Core.fEffectiveUS));
857 if (RT_SUCCESS(rc))
858 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
859 else
860 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
861 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
862 return rc;
863 }
864# endif
865 /// @todo count the above case; else
866 if (uErr & X86_TRAP_PF_US)
867 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
868 else /* supervisor */
869 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
870
871 /*
872 * Sync the page.
873 *
874 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
875 * page is not present, which is not true in this case.
876 */
877# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
878 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
879# else
880 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
881# endif
882 if (RT_SUCCESS(rc))
883 {
884 /*
885 * Page was successfully synced, return to guest but invalidate
886 * the TLB first as the page is very likely to be in it.
887 */
888# if PGM_SHW_TYPE == PGM_TYPE_EPT
889 HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
890# else
891 PGM_INVL_PG(pVCpu, pvFault);
892# endif
893# ifdef VBOX_STRICT
894 RTGCPHYS GCPhys2 = RTGCPHYS_MAX;
895 uint64_t fPageGst = UINT64_MAX;
896 if (!pVM->pgm.s.fNestedPaging)
897 {
898 rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
899 AssertMsg(RT_SUCCESS(rc) && ((fPageGst & X86_PTE_RW) || ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
900 LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
901 }
902# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
903 uint64_t fPageShw = 0;
904 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
905 AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
906 ("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GCPhys2, fPageGst, pvFault));
907# endif
908# endif /* VBOX_STRICT */
909 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
910 return VINF_SUCCESS;
911 }
912 }
913# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
914 /*
915 * Check for Netware WP0+RO+US hack from above and undo it when user
916 * mode accesses the page again.
917 */
918 else if ( GstWalk.Core.fEffectiveUS
919 && !GstWalk.Core.fEffectiveRW
920 && (GstWalk.Core.fBigPage || GstWalk.Pde.n.u1Write)
921 && pVCpu->pgm.s.cNetwareWp0Hacks > 0
922 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
923 && CPUMGetGuestCPL(pVCpu) == 3
924 && pVM->cCpus == 1
925 )
926 {
927 Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
928 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
929 if (RT_SUCCESS(rc))
930 {
931 PGM_INVL_PG(pVCpu, pvFault);
932 pVCpu->pgm.s.cNetwareWp0Hacks--;
933 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsUnhack; });
934 return VINF_SUCCESS;
935 }
936 }
937# endif /* PGM_WITH_PAGING */
938
939 /** @todo else: why are we here? */
940
941# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
942 /*
943 * Check for VMM page flags vs. Guest page flags consistency.
944 * Currently only for debug purposes.
945 */
946 if (RT_SUCCESS(rc))
947 {
948 /* Get guest page flags. */
949 uint64_t fPageGst;
950 int rc2 = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
951 if (RT_SUCCESS(rc2))
952 {
953 uint64_t fPageShw = 0;
954 rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
955
956#if 0
957 /*
958 * Compare page flags.
959 * Note: we have AVL, A, D bits desynced.
960 */
961 AssertMsg( (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
962 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
963 || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
964 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
965 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
966 && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW
967 && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US),
968 ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
969 pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
97001:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
97101:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
972
97301:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
97401:01:15.625516 00:08:43.268051 Location :
975e:\vbox\svn\trunk\srcPage flags mismatch!
976pvFault=fffff801b0d7b000
977 uErr=11 X86_TRAP_PF_ID | X86_TRAP_PF_P
978GCPhys=0000000019b52000
979fPageShw=0
980fPageGst=77b0000000000121
981rc=0
982#endif
983
984 }
985 else
986 AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
987 }
988 else
989 AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
990# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
991 }
992
993
994 /*
995 * If we get here it is because something failed above, i.e. most like guru
996 * meditiation time.
997 */
998 LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
999 __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
1000 return rc;
1001
1002# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
1003 NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
1004 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1005 return VERR_PGM_NOT_USED_IN_MODE;
1006# endif
1007}
1008
1009#endif /* !IN_RING3 */
1010
1011
1012/**
1013 * Emulation of the invlpg instruction.
1014 *
1015 *
1016 * @returns VBox status code.
1017 *
1018 * @param pVCpu The cross context virtual CPU structure.
1019 * @param GCPtrPage Page to invalidate.
1020 *
1021 * @remark ASSUMES that the guest is updating before invalidating. This order
1022 * isn't required by the CPU, so this is speculative and could cause
1023 * trouble.
1024 * @remark No TLB shootdown is done on any other VCPU as we assume that
1025 * invlpg emulation is the *only* reason for calling this function.
1026 * (The guest has to shoot down TLB entries on other CPUs itself)
1027 * Currently true, but keep in mind!
1028 *
1029 * @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1030 * Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1031 */
1032PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1033{
1034#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1035 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1036 && PGM_SHW_TYPE != PGM_TYPE_NONE
1037 int rc;
1038 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1039 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1040
1041 PGM_LOCK_ASSERT_OWNER(pVM);
1042
1043 LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1044
1045 /*
1046 * Get the shadow PD entry and skip out if this PD isn't present.
1047 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1048 */
1049# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1050 const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1051 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1052
1053 /* Fetch the pgm pool shadow descriptor. */
1054 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1055# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1056 if (!pShwPde)
1057 {
1058 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1059 return VINF_SUCCESS;
1060 }
1061# else
1062 Assert(pShwPde);
1063# endif
1064
1065# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1066 const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1067 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1068
1069 /* If the shadow PDPE isn't present, then skip the invalidate. */
1070# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1071 if (!pPdptDst || !pPdptDst->a[iPdpt].n.u1Present)
1072# else
1073 if (!pPdptDst->a[iPdpt].n.u1Present)
1074# endif
1075 {
1076#ifndef PGM_WITHOUT_MAPPINGS
1077 Assert(!pPdptDst || !(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1078#endif
1079 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1080 PGM_INVL_PG(pVCpu, GCPtrPage);
1081 return VINF_SUCCESS;
1082 }
1083
1084 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1085 PPGMPOOLPAGE pShwPde = NULL;
1086 PX86PDPAE pPDDst;
1087
1088 /* Fetch the pgm pool shadow descriptor. */
1089 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1090 AssertRCSuccessReturn(rc, rc);
1091 Assert(pShwPde);
1092
1093 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1094 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1095
1096# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1097 /* PML4 */
1098 /*const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/
1099 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1100 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1101 PX86PDPAE pPDDst;
1102 PX86PDPT pPdptDst;
1103 PX86PML4E pPml4eDst;
1104 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1105 if (rc != VINF_SUCCESS)
1106 {
1107 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1108 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1109 PGM_INVL_PG(pVCpu, GCPtrPage);
1110 return VINF_SUCCESS;
1111 }
1112 Assert(pPDDst);
1113
1114 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1115 PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1116
1117 if (!pPdpeDst->n.u1Present)
1118 {
1119 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1120 PGM_INVL_PG(pVCpu, GCPtrPage);
1121 return VINF_SUCCESS;
1122 }
1123
1124 /* Fetch the pgm pool shadow descriptor. */
1125 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1126 Assert(pShwPde);
1127
1128# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1129
1130 const SHWPDE PdeDst = *pPdeDst;
1131 if (!PdeDst.n.u1Present)
1132 {
1133 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1134 PGM_INVL_PG(pVCpu, GCPtrPage);
1135 return VINF_SUCCESS;
1136 }
1137
1138 /*
1139 * Get the guest PD entry and calc big page.
1140 */
1141# if PGM_GST_TYPE == PGM_TYPE_32BIT
1142 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1143 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1144 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1145# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1146 unsigned iPDSrc = 0;
1147# if PGM_GST_TYPE == PGM_TYPE_PAE
1148 X86PDPE PdpeSrcIgn;
1149 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1150# else /* AMD64 */
1151 PX86PML4E pPml4eSrcIgn;
1152 X86PDPE PdpeSrcIgn;
1153 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1154# endif
1155 GSTPDE PdeSrc;
1156
1157 if (pPDSrc)
1158 PdeSrc = pPDSrc->a[iPDSrc];
1159 else
1160 PdeSrc.u = 0;
1161# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1162 const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1163 const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1164 if (fWasBigPage != fIsBigPage)
1165 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1166
1167# ifdef IN_RING3
1168 /*
1169 * If a CR3 Sync is pending we may ignore the invalidate page operation
1170 * depending on the kind of sync and if it's a global page or not.
1171 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
1172 */
1173# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1174 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1175 || ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1176 && fIsBigPage
1177 && PdeSrc.b.u1Global
1178 )
1179 )
1180# else
1181 if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1182# endif
1183 {
1184 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1185 return VINF_SUCCESS;
1186 }
1187# endif /* IN_RING3 */
1188
1189 /*
1190 * Deal with the Guest PDE.
1191 */
1192 rc = VINF_SUCCESS;
1193 if (PdeSrc.n.u1Present)
1194 {
1195 Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1196 && (PdeSrc.n.u1Write || !PdeDst.n.u1Write || pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1197# ifndef PGM_WITHOUT_MAPPINGS
1198 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1199 {
1200 /*
1201 * Conflict - Let SyncPT deal with it to avoid duplicate code.
1202 */
1203 Assert(pgmMapAreMappingsEnabled(pVM));
1204 Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1205 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1206 }
1207 else
1208# endif /* !PGM_WITHOUT_MAPPINGS */
1209 if (!fIsBigPage)
1210 {
1211 /*
1212 * 4KB - page.
1213 */
1214 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1215 RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1216
1217# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1218 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1219 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
1220# endif
1221 if (pShwPage->GCPhys == GCPhys)
1222 {
1223 /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1224 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1225
1226 PGSTPT pPTSrc;
1227 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1228 if (RT_SUCCESS(rc))
1229 {
1230 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1231 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1232 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1233 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1234 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1235 GCPtrPage, PteSrc.n.u1Present,
1236 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1237 PteSrc.n.u1User & PdeSrc.n.u1User,
1238 (uint64_t)PteSrc.u,
1239 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1240 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1241 }
1242 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1243 PGM_INVL_PG(pVCpu, GCPtrPage);
1244 }
1245 else
1246 {
1247 /*
1248 * The page table address changed.
1249 */
1250 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1251 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1252 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1253 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1254 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1255 PGM_INVL_VCPU_TLBS(pVCpu);
1256 }
1257 }
1258 else
1259 {
1260 /*
1261 * 2/4MB - page.
1262 */
1263 /* Before freeing the page, check if anything really changed. */
1264 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1265 RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1266# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1267 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1268 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1269# endif
1270 if ( pShwPage->GCPhys == GCPhys
1271 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1272 {
1273 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1274 /** @todo This test is wrong as it cannot check the G bit!
1275 * FIXME */
1276 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1277 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1278 && ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1279 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1280 {
1281 LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1282 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1283 return VINF_SUCCESS;
1284 }
1285 }
1286
1287 /*
1288 * Ok, the page table is present and it's been changed in the guest.
1289 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1290 * We could do this for some flushes in GC too, but we need an algorithm for
1291 * deciding which 4MB pages containing code likely to be executed very soon.
1292 */
1293 LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1294 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1295 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1296 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1297 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1298 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1299 }
1300 }
1301 else
1302 {
1303 /*
1304 * Page directory is not present, mark shadow PDE not present.
1305 */
1306# ifndef PGM_WITHOUT_MAPPINGS
1307 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1308# endif
1309 {
1310 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1311 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1312 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1313 PGM_INVL_PG(pVCpu, GCPtrPage);
1314 }
1315# ifndef PGM_WITHOUT_MAPPINGS
1316 else
1317 {
1318 Assert(pgmMapAreMappingsEnabled(pVM));
1319 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1320 }
1321# endif
1322 }
1323 return rc;
1324
1325#else /* guest real and protected mode, nested + ept, none. */
1326 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1327 NOREF(pVCpu); NOREF(GCPtrPage);
1328 return VINF_SUCCESS;
1329#endif
1330}
1331
1332#if PGM_SHW_TYPE != PGM_TYPE_NONE
1333
1334/**
1335 * Update the tracking of shadowed pages.
1336 *
1337 * @param pVCpu The cross context virtual CPU structure.
1338 * @param pShwPage The shadow page.
1339 * @param HCPhys The physical page we is being dereferenced.
1340 * @param iPte Shadow PTE index
1341 * @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1342 */
1343DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1344 RTGCPHYS GCPhysPage)
1345{
1346 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1347
1348# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1349 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1350 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1351
1352 /* Use the hint we retrieved from the cached guest PT. */
1353 if (pShwPage->fDirty)
1354 {
1355 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1356
1357 Assert(pShwPage->cPresent);
1358 Assert(pPool->cPresent);
1359 pShwPage->cPresent--;
1360 pPool->cPresent--;
1361
1362 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1363 AssertRelease(pPhysPage);
1364 pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1365 return;
1366 }
1367# else
1368 NOREF(GCPhysPage);
1369# endif
1370
1371 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1372 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1373
1374 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1375 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1376 * 2. write protect all shadowed pages. I.e. implement caching.
1377 */
1378 /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1379
1380 /*
1381 * Find the guest address.
1382 */
1383 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1384 pRam;
1385 pRam = pRam->CTX_SUFF(pNext))
1386 {
1387 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1388 while (iPage-- > 0)
1389 {
1390 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1391 {
1392 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1393
1394 Assert(pShwPage->cPresent);
1395 Assert(pPool->cPresent);
1396 pShwPage->cPresent--;
1397 pPool->cPresent--;
1398
1399 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1400 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1401 return;
1402 }
1403 }
1404 }
1405
1406 for (;;)
1407 AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1408}
1409
1410
1411/**
1412 * Update the tracking of shadowed pages.
1413 *
1414 * @param pVCpu The cross context virtual CPU structure.
1415 * @param pShwPage The shadow page.
1416 * @param u16 The top 16-bit of the pPage->HCPhys.
1417 * @param pPage Pointer to the guest page. this will be modified.
1418 * @param iPTDst The index into the shadow table.
1419 */
1420DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1421{
1422 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1423
1424 /*
1425 * Just deal with the simple first time here.
1426 */
1427 if (!u16)
1428 {
1429 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1430 u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1431 /* Save the page table index. */
1432 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1433 }
1434 else
1435 u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1436
1437 /* write back */
1438 Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1439 PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1440
1441 /* update statistics. */
1442 pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1443 pShwPage->cPresent++;
1444 if (pShwPage->iFirstPresent > iPTDst)
1445 pShwPage->iFirstPresent = iPTDst;
1446}
1447
1448
1449/**
1450 * Modifies a shadow PTE to account for access handlers.
1451 *
1452 * @param pVM The cross context VM structure.
1453 * @param pPage The page in question.
1454 * @param fPteSrc The shadowed flags of the source PTE. Must include the
1455 * A (accessed) bit so it can be emulated correctly.
1456 * @param pPteDst The shadow PTE (output). This is temporary storage and
1457 * does not need to be set atomically.
1458 */
1459DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1460{
1461 NOREF(pVM); RT_NOREF_PV(fPteSrc);
1462
1463 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1464 * Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1465 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1466 {
1467 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1468# if PGM_SHW_TYPE == PGM_TYPE_EPT
1469 pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1470 pPteDst->n.u1Present = 1;
1471 pPteDst->n.u1Execute = 1;
1472 pPteDst->n.u1IgnorePAT = 1;
1473 pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1474 /* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1475# else
1476 if (fPteSrc & X86_PTE_A)
1477 {
1478 SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage));
1479 SHW_PTE_SET_RO(*pPteDst);
1480 }
1481 else
1482 SHW_PTE_SET(*pPteDst, 0);
1483# endif
1484 }
1485# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1486# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
1487 else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1488 && ( BTH_IS_NP_ACTIVE(pVM)
1489 || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1490# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1491 && pVM->pgm.s.fLessThan52PhysicalAddressBits
1492# endif
1493 )
1494 {
1495 LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1496# if PGM_SHW_TYPE == PGM_TYPE_EPT
1497 /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1498 pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1499 /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1500 pPteDst->n.u1Present = 0;
1501 pPteDst->n.u1Write = 1;
1502 pPteDst->n.u1Execute = 0;
1503 /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1504 pPteDst->n.u3EMT = 7;
1505# else
1506 /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1507 SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P);
1508# endif
1509 }
1510# endif
1511# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1512 else
1513 {
1514 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1515 SHW_PTE_SET(*pPteDst, 0);
1516 }
1517 /** @todo count these kinds of entries. */
1518}
1519
1520
1521/**
1522 * Creates a 4K shadow page for a guest page.
1523 *
1524 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1525 * physical address. The PdeSrc argument only the flags are used. No page
1526 * structured will be mapped in this function.
1527 *
1528 * @param pVCpu The cross context virtual CPU structure.
1529 * @param pPteDst Destination page table entry.
1530 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1531 * Can safely assume that only the flags are being used.
1532 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1533 * @param pShwPage Pointer to the shadow page.
1534 * @param iPTDst The index into the shadow table.
1535 *
1536 * @remark Not used for 2/4MB pages!
1537 */
1538# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
1539static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1540 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1541# else
1542static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1543 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1544# endif
1545{
1546 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1547 RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1548
1549# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1550 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1551 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1552
1553 if (pShwPage->fDirty)
1554 {
1555 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1556 PGSTPT pGstPT;
1557
1558 /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1559 pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1560 GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1561 pGstPT->a[iPTDst].u = PteSrc.u;
1562 }
1563# else
1564 Assert(!pShwPage->fDirty);
1565# endif
1566
1567# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1568 if ( PteSrc.n.u1Present
1569 && GST_IS_PTE_VALID(pVCpu, PteSrc))
1570# endif
1571 {
1572# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1573 RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1574# endif
1575 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1576
1577 /*
1578 * Find the ram range.
1579 */
1580 PPGMPAGE pPage;
1581 int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1582 if (RT_SUCCESS(rc))
1583 {
1584 /* Ignore ballooned pages.
1585 Don't return errors or use a fatal assert here as part of a
1586 shadow sync range might included ballooned pages. */
1587 if (PGM_PAGE_IS_BALLOONED(pPage))
1588 {
1589 Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */
1590 return;
1591 }
1592
1593# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1594 /* Make the page writable if necessary. */
1595 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1596 && ( PGM_PAGE_IS_ZERO(pPage)
1597# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1598 || ( PteSrc.n.u1Write
1599# else
1600 || ( 1
1601# endif
1602 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1603# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1604 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1605# endif
1606# ifdef VBOX_WITH_PAGE_SHARING
1607 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1608# endif
1609 )
1610 )
1611 )
1612 {
1613 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1614 AssertRC(rc);
1615 }
1616# endif
1617
1618 /*
1619 * Make page table entry.
1620 */
1621 SHWPTE PteDst;
1622# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1623 uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1624# else
1625 uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D;
1626# endif
1627 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1628 PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1629 else
1630 {
1631# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1632 /*
1633 * If the page or page directory entry is not marked accessed,
1634 * we mark the page not present.
1635 */
1636 if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
1637 {
1638 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1639 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1640 SHW_PTE_SET(PteDst, 0);
1641 }
1642 /*
1643 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1644 * when the page is modified.
1645 */
1646 else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1647 {
1648 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1649 SHW_PTE_SET(PteDst,
1650 fGstShwPteFlags
1651 | PGM_PAGE_GET_HCPHYS(pPage)
1652 | PGM_PTFLAGS_TRACK_DIRTY);
1653 SHW_PTE_SET_RO(PteDst);
1654 }
1655 else
1656# endif
1657 {
1658 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1659# if PGM_SHW_TYPE == PGM_TYPE_EPT
1660 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1661 PteDst.n.u1Present = 1;
1662 PteDst.n.u1Write = 1;
1663 PteDst.n.u1Execute = 1;
1664 PteDst.n.u1IgnorePAT = 1;
1665 PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1666 /* PteDst.n.u1Size = 0 */
1667# else
1668 SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage));
1669# endif
1670 }
1671
1672 /*
1673 * Make sure only allocated pages are mapped writable.
1674 */
1675 if ( SHW_PTE_IS_P_RW(PteDst)
1676 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1677 {
1678 /* Still applies to shared pages. */
1679 Assert(!PGM_PAGE_IS_ZERO(pPage));
1680 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1681 Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1682 }
1683 }
1684
1685 /*
1686 * Keep user track up to date.
1687 */
1688 if (SHW_PTE_IS_P(PteDst))
1689 {
1690 if (!SHW_PTE_IS_P(*pPteDst))
1691 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1692 else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1693 {
1694 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst)));
1695 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1696 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1697 }
1698 }
1699 else if (SHW_PTE_IS_P(*pPteDst))
1700 {
1701 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
1702 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1703 }
1704
1705 /*
1706 * Update statistics and commit the entry.
1707 */
1708# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1709 if (!PteSrc.n.u1Global)
1710 pShwPage->fSeenNonGlobal = true;
1711# endif
1712 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1713 return;
1714 }
1715
1716/** @todo count these three different kinds. */
1717 Log2(("SyncPageWorker: invalid address in Pte\n"));
1718 }
1719# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1720 else if (!PteSrc.n.u1Present)
1721 Log2(("SyncPageWorker: page not present in Pte\n"));
1722 else
1723 Log2(("SyncPageWorker: invalid Pte\n"));
1724# endif
1725
1726 /*
1727 * The page is not present or the PTE is bad. Replace the shadow PTE by
1728 * an empty entry, making sure to keep the user tracking up to date.
1729 */
1730 if (SHW_PTE_IS_P(*pPteDst))
1731 {
1732 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
1733 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1734 }
1735 SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1736}
1737
1738
1739/**
1740 * Syncs a guest OS page.
1741 *
1742 * There are no conflicts at this point, neither is there any need for
1743 * page table allocations.
1744 *
1745 * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1746 * When called in AMD64 guest mode, the guest PML4E shall be valid.
1747 *
1748 * @returns VBox status code.
1749 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1750 * @param pVCpu The cross context virtual CPU structure.
1751 * @param PdeSrc Page directory entry of the guest.
1752 * @param GCPtrPage Guest context page address.
1753 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1754 * @param uErr Fault error (X86_TRAP_PF_*).
1755 */
1756static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1757{
1758 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1759 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
1760 LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1761 RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
1762
1763 PGM_LOCK_ASSERT_OWNER(pVM);
1764
1765# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1766 || PGM_GST_TYPE == PGM_TYPE_PAE \
1767 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
1768 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
1769
1770 /*
1771 * Assert preconditions.
1772 */
1773 Assert(PdeSrc.n.u1Present);
1774 Assert(cPages);
1775# if 0 /* rarely useful; leave for debugging. */
1776 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1777# endif
1778
1779 /*
1780 * Get the shadow PDE, find the shadow page table in the pool.
1781 */
1782# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1783 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1784 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1785
1786 /* Fetch the pgm pool shadow descriptor. */
1787 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1788 Assert(pShwPde);
1789
1790# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1791 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1792 PPGMPOOLPAGE pShwPde = NULL;
1793 PX86PDPAE pPDDst;
1794
1795 /* Fetch the pgm pool shadow descriptor. */
1796 int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1797 AssertRCSuccessReturn(rc2, rc2);
1798 Assert(pShwPde);
1799
1800 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1801 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1802
1803# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1804 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1805 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1806 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1807 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1808
1809 int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1810 AssertRCSuccessReturn(rc2, rc2);
1811 Assert(pPDDst && pPdptDst);
1812 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1813# endif
1814 SHWPDE PdeDst = *pPdeDst;
1815
1816 /*
1817 * - In the guest SMP case we could have blocked while another VCPU reused
1818 * this page table.
1819 * - With W7-64 we may also take this path when the A bit is cleared on
1820 * higher level tables (PDPE/PML4E). The guest does not invalidate the
1821 * relevant TLB entries. If we're write monitoring any page mapped by
1822 * the modified entry, we may end up here with a "stale" TLB entry.
1823 */
1824 if (!PdeDst.n.u1Present)
1825 {
1826 Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1827 AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW),
1828 ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1829 if (uErr & X86_TRAP_PF_P)
1830 PGM_INVL_PG(pVCpu, GCPtrPage);
1831 return VINF_SUCCESS; /* force the instruction to be executed again. */
1832 }
1833
1834 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1835 Assert(pShwPage);
1836
1837# if PGM_GST_TYPE == PGM_TYPE_AMD64
1838 /* Fetch the pgm pool shadow descriptor. */
1839 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1840 Assert(pShwPde);
1841# endif
1842
1843 /*
1844 * Check that the page is present and that the shadow PDE isn't out of sync.
1845 */
1846 const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1847 const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1848 RTGCPHYS GCPhys;
1849 if (!fBigPage)
1850 {
1851 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1852# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1853 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1854 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
1855# endif
1856 }
1857 else
1858 {
1859 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1860# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1861 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1862 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1863# endif
1864 }
1865 /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
1866 if ( fPdeValid
1867 && pShwPage->GCPhys == GCPhys
1868 && PdeSrc.n.u1Present
1869 && PdeSrc.n.u1User == PdeDst.n.u1User
1870 && (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
1871# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1872 && (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute || !GST_IS_NX_ACTIVE(pVCpu))
1873# endif
1874 )
1875 {
1876 /*
1877 * Check that the PDE is marked accessed already.
1878 * Since we set the accessed bit *before* getting here on a #PF, this
1879 * check is only meant for dealing with non-#PF'ing paths.
1880 */
1881 if (PdeSrc.n.u1Accessed)
1882 {
1883 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1884 if (!fBigPage)
1885 {
1886 /*
1887 * 4KB Page - Map the guest page table.
1888 */
1889 PGSTPT pPTSrc;
1890 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1891 if (RT_SUCCESS(rc))
1892 {
1893# ifdef PGM_SYNC_N_PAGES
1894 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1895 if ( cPages > 1
1896 && !(uErr & X86_TRAP_PF_P)
1897 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
1898 {
1899 /*
1900 * This code path is currently only taken when the caller is PGMTrap0eHandler
1901 * for non-present pages!
1902 *
1903 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1904 * deal with locality.
1905 */
1906 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1907# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1908 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1909 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1910# else
1911 const unsigned offPTSrc = 0;
1912# endif
1913 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1914 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1915 iPTDst = 0;
1916 else
1917 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1918
1919 for (; iPTDst < iPTDstEnd; iPTDst++)
1920 {
1921 const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
1922
1923 if ( pPteSrc->n.u1Present
1924 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
1925 {
1926 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1927 NOREF(GCPtrCurPage);
1928 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
1929 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1930 GCPtrCurPage, pPteSrc->n.u1Present,
1931 pPteSrc->n.u1Write & PdeSrc.n.u1Write,
1932 pPteSrc->n.u1User & PdeSrc.n.u1User,
1933 (uint64_t)pPteSrc->u,
1934 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1935 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1936 }
1937 }
1938 }
1939 else
1940# endif /* PGM_SYNC_N_PAGES */
1941 {
1942 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1943 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1944 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1945 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1946 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1947 GCPtrPage, PteSrc.n.u1Present,
1948 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1949 PteSrc.n.u1User & PdeSrc.n.u1User,
1950 (uint64_t)PteSrc.u,
1951 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1952 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1953 }
1954 }
1955 else /* MMIO or invalid page: emulated in #PF handler. */
1956 {
1957 LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1958 Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
1959 }
1960 }
1961 else
1962 {
1963 /*
1964 * 4/2MB page - lazy syncing shadow 4K pages.
1965 * (There are many causes of getting here, it's no longer only CSAM.)
1966 */
1967 /* Calculate the GC physical address of this 4KB shadow page. */
1968 GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
1969 /* Find ram range. */
1970 PPGMPAGE pPage;
1971 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1972 if (RT_SUCCESS(rc))
1973 {
1974 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1975
1976# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1977 /* Try to make the page writable if necessary. */
1978 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1979 && ( PGM_PAGE_IS_ZERO(pPage)
1980 || ( PdeSrc.n.u1Write
1981 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1982# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1983 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1984# endif
1985# ifdef VBOX_WITH_PAGE_SHARING
1986 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1987# endif
1988 )
1989 )
1990 )
1991 {
1992 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1993 AssertRC(rc);
1994 }
1995# endif
1996
1997 /*
1998 * Make shadow PTE entry.
1999 */
2000 SHWPTE PteDst;
2001 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2002 PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2003 else
2004 SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
2005
2006 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2007 if ( SHW_PTE_IS_P(PteDst)
2008 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2009 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2010
2011 /* Make sure only allocated pages are mapped writable. */
2012 if ( SHW_PTE_IS_P_RW(PteDst)
2013 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2014 {
2015 /* Still applies to shared pages. */
2016 Assert(!PGM_PAGE_IS_ZERO(pPage));
2017 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2018 Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2019 }
2020
2021 SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2022
2023 /*
2024 * If the page is not flagged as dirty and is writable, then make it read-only
2025 * at PD level, so we can set the dirty bit when the page is modified.
2026 *
2027 * ASSUMES that page access handlers are implemented on page table entry level.
2028 * Thus we will first catch the dirty access and set PDE.D and restart. If
2029 * there is an access handler, we'll trap again and let it work on the problem.
2030 */
2031 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2032 * As for invlpg, it simply frees the whole shadow PT.
2033 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2034 if ( !PdeSrc.b.u1Dirty
2035 && PdeSrc.b.u1Write)
2036 {
2037 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2038 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2039 PdeDst.n.u1Write = 0;
2040 }
2041 else
2042 {
2043 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2044 PdeDst.n.u1Write = PdeSrc.n.u1Write;
2045 }
2046 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2047 Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2048 GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2049 PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2050 }
2051 else
2052 {
2053 LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2054 /** @todo must wipe the shadow page table entry in this
2055 * case. */
2056 }
2057 }
2058 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2059 return VINF_SUCCESS;
2060 }
2061
2062 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2063 }
2064 else if (fPdeValid)
2065 {
2066 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2067 Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2068 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2069 }
2070 else
2071 {
2072/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2073 Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2074 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2075 }
2076
2077 /*
2078 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2079 * Yea, I'm lazy.
2080 */
2081 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2082 SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
2083
2084 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2085 PGM_INVL_VCPU_TLBS(pVCpu);
2086 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2087
2088
2089# elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2090 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2091 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
2092 NOREF(PdeSrc);
2093
2094# ifdef PGM_SYNC_N_PAGES
2095 /*
2096 * Get the shadow PDE, find the shadow page table in the pool.
2097 */
2098# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2099 X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2100
2101# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2102 X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2103
2104# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2105 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2106 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2107 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2108 X86PDEPAE PdeDst;
2109 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2110
2111 int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2112 AssertRCSuccessReturn(rc, rc);
2113 Assert(pPDDst && pPdptDst);
2114 PdeDst = pPDDst->a[iPDDst];
2115# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2116 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2117 PEPTPD pPDDst;
2118 EPTPDE PdeDst;
2119
2120 int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2121 if (rc != VINF_SUCCESS)
2122 {
2123 AssertRC(rc);
2124 return rc;
2125 }
2126 Assert(pPDDst);
2127 PdeDst = pPDDst->a[iPDDst];
2128# endif
2129 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2130 if (!PdeDst.n.u1Present)
2131 {
2132 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2133 Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2134 return VINF_SUCCESS; /* force the instruction to be executed again. */
2135 }
2136
2137 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2138 if (PdeDst.n.u1Size)
2139 {
2140 Assert(pVM->pgm.s.fNestedPaging);
2141 Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2142 return VINF_SUCCESS;
2143 }
2144
2145 /* Mask away the page offset. */
2146 GCPtrPage &= ~((RTGCPTR)0xfff);
2147
2148 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2149 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2150
2151 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2152 if ( cPages > 1
2153 && !(uErr & X86_TRAP_PF_P)
2154 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2155 {
2156 /*
2157 * This code path is currently only taken when the caller is PGMTrap0eHandler
2158 * for non-present pages!
2159 *
2160 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2161 * deal with locality.
2162 */
2163 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2164 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2165 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2166 iPTDst = 0;
2167 else
2168 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2169 for (; iPTDst < iPTDstEnd; iPTDst++)
2170 {
2171 if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2172 {
2173 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2174 | (iPTDst << PAGE_SHIFT));
2175
2176 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2177 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2178 GCPtrCurPage,
2179 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2180 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2181
2182 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2183 break;
2184 }
2185 else
2186 Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2187 }
2188 }
2189 else
2190# endif /* PGM_SYNC_N_PAGES */
2191 {
2192 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2193 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2194 | (iPTDst << PAGE_SHIFT));
2195
2196 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2197
2198 Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2199 GCPtrPage,
2200 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2201 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2202 }
2203 return VINF_SUCCESS;
2204
2205# else
2206 NOREF(PdeSrc);
2207 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2208 return VERR_PGM_NOT_USED_IN_MODE;
2209# endif
2210}
2211
2212#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
2213#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
2214
2215/**
2216 * CheckPageFault helper for returning a page fault indicating a non-present
2217 * (NP) entry in the page translation structures.
2218 *
2219 * @returns VINF_EM_RAW_GUEST_TRAP.
2220 * @param pVCpu The cross context virtual CPU structure.
2221 * @param uErr The error code of the shadow fault. Corrections to
2222 * TRPM's copy will be made if necessary.
2223 * @param GCPtrPage For logging.
2224 * @param uPageFaultLevel For logging.
2225 */
2226DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2227{
2228 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2229 AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2230 AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2231 if (uErr & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
2232 TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD | X86_TRAP_PF_P));
2233
2234 Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2235 RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2236 return VINF_EM_RAW_GUEST_TRAP;
2237}
2238
2239
2240/**
2241 * CheckPageFault helper for returning a page fault indicating a reserved bit
2242 * (RSVD) error in the page translation structures.
2243 *
2244 * @returns VINF_EM_RAW_GUEST_TRAP.
2245 * @param pVCpu The cross context virtual CPU structure.
2246 * @param uErr The error code of the shadow fault. Corrections to
2247 * TRPM's copy will be made if necessary.
2248 * @param GCPtrPage For logging.
2249 * @param uPageFaultLevel For logging.
2250 */
2251DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2252{
2253 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2254 if ((uErr & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
2255 TRPMSetErrorCode(pVCpu, uErr | X86_TRAP_PF_RSVD | X86_TRAP_PF_P);
2256
2257 Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2258 RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2259 return VINF_EM_RAW_GUEST_TRAP;
2260}
2261
2262
2263/**
2264 * CheckPageFault helper for returning a page protection fault (P).
2265 *
2266 * @returns VINF_EM_RAW_GUEST_TRAP.
2267 * @param pVCpu The cross context virtual CPU structure.
2268 * @param uErr The error code of the shadow fault. Corrections to
2269 * TRPM's copy will be made if necessary.
2270 * @param GCPtrPage For logging.
2271 * @param uPageFaultLevel For logging.
2272 */
2273DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2274{
2275 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2276 AssertMsg(uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID), ("%#x\n", uErr));
2277 if ((uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2278 TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) | X86_TRAP_PF_P);
2279
2280 Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2281 RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2282 return VINF_EM_RAW_GUEST_TRAP;
2283}
2284
2285
2286/**
2287 * Handle dirty bit tracking faults.
2288 *
2289 * @returns VBox status code.
2290 * @param pVCpu The cross context virtual CPU structure.
2291 * @param uErr Page fault error code.
2292 * @param pPdeSrc Guest page directory entry.
2293 * @param pPdeDst Shadow page directory entry.
2294 * @param GCPtrPage Guest context page address.
2295 */
2296static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2297 RTGCPTR GCPtrPage)
2298{
2299 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2300 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2301 NOREF(uErr);
2302
2303 PGM_LOCK_ASSERT_OWNER(pVM);
2304
2305 /*
2306 * Handle big page.
2307 */
2308 if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2309 {
2310 if ( pPdeDst->n.u1Present
2311 && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2312 {
2313 SHWPDE PdeDst = *pPdeDst;
2314
2315 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2316 Assert(pPdeSrc->b.u1Write);
2317
2318 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2319 * fault again and take this path to only invalidate the entry (see below).
2320 */
2321 PdeDst.n.u1Write = 1;
2322 PdeDst.n.u1Accessed = 1;
2323 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2324 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2325 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2326 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2327 }
2328
2329# ifdef IN_RING0
2330 /* Check for stale TLB entry; only applies to the SMP guest case. */
2331 if ( pVM->cCpus > 1
2332 && pPdeDst->n.u1Write
2333 && pPdeDst->n.u1Accessed)
2334 {
2335 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2336 if (pShwPage)
2337 {
2338 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2339 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2340 if (SHW_PTE_IS_P_RW(*pPteDst))
2341 {
2342 /* Stale TLB entry. */
2343 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2344 PGM_INVL_PG(pVCpu, GCPtrPage);
2345 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2346 }
2347 }
2348 }
2349# endif /* IN_RING0 */
2350 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2351 }
2352
2353 /*
2354 * Map the guest page table.
2355 */
2356 PGSTPT pPTSrc;
2357 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2358 if (RT_FAILURE(rc))
2359 {
2360 AssertRC(rc);
2361 return rc;
2362 }
2363
2364 if (pPdeDst->n.u1Present)
2365 {
2366 GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2367 const GSTPTE PteSrc = *pPteSrc;
2368
2369 /*
2370 * Map shadow page table.
2371 */
2372 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2373 if (pShwPage)
2374 {
2375 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2376 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2377 if (SHW_PTE_IS_P(*pPteDst)) /** @todo Optimize accessed bit emulation? */
2378 {
2379 if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2380 {
2381 PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2382 SHWPTE PteDst = *pPteDst;
2383
2384 LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2385 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2386
2387 Assert(PteSrc.n.u1Write);
2388
2389 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2390 * entry will not harm; write access will simply fault again and
2391 * take this path to only invalidate the entry.
2392 */
2393 if (RT_LIKELY(pPage))
2394 {
2395 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2396 {
2397 //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2398 Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2399 /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2400 SHW_PTE_SET_RO(PteDst);
2401 }
2402 else
2403 {
2404 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2405 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2406 {
2407 rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2408 AssertRC(rc);
2409 }
2410 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2411 SHW_PTE_SET_RW(PteDst);
2412 else
2413 {
2414 /* Still applies to shared pages. */
2415 Assert(!PGM_PAGE_IS_ZERO(pPage));
2416 SHW_PTE_SET_RO(PteDst);
2417 }
2418 }
2419 }
2420 else
2421 SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2422
2423 SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2424 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2425 PGM_INVL_PG(pVCpu, GCPtrPage);
2426 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2427 }
2428
2429# ifdef IN_RING0
2430 /* Check for stale TLB entry; only applies to the SMP guest case. */
2431 if ( pVM->cCpus > 1
2432 && SHW_PTE_IS_RW(*pPteDst)
2433 && SHW_PTE_IS_A(*pPteDst))
2434 {
2435 /* Stale TLB entry. */
2436 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2437 PGM_INVL_PG(pVCpu, GCPtrPage);
2438 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2439 }
2440# endif
2441 }
2442 }
2443 else
2444 AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2445 }
2446
2447 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2448}
2449
2450#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
2451
2452/**
2453 * Sync a shadow page table.
2454 *
2455 * The shadow page table is not present in the shadow PDE.
2456 *
2457 * Handles mapping conflicts.
2458 *
2459 * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2460 * conflict), and Trap0eHandler.
2461 *
2462 * A precondition for this method is that the shadow PDE is not present. The
2463 * caller must take the PGM lock before checking this and continue to hold it
2464 * when calling this method.
2465 *
2466 * @returns VBox status code.
2467 * @param pVCpu The cross context virtual CPU structure.
2468 * @param iPDSrc Page directory index.
2469 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
2470 * Assume this is a temporary mapping.
2471 * @param GCPtrPage GC Pointer of the page that caused the fault
2472 */
2473static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2474{
2475 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2476 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2477
2478#if 0 /* rarely useful; leave for debugging. */
2479 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2480#endif
2481 LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2482
2483 PGM_LOCK_ASSERT_OWNER(pVM);
2484
2485#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2486 || PGM_GST_TYPE == PGM_TYPE_PAE \
2487 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2488 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
2489 && PGM_SHW_TYPE != PGM_TYPE_NONE
2490 int rc = VINF_SUCCESS;
2491
2492 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2493
2494 /*
2495 * Some input validation first.
2496 */
2497 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2498
2499 /*
2500 * Get the relevant shadow PDE entry.
2501 */
2502# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2503 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2504 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2505
2506 /* Fetch the pgm pool shadow descriptor. */
2507 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2508 Assert(pShwPde);
2509
2510# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2511 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2512 PPGMPOOLPAGE pShwPde = NULL;
2513 PX86PDPAE pPDDst;
2514 PSHWPDE pPdeDst;
2515
2516 /* Fetch the pgm pool shadow descriptor. */
2517 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2518 AssertRCSuccessReturn(rc, rc);
2519 Assert(pShwPde);
2520
2521 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2522 pPdeDst = &pPDDst->a[iPDDst];
2523
2524# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2525 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2526 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2527 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2528 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2529 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2530 AssertRCSuccessReturn(rc, rc);
2531 Assert(pPDDst);
2532 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2533# endif
2534 SHWPDE PdeDst = *pPdeDst;
2535
2536# if PGM_GST_TYPE == PGM_TYPE_AMD64
2537 /* Fetch the pgm pool shadow descriptor. */
2538 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2539 Assert(pShwPde);
2540# endif
2541
2542# ifndef PGM_WITHOUT_MAPPINGS
2543 /*
2544 * Check for conflicts.
2545 * RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2546 * R3: Simply resolve the conflict.
2547 */
2548 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2549 {
2550 Assert(pgmMapAreMappingsEnabled(pVM));
2551# ifndef IN_RING3
2552 Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2553 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2554 return VERR_ADDRESS_CONFLICT;
2555
2556# else /* IN_RING3 */
2557 PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2558 Assert(pMapping);
2559# if PGM_GST_TYPE == PGM_TYPE_32BIT
2560 rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2561# elif PGM_GST_TYPE == PGM_TYPE_PAE
2562 rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2563# else
2564 AssertFailed(); NOREF(pMapping); /* can't happen for amd64 */
2565# endif
2566 if (RT_FAILURE(rc))
2567 {
2568 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2569 return rc;
2570 }
2571 PdeDst = *pPdeDst;
2572# endif /* IN_RING3 */
2573 }
2574# endif /* !PGM_WITHOUT_MAPPINGS */
2575 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2576
2577 /*
2578 * Sync the page directory entry.
2579 */
2580 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2581 const bool fPageTable = !PdeSrc.b.u1Size || !GST_IS_PSE_ACTIVE(pVCpu);
2582 if ( PdeSrc.n.u1Present
2583 && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2584 {
2585 /*
2586 * Allocate & map the page table.
2587 */
2588 PSHWPT pPTDst;
2589 PPGMPOOLPAGE pShwPage;
2590 RTGCPHYS GCPhys;
2591 if (fPageTable)
2592 {
2593 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2594# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2595 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2596 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
2597# endif
2598 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2599 pShwPde->idx, iPDDst, false /*fLockPage*/,
2600 &pShwPage);
2601 }
2602 else
2603 {
2604 PGMPOOLACCESS enmAccess;
2605# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2606 const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2607# else
2608 const bool fNoExecute = false;
2609# endif
2610
2611 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2612# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2613 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2614 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2615# endif
2616 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2617 if (PdeSrc.n.u1User)
2618 {
2619 if (PdeSrc.n.u1Write)
2620 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2621 else
2622 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2623 }
2624 else
2625 {
2626 if (PdeSrc.n.u1Write)
2627 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2628 else
2629 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2630 }
2631 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2632 pShwPde->idx, iPDDst, false /*fLockPage*/,
2633 &pShwPage);
2634 }
2635 if (rc == VINF_SUCCESS)
2636 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2637 else if (rc == VINF_PGM_CACHED_PAGE)
2638 {
2639 /*
2640 * The PT was cached, just hook it up.
2641 */
2642 if (fPageTable)
2643 PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2644 else
2645 {
2646 PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2647 /* (see explanation and assumptions further down.) */
2648 if ( !PdeSrc.b.u1Dirty
2649 && PdeSrc.b.u1Write)
2650 {
2651 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2652 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2653 PdeDst.b.u1Write = 0;
2654 }
2655 }
2656 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2657 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2658 return VINF_SUCCESS;
2659 }
2660 else
2661 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2662 /** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2663 * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2664 * irrelevant at this point. */
2665 PdeDst.u &= X86_PDE_AVL_MASK;
2666 PdeDst.u |= pShwPage->Core.Key;
2667
2668 /*
2669 * Page directory has been accessed (this is a fault situation, remember).
2670 */
2671 /** @todo
2672 * Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2673 * fault situation. What's more, the Trap0eHandler has already set the
2674 * accessed bit. So, it's actually just VerifyAccessSyncPage which
2675 * might need setting the accessed flag.
2676 *
2677 * The best idea is to leave this change to the caller and add an
2678 * assertion that it's set already. */
2679 pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2680 if (fPageTable)
2681 {
2682 /*
2683 * Page table - 4KB.
2684 *
2685 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2686 */
2687 Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2688 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2689 PGSTPT pPTSrc;
2690 rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2691 if (RT_SUCCESS(rc))
2692 {
2693 /*
2694 * Start by syncing the page directory entry so CSAM's TLB trick works.
2695 */
2696 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
2697 | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2698 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2699 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2700
2701 /*
2702 * Directory/page user or supervisor privilege: (same goes for read/write)
2703 *
2704 * Directory Page Combined
2705 * U/S U/S U/S
2706 * 0 0 0
2707 * 0 1 0
2708 * 1 0 0
2709 * 1 1 1
2710 *
2711 * Simple AND operation. Table listed for completeness.
2712 *
2713 */
2714 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2715# ifdef PGM_SYNC_N_PAGES
2716 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2717 unsigned iPTDst = iPTBase;
2718 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2719 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2720 iPTDst = 0;
2721 else
2722 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2723# else /* !PGM_SYNC_N_PAGES */
2724 unsigned iPTDst = 0;
2725 const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2726# endif /* !PGM_SYNC_N_PAGES */
2727 RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2728 | ((RTGCPTR)iPTDst << PAGE_SHIFT);
2729# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2730 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2731 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2732# else
2733 const unsigned offPTSrc = 0;
2734# endif
2735 for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
2736 {
2737 const unsigned iPTSrc = iPTDst + offPTSrc;
2738 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2739
2740 if (PteSrc.n.u1Present)
2741 {
2742 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2743 Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2744 GCPtrCur,
2745 PteSrc.n.u1Present,
2746 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2747 PteSrc.n.u1User & PdeSrc.n.u1User,
2748 (uint64_t)PteSrc.u,
2749 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2750 (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2751 }
2752 /* else: the page table was cleared by the pool */
2753 } /* for PTEs */
2754 }
2755 }
2756 else
2757 {
2758 /*
2759 * Big page - 2/4MB.
2760 *
2761 * We'll walk the ram range list in parallel and optimize lookups.
2762 * We will only sync one shadow page table at a time.
2763 */
2764 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2765
2766 /**
2767 * @todo It might be more efficient to sync only a part of the 4MB
2768 * page (similar to what we do for 4KB PDs).
2769 */
2770
2771 /*
2772 * Start by syncing the page directory entry.
2773 */
2774 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2775 | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2776
2777 /*
2778 * If the page is not flagged as dirty and is writable, then make it read-only
2779 * at PD level, so we can set the dirty bit when the page is modified.
2780 *
2781 * ASSUMES that page access handlers are implemented on page table entry level.
2782 * Thus we will first catch the dirty access and set PDE.D and restart. If
2783 * there is an access handler, we'll trap again and let it work on the problem.
2784 */
2785 /** @todo move the above stuff to a section in the PGM documentation. */
2786 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2787 if ( !PdeSrc.b.u1Dirty
2788 && PdeSrc.b.u1Write)
2789 {
2790 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2791 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2792 PdeDst.b.u1Write = 0;
2793 }
2794 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2795 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2796
2797 /*
2798 * Fill the shadow page table.
2799 */
2800 /* Get address and flags from the source PDE. */
2801 SHWPTE PteDstBase;
2802 SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
2803
2804 /* Loop thru the entries in the shadow PT. */
2805 const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2806 Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2807 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2808 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2809 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2810 unsigned iPTDst = 0;
2811 while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2812 && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2813 {
2814 if (pRam && GCPhys >= pRam->GCPhys)
2815 {
2816# ifndef PGM_WITH_A20
2817 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2818# endif
2819 do
2820 {
2821 /* Make shadow PTE. */
2822# ifdef PGM_WITH_A20
2823 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2824# else
2825 PPGMPAGE pPage = &pRam->aPages[iHCPage];
2826# endif
2827 SHWPTE PteDst;
2828
2829# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2830 /* Try to make the page writable if necessary. */
2831 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2832 && ( PGM_PAGE_IS_ZERO(pPage)
2833 || ( SHW_PTE_IS_RW(PteDstBase)
2834 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2835# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2836 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2837# endif
2838# ifdef VBOX_WITH_PAGE_SHARING
2839 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2840# endif
2841 && !PGM_PAGE_IS_BALLOONED(pPage))
2842 )
2843 )
2844 {
2845 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2846 AssertRCReturn(rc, rc);
2847 if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2848 break;
2849 }
2850# endif
2851
2852 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2853 PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
2854 else if (PGM_PAGE_IS_BALLOONED(pPage))
2855 SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
2856 else
2857 SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase));
2858
2859 /* Only map writable pages writable. */
2860 if ( SHW_PTE_IS_P_RW(PteDst)
2861 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2862 {
2863 /* Still applies to shared pages. */
2864 Assert(!PGM_PAGE_IS_ZERO(pPage));
2865 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2866 Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
2867 }
2868
2869 if (SHW_PTE_IS_P(PteDst))
2870 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2871
2872 /* commit it (not atomic, new table) */
2873 pPTDst->a[iPTDst] = PteDst;
2874 Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2875 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
2876 SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
2877
2878 /* advance */
2879 GCPhys += PAGE_SIZE;
2880 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
2881# ifndef PGM_WITH_A20
2882 iHCPage++;
2883# endif
2884 iPTDst++;
2885 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2886 && GCPhys <= pRam->GCPhysLast);
2887
2888 /* Advance ram range list. */
2889 while (pRam && GCPhys > pRam->GCPhysLast)
2890 pRam = pRam->CTX_SUFF(pNext);
2891 }
2892 else if (pRam)
2893 {
2894 Log(("Invalid pages at %RGp\n", GCPhys));
2895 do
2896 {
2897 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2898 GCPhys += PAGE_SIZE;
2899 iPTDst++;
2900 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2901 && GCPhys < pRam->GCPhys);
2902 PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
2903 }
2904 else
2905 {
2906 Log(("Invalid pages at %RGp (2)\n", GCPhys));
2907 for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2908 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2909 }
2910 } /* while more PTEs */
2911 } /* 4KB / 4MB */
2912 }
2913 else
2914 AssertRelease(!PdeDst.n.u1Present);
2915
2916 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2917 if (RT_FAILURE(rc))
2918 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
2919 return rc;
2920
2921#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2922 && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2923 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
2924 && PGM_SHW_TYPE != PGM_TYPE_NONE
2925 NOREF(iPDSrc); NOREF(pPDSrc);
2926
2927 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2928
2929 /*
2930 * Validate input a little bit.
2931 */
2932 int rc = VINF_SUCCESS;
2933# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2934 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2935 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2936
2937 /* Fetch the pgm pool shadow descriptor. */
2938 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2939 Assert(pShwPde);
2940
2941# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2942 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2943 PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2944 PX86PDPAE pPDDst;
2945 PSHWPDE pPdeDst;
2946
2947 /* Fetch the pgm pool shadow descriptor. */
2948 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2949 AssertRCSuccessReturn(rc, rc);
2950 Assert(pShwPde);
2951
2952 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2953 pPdeDst = &pPDDst->a[iPDDst];
2954
2955# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2956 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2957 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2958 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2959 PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
2960 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2961 AssertRCSuccessReturn(rc, rc);
2962 Assert(pPDDst);
2963 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2964
2965 /* Fetch the pgm pool shadow descriptor. */
2966 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2967 Assert(pShwPde);
2968
2969# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2970 const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2971 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2972 PEPTPD pPDDst;
2973 PEPTPDPT pPdptDst;
2974
2975 rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2976 if (rc != VINF_SUCCESS)
2977 {
2978 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2979 AssertRC(rc);
2980 return rc;
2981 }
2982 Assert(pPDDst);
2983 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2984
2985 /* Fetch the pgm pool shadow descriptor. */
2986 /** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
2987 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2988 Assert(pShwPde);
2989# endif
2990 SHWPDE PdeDst = *pPdeDst;
2991
2992# ifndef PGM_WITHOUT_MAPPINGS
2993 Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2994# endif
2995 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2996
2997# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2998 if ( BTH_IS_NP_ACTIVE(pVM)
2999 && !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Large page support. */
3000 {
3001 /* Check if we allocated a big page before for this 2 MB range. */
3002 PPGMPAGE pPage;
3003 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3004 if (RT_SUCCESS(rc))
3005 {
3006 RTHCPHYS HCPhys = NIL_RTHCPHYS;
3007 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3008 {
3009 if (PGM_A20_IS_ENABLED(pVCpu))
3010 {
3011 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3012 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3013 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3014 }
3015 else
3016 {
3017 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3018 pVM->pgm.s.cLargePagesDisabled++;
3019 }
3020 }
3021 else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3022 && PGM_A20_IS_ENABLED(pVCpu))
3023 {
3024 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3025 rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3026 if (RT_SUCCESS(rc))
3027 {
3028 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3029 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3030 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3031 }
3032 }
3033 else if ( PGMIsUsingLargePages(pVM)
3034 && PGM_A20_IS_ENABLED(pVCpu))
3035 {
3036 rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3037 if (RT_SUCCESS(rc))
3038 {
3039 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3040 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3041 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3042 }
3043 else
3044 LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3045 }
3046
3047 if (HCPhys != NIL_RTHCPHYS)
3048 {
3049# if PGM_SHW_TYPE == PGM_TYPE_EPT
3050 PdeDst.u = HCPhys | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_LEAF | EPT_E_IGNORE_PAT | EPT_E_TYPE_WB
3051 | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
3052# else
3053 PdeDst.u &= X86_PDE_AVL_MASK;
3054 PdeDst.n.u1Present = 1;
3055 PdeDst.n.u1Write = 1;
3056 PdeDst.b.u1Size = 1;
3057 PdeDst.n.u1User = 1;
3058 PdeDst.u |= HCPhys; /* Note! Must be done last of gcc v10.2.1 20200723 (Red Hat 10.2.1-1) may drop the top 32 bits. */
3059# endif
3060 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3061
3062 Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3063 /* Add a reference to the first page only. */
3064 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3065
3066 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3067 return VINF_SUCCESS;
3068 }
3069 }
3070 }
3071# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
3072
3073 /*
3074 * Allocate & map the page table.
3075 */
3076 PSHWPT pPTDst;
3077 PPGMPOOLPAGE pShwPage;
3078 RTGCPHYS GCPhys;
3079
3080 /* Virtual address = physical address */
3081 GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3082 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3083 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/,
3084 &pShwPage);
3085 if ( rc == VINF_SUCCESS
3086 || rc == VINF_PGM_CACHED_PAGE)
3087 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3088 else
3089 {
3090 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3091 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3092 }
3093
3094 if (rc == VINF_SUCCESS)
3095 {
3096 /* New page table; fully set it up. */
3097 Assert(pPTDst);
3098
3099 /* Mask away the page offset. */
3100 GCPtrPage &= ~(RTGCPTR)PAGE_OFFSET_MASK;
3101
3102 for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3103 {
3104 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3105 | (iPTDst << PAGE_SHIFT));
3106
3107 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3108 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3109 GCPtrCurPage,
3110 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3111 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3112
3113 if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3114 break;
3115 }
3116 }
3117 else
3118 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3119
3120 /* Save the new PDE. */
3121# if PGM_SHW_TYPE == PGM_TYPE_EPT
3122 PdeDst.u = pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE
3123 | (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
3124# else
3125 PdeDst.u &= X86_PDE_AVL_MASK;
3126 PdeDst.n.u1Present = 1;
3127 PdeDst.n.u1Write = 1;
3128 PdeDst.n.u1User = 1;
3129 PdeDst.n.u1Accessed = 1;
3130 PdeDst.u |= pShwPage->Core.Key; /* Note! Must be done last of gcc v10.2.1 20200723 (Red Hat 10.2.1-1) drops the top 32 bits. */
3131 /** @todo r=bird: Stop using bitfields. But we need to defined/find the EPT flags then. */
3132# endif
3133 SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3134
3135 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3136 if (RT_FAILURE(rc))
3137 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3138 return rc;
3139
3140#else
3141 NOREF(iPDSrc); NOREF(pPDSrc);
3142 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3143 return VERR_PGM_NOT_USED_IN_MODE;
3144#endif
3145}
3146
3147
3148
3149/**
3150 * Prefetch a page/set of pages.
3151 *
3152 * Typically used to sync commonly used pages before entering raw mode
3153 * after a CR3 reload.
3154 *
3155 * @returns VBox status code.
3156 * @param pVCpu The cross context virtual CPU structure.
3157 * @param GCPtrPage Page to invalidate.
3158 */
3159PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
3160{
3161#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3162 || PGM_GST_TYPE == PGM_TYPE_REAL \
3163 || PGM_GST_TYPE == PGM_TYPE_PROT \
3164 || PGM_GST_TYPE == PGM_TYPE_PAE \
3165 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3166 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3167 && PGM_SHW_TYPE != PGM_TYPE_NONE
3168 /*
3169 * Check that all Guest levels thru the PDE are present, getting the
3170 * PD and PDE in the processes.
3171 */
3172 int rc = VINF_SUCCESS;
3173# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3174# if PGM_GST_TYPE == PGM_TYPE_32BIT
3175 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3176 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3177# elif PGM_GST_TYPE == PGM_TYPE_PAE
3178 unsigned iPDSrc;
3179 X86PDPE PdpeSrc;
3180 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3181 if (!pPDSrc)
3182 return VINF_SUCCESS; /* not present */
3183# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3184 unsigned iPDSrc;
3185 PX86PML4E pPml4eSrc;
3186 X86PDPE PdpeSrc;
3187 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3188 if (!pPDSrc)
3189 return VINF_SUCCESS; /* not present */
3190# endif
3191 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3192# else
3193 PGSTPD pPDSrc = NULL;
3194 const unsigned iPDSrc = 0;
3195 GSTPDE PdeSrc;
3196
3197 PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3198 PdeSrc.n.u1Present = 1;
3199 PdeSrc.n.u1Write = 1;
3200 PdeSrc.n.u1Accessed = 1;
3201 PdeSrc.n.u1User = 1;
3202# endif
3203
3204 if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3205 {
3206 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3207 pgmLock(pVM);
3208
3209# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3210 const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3211# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3212 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3213 PX86PDPAE pPDDst;
3214 X86PDEPAE PdeDst;
3215# if PGM_GST_TYPE != PGM_TYPE_PAE
3216 X86PDPE PdpeSrc;
3217
3218 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3219 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3220# endif
3221 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3222 if (rc != VINF_SUCCESS)
3223 {
3224 pgmUnlock(pVM);
3225 AssertRC(rc);
3226 return rc;
3227 }
3228 Assert(pPDDst);
3229 PdeDst = pPDDst->a[iPDDst];
3230
3231# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3232 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3233 PX86PDPAE pPDDst;
3234 X86PDEPAE PdeDst;
3235
3236# if PGM_GST_TYPE == PGM_TYPE_PROT
3237 /* AMD-V nested paging */
3238 X86PML4E Pml4eSrc;
3239 X86PDPE PdpeSrc;
3240 PX86PML4E pPml4eSrc = &Pml4eSrc;
3241
3242 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3243 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
3244 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
3245# endif
3246
3247 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3248 if (rc != VINF_SUCCESS)
3249 {
3250 pgmUnlock(pVM);
3251 AssertRC(rc);
3252 return rc;
3253 }
3254 Assert(pPDDst);
3255 PdeDst = pPDDst->a[iPDDst];
3256# endif
3257# ifndef PGM_WITHOUT_MAPPINGS
3258 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3259# endif
3260 {
3261 if (!PdeDst.n.u1Present)
3262 {
3263 /** @todo r=bird: This guy will set the A bit on the PDE,
3264 * probably harmless. */
3265 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3266 }
3267 else
3268 {
3269 /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3270 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3271 * makes no sense to prefetch more than one page.
3272 */
3273 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3274 if (RT_SUCCESS(rc))
3275 rc = VINF_SUCCESS;
3276 }
3277 }
3278 pgmUnlock(pVM);
3279 }
3280 return rc;
3281
3282#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
3283 NOREF(pVCpu); NOREF(GCPtrPage);
3284 return VINF_SUCCESS; /* ignore */
3285#else
3286 AssertCompile(0);
3287#endif
3288}
3289
3290
3291
3292
3293/**
3294 * Syncs a page during a PGMVerifyAccess() call.
3295 *
3296 * @returns VBox status code (informational included).
3297 * @param pVCpu The cross context virtual CPU structure.
3298 * @param GCPtrPage The address of the page to sync.
3299 * @param fPage The effective guest page flags.
3300 * @param uErr The trap error code.
3301 * @remarks This will normally never be called on invalid guest page
3302 * translation entries.
3303 */
3304PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3305{
3306 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3307
3308 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3309 RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
3310
3311 Assert(!pVM->pgm.s.fNestedPaging);
3312#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3313 || PGM_GST_TYPE == PGM_TYPE_REAL \
3314 || PGM_GST_TYPE == PGM_TYPE_PROT \
3315 || PGM_GST_TYPE == PGM_TYPE_PAE \
3316 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3317 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3318 && PGM_SHW_TYPE != PGM_TYPE_NONE
3319
3320 /*
3321 * Get guest PD and index.
3322 */
3323 /** @todo Performance: We've done all this a jiffy ago in the
3324 * PGMGstGetPage call. */
3325# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3326# if PGM_GST_TYPE == PGM_TYPE_32BIT
3327 const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3328 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3329
3330# elif PGM_GST_TYPE == PGM_TYPE_PAE
3331 unsigned iPDSrc = 0;
3332 X86PDPE PdpeSrc;
3333 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3334 if (RT_UNLIKELY(!pPDSrc))
3335 {
3336 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3337 return VINF_EM_RAW_GUEST_TRAP;
3338 }
3339
3340# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3341 unsigned iPDSrc = 0; /* shut up gcc */
3342 PX86PML4E pPml4eSrc = NULL; /* ditto */
3343 X86PDPE PdpeSrc;
3344 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3345 if (RT_UNLIKELY(!pPDSrc))
3346 {
3347 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3348 return VINF_EM_RAW_GUEST_TRAP;
3349 }
3350# endif
3351
3352# else /* !PGM_WITH_PAGING */
3353 PGSTPD pPDSrc = NULL;
3354 const unsigned iPDSrc = 0;
3355# endif /* !PGM_WITH_PAGING */
3356 int rc = VINF_SUCCESS;
3357
3358 pgmLock(pVM);
3359
3360 /*
3361 * First check if the shadow pd is present.
3362 */
3363# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3364 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3365
3366# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3367 PX86PDEPAE pPdeDst;
3368 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3369 PX86PDPAE pPDDst;
3370# if PGM_GST_TYPE != PGM_TYPE_PAE
3371 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3372 X86PDPE PdpeSrc;
3373 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3374# endif
3375 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3376 if (rc != VINF_SUCCESS)
3377 {
3378 pgmUnlock(pVM);
3379 AssertRC(rc);
3380 return rc;
3381 }
3382 Assert(pPDDst);
3383 pPdeDst = &pPDDst->a[iPDDst];
3384
3385# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3386 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3387 PX86PDPAE pPDDst;
3388 PX86PDEPAE pPdeDst;
3389
3390# if PGM_GST_TYPE == PGM_TYPE_PROT
3391 /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3392 X86PML4E Pml4eSrc;
3393 X86PDPE PdpeSrc;
3394 PX86PML4E pPml4eSrc = &Pml4eSrc;
3395 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
3396 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
3397# endif
3398
3399 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3400 if (rc != VINF_SUCCESS)
3401 {
3402 pgmUnlock(pVM);
3403 AssertRC(rc);
3404 return rc;
3405 }
3406 Assert(pPDDst);
3407 pPdeDst = &pPDDst->a[iPDDst];
3408# endif
3409
3410 if (!pPdeDst->n.u1Present)
3411 {
3412 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3413 if (rc != VINF_SUCCESS)
3414 {
3415 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3416 pgmUnlock(pVM);
3417 AssertRC(rc);
3418 return rc;
3419 }
3420 }
3421
3422# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3423 /* Check for dirty bit fault */
3424 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3425 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3426 Log(("PGMVerifyAccess: success (dirty)\n"));
3427 else
3428# endif
3429 {
3430# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3431 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3432# else
3433 GSTPDE PdeSrc;
3434 PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3435 PdeSrc.n.u1Present = 1;
3436 PdeSrc.n.u1Write = 1;
3437 PdeSrc.n.u1Accessed = 1;
3438 PdeSrc.n.u1User = 1;
3439# endif
3440
3441 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3442 if (uErr & X86_TRAP_PF_US)
3443 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3444 else /* supervisor */
3445 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3446
3447 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3448 if (RT_SUCCESS(rc))
3449 {
3450 /* Page was successfully synced */
3451 Log2(("PGMVerifyAccess: success (sync)\n"));
3452 rc = VINF_SUCCESS;
3453 }
3454 else
3455 {
3456 Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3457 rc = VINF_EM_RAW_GUEST_TRAP;
3458 }
3459 }
3460 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3461 pgmUnlock(pVM);
3462 return rc;
3463
3464#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3465
3466 AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3467 return VERR_PGM_NOT_USED_IN_MODE;
3468#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3469}
3470
3471
3472/**
3473 * Syncs the paging hierarchy starting at CR3.
3474 *
3475 * @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
3476 * informational status codes.
3477 * @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
3478 * the VMM into guest context.
3479 * @param pVCpu The cross context virtual CPU structure.
3480 * @param cr0 Guest context CR0 register.
3481 * @param cr3 Guest context CR3 register. Not subjected to the A20
3482 * mask.
3483 * @param cr4 Guest context CR4 register.
3484 * @param fGlobal Including global page directories or not
3485 */
3486PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3487{
3488 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3489 NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3490
3491 LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3492
3493#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
3494# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3495 pgmLock(pVM);
3496 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3497 if (pPool->cDirtyPages)
3498 pgmPoolResetDirtyPages(pVM);
3499 pgmUnlock(pVM);
3500# endif
3501#endif /* !NESTED && !EPT */
3502
3503#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
3504 /*
3505 * Nested / EPT / None - No work.
3506 */
3507 Assert(!pgmMapAreMappingsEnabled(pVM));
3508 return VINF_SUCCESS;
3509
3510#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3511 /*
3512 * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3513 * out the shadow parts when the guest modifies its tables.
3514 */
3515 Assert(!pgmMapAreMappingsEnabled(pVM));
3516 return VINF_SUCCESS;
3517
3518#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3519
3520# ifndef PGM_WITHOUT_MAPPINGS
3521 /*
3522 * Check for and resolve conflicts with our guest mappings if they
3523 * are enabled and not fixed.
3524 */
3525 if (pgmMapAreMappingsFloating(pVM))
3526 {
3527 int rc = pgmMapResolveConflicts(pVM);
3528 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3);
3529 if (rc == VINF_SUCCESS)
3530 { /* likely */ }
3531 else if (rc == VINF_PGM_SYNC_CR3)
3532 {
3533 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3534 return VINF_PGM_SYNC_CR3;
3535 }
3536 else if (RT_FAILURE(rc))
3537 return rc;
3538 else
3539 AssertMsgFailed(("%Rrc\n", rc));
3540 }
3541# else
3542 Assert(!pgmMapAreMappingsEnabled(pVM));
3543# endif
3544 return VINF_SUCCESS;
3545#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3546}
3547
3548
3549
3550
3551#ifdef VBOX_STRICT
3552
3553/**
3554 * Checks that the shadow page table is in sync with the guest one.
3555 *
3556 * @returns The number of errors.
3557 * @param pVCpu The cross context virtual CPU structure.
3558 * @param cr3 Guest context CR3 register.
3559 * @param cr4 Guest context CR4 register.
3560 * @param GCPtr Where to start. Defaults to 0.
3561 * @param cb How much to check. Defaults to everything.
3562 */
3563PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3564{
3565 NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3566#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
3567 return 0;
3568#else
3569 unsigned cErrors = 0;
3570 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3571 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3572
3573# if PGM_GST_TYPE == PGM_TYPE_PAE
3574 /** @todo currently broken; crashes below somewhere */
3575 AssertFailed();
3576# endif
3577
3578# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3579 || PGM_GST_TYPE == PGM_TYPE_PAE \
3580 || PGM_GST_TYPE == PGM_TYPE_AMD64
3581
3582 bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3583 PPGMCPU pPGM = &pVCpu->pgm.s;
3584 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3585 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3586# ifndef IN_RING0
3587 RTHCPHYS HCPhys; /* general usage. */
3588# endif
3589 int rc;
3590
3591 /*
3592 * Check that the Guest CR3 and all its mappings are correct.
3593 */
3594 AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3595 ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3596 false);
3597# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3598# if 0
3599# if PGM_GST_TYPE == PGM_TYPE_32BIT
3600 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3601# else
3602 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3603# endif
3604 AssertRCReturn(rc, 1);
3605 HCPhys = NIL_RTHCPHYS;
3606 rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3607 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3608# endif
3609# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3610 pgmGstGet32bitPDPtr(pVCpu);
3611 RTGCPHYS GCPhys;
3612 rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3613 AssertRCReturn(rc, 1);
3614 AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3615# endif
3616# endif /* !IN_RING0 */
3617
3618 /*
3619 * Get and check the Shadow CR3.
3620 */
3621# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3622 unsigned cPDEs = X86_PG_ENTRIES;
3623 unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3624# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3625# if PGM_GST_TYPE == PGM_TYPE_32BIT
3626 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3627# else
3628 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3629# endif
3630 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3631# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3632 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3633 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3634# endif
3635 if (cb != ~(RTGCPTR)0)
3636 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3637
3638/** @todo call the other two PGMAssert*() functions. */
3639
3640# if PGM_GST_TYPE == PGM_TYPE_AMD64
3641 unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3642
3643 for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3644 {
3645 PPGMPOOLPAGE pShwPdpt = NULL;
3646 PX86PML4E pPml4eSrc;
3647 PX86PML4E pPml4eDst;
3648 RTGCPHYS GCPhysPdptSrc;
3649
3650 pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3651 pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3652
3653 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3654 if (!pPml4eDst->n.u1Present)
3655 {
3656 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3657 continue;
3658 }
3659
3660 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3661 GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
3662
3663 if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3664 {
3665 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3666 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3667 cErrors++;
3668 continue;
3669 }
3670
3671 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3672 {
3673 AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3674 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3675 cErrors++;
3676 continue;
3677 }
3678
3679 if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3680 || pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3681 || pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3682 {
3683 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3684 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3685 cErrors++;
3686 continue;
3687 }
3688# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3689 {
3690# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3691
3692# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
3693 /*
3694 * Check the PDPTEs too.
3695 */
3696 unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3697
3698 for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3699 {
3700 unsigned iPDSrc = 0; /* initialized to shut up gcc */
3701 PPGMPOOLPAGE pShwPde = NULL;
3702 PX86PDPE pPdpeDst;
3703 RTGCPHYS GCPhysPdeSrc;
3704 X86PDPE PdpeSrc;
3705 PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
3706# if PGM_GST_TYPE == PGM_TYPE_PAE
3707 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3708 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3709# else
3710 PX86PML4E pPml4eSrcIgn;
3711 PX86PDPT pPdptDst;
3712 PX86PDPAE pPDDst;
3713 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3714
3715 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3716 if (rc != VINF_SUCCESS)
3717 {
3718 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3719 GCPtr += 512 * _2M;
3720 continue; /* next PDPTE */
3721 }
3722 Assert(pPDDst);
3723# endif
3724 Assert(iPDSrc == 0);
3725
3726 pPdpeDst = &pPdptDst->a[iPdpt];
3727
3728 if (!pPdpeDst->n.u1Present)
3729 {
3730 GCPtr += 512 * _2M;
3731 continue; /* next PDPTE */
3732 }
3733
3734 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3735 GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
3736
3737 if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3738 {
3739 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3740 GCPtr += 512 * _2M;
3741 cErrors++;
3742 continue;
3743 }
3744
3745 if (GCPhysPdeSrc != pShwPde->GCPhys)
3746 {
3747# if PGM_GST_TYPE == PGM_TYPE_AMD64
3748 AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3749# else
3750 AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3751# endif
3752 GCPtr += 512 * _2M;
3753 cErrors++;
3754 continue;
3755 }
3756
3757# if PGM_GST_TYPE == PGM_TYPE_AMD64
3758 if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3759 || pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3760 || pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3761 {
3762 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3763 GCPtr += 512 * _2M;
3764 cErrors++;
3765 continue;
3766 }
3767# endif
3768
3769# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3770 {
3771# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3772# if PGM_GST_TYPE == PGM_TYPE_32BIT
3773 GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3774# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3775 PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3776# endif
3777# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3778 /*
3779 * Iterate the shadow page directory.
3780 */
3781 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3782 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3783
3784 for (;
3785 iPDDst < cPDEs;
3786 iPDDst++, GCPtr += cIncrement)
3787 {
3788# if PGM_SHW_TYPE == PGM_TYPE_PAE
3789 const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3790# else
3791 const SHWPDE PdeDst = pPDDst->a[iPDDst];
3792# endif
3793# ifndef PGM_WITHOUT_MAPPINGS
3794 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3795 {
3796 Assert(pgmMapAreMappingsEnabled(pVM));
3797 if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3798 {
3799 AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3800 cErrors++;
3801 continue;
3802 }
3803 }
3804 else
3805# endif
3806 if ( (PdeDst.u & X86_PDE_P)
3807 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) )
3808 {
3809 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3810 PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3811 if (!pPoolPage)
3812 {
3813 AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3814 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3815 cErrors++;
3816 continue;
3817 }
3818 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3819
3820 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
3821 {
3822 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3823 GCPtr, (uint64_t)PdeDst.u));
3824 cErrors++;
3825 }
3826
3827 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
3828 {
3829 AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3830 GCPtr, (uint64_t)PdeDst.u));
3831 cErrors++;
3832 }
3833
3834 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3835 if (!PdeSrc.n.u1Present)
3836 {
3837 AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3838 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3839 cErrors++;
3840 continue;
3841 }
3842
3843 if ( !PdeSrc.b.u1Size
3844 || !fBigPagesSupported)
3845 {
3846 GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
3847# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3848 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (PAGE_SIZE / 2)));
3849# endif
3850 }
3851 else
3852 {
3853# if PGM_GST_TYPE == PGM_TYPE_32BIT
3854 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3855 {
3856 AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3857 GCPtr, (uint64_t)PdeSrc.u));
3858 cErrors++;
3859 continue;
3860 }
3861# endif
3862 GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3863# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3864 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
3865# endif
3866 }
3867
3868 if ( pPoolPage->enmKind
3869 != (!PdeSrc.b.u1Size || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3870 {
3871 AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3872 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3873 cErrors++;
3874 }
3875
3876 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
3877 if (!pPhysPage)
3878 {
3879 AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3880 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3881 cErrors++;
3882 continue;
3883 }
3884
3885 if (GCPhysGst != pPoolPage->GCPhys)
3886 {
3887 AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3888 GCPhysGst, pPoolPage->GCPhys, GCPtr));
3889 cErrors++;
3890 continue;
3891 }
3892
3893 if ( !PdeSrc.b.u1Size
3894 || !fBigPagesSupported)
3895 {
3896 /*
3897 * Page Table.
3898 */
3899 const GSTPT *pPTSrc;
3900 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1)),
3901 &pPTSrc);
3902 if (RT_FAILURE(rc))
3903 {
3904 AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3905 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3906 cErrors++;
3907 continue;
3908 }
3909 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
3910 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
3911 {
3912 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3913 // (This problem will go away when/if we shadow multiple CR3s.)
3914 AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3915 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3916 cErrors++;
3917 continue;
3918 }
3919 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3920 {
3921 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3922 GCPtr, (uint64_t)PdeDst.u));
3923 cErrors++;
3924 continue;
3925 }
3926
3927 /* iterate the page table. */
3928# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3929 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3930 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3931# else
3932 const unsigned offPTSrc = 0;
3933# endif
3934 for (unsigned iPT = 0, off = 0;
3935 iPT < RT_ELEMENTS(pPTDst->a);
3936 iPT++, off += PAGE_SIZE)
3937 {
3938 const SHWPTE PteDst = pPTDst->a[iPT];
3939
3940 /* skip not-present and dirty tracked entries. */
3941 if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3942 continue;
3943 Assert(SHW_PTE_IS_P(PteDst));
3944
3945 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3946 if (!PteSrc.n.u1Present)
3947 {
3948# ifdef IN_RING3
3949 PGMAssertHandlerAndFlagsInSync(pVM);
3950 DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE
3951 | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3,
3952 0, 0, UINT64_MAX, 99, NULL);
3953# endif
3954 AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3955 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3956 (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
3957 cErrors++;
3958 continue;
3959 }
3960
3961 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
3962# if 1 /** @todo sync accessed bit properly... */
3963 fIgnoreFlags |= X86_PTE_A;
3964# endif
3965
3966 /* match the physical addresses */
3967 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
3968 GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
3969
3970# ifdef IN_RING3
3971 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3972 if (RT_FAILURE(rc))
3973 {
3974 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3975 {
3976 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3977 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3978 cErrors++;
3979 continue;
3980 }
3981 }
3982 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3983 {
3984 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3985 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3986 cErrors++;
3987 continue;
3988 }
3989# endif
3990
3991 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
3992 if (!pPhysPage)
3993 {
3994# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3995 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3996 {
3997 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3998 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3999 cErrors++;
4000 continue;
4001 }
4002# endif
4003 if (SHW_PTE_IS_RW(PteDst))
4004 {
4005 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4006 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4007 cErrors++;
4008 }
4009 fIgnoreFlags |= X86_PTE_RW;
4010 }
4011 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4012 {
4013 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4014 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4015 cErrors++;
4016 continue;
4017 }
4018
4019 /* flags */
4020 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4021 {
4022 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4023 {
4024 if (SHW_PTE_IS_RW(PteDst))
4025 {
4026 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4027 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4028 cErrors++;
4029 continue;
4030 }
4031 fIgnoreFlags |= X86_PTE_RW;
4032 }
4033 else
4034 {
4035 if ( SHW_PTE_IS_P(PteDst)
4036# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
4037 && !PGM_PAGE_IS_MMIO(pPhysPage)
4038# endif
4039 )
4040 {
4041 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4042 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4043 cErrors++;
4044 continue;
4045 }
4046 fIgnoreFlags |= X86_PTE_P;
4047 }
4048 }
4049 else
4050 {
4051 if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4052 {
4053 if (SHW_PTE_IS_RW(PteDst))
4054 {
4055 AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4056 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4057 cErrors++;
4058 continue;
4059 }
4060 if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4061 {
4062 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4063 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4064 cErrors++;
4065 continue;
4066 }
4067 if (SHW_PTE_IS_D(PteDst))
4068 {
4069 AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4070 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4071 cErrors++;
4072 }
4073# if 0 /** @todo sync access bit properly... */
4074 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4075 {
4076 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4077 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4078 cErrors++;
4079 }
4080 fIgnoreFlags |= X86_PTE_RW;
4081# else
4082 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4083# endif
4084 }
4085 else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4086 {
4087 /* access bit emulation (not implemented). */
4088 if (PteSrc.n.u1Accessed || SHW_PTE_IS_P(PteDst))
4089 {
4090 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4091 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4092 cErrors++;
4093 continue;
4094 }
4095 if (!SHW_PTE_IS_A(PteDst))
4096 {
4097 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4098 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4099 cErrors++;
4100 }
4101 fIgnoreFlags |= X86_PTE_P;
4102 }
4103# ifdef DEBUG_sandervl
4104 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4105# endif
4106 }
4107
4108 if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4109 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4110 )
4111 {
4112 AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4113 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4114 fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4115 cErrors++;
4116 continue;
4117 }
4118 } /* foreach PTE */
4119 }
4120 else
4121 {
4122 /*
4123 * Big Page.
4124 */
4125 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4126 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4127 {
4128 if (PdeDst.n.u1Write)
4129 {
4130 AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4131 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4132 cErrors++;
4133 continue;
4134 }
4135 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4136 {
4137 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4138 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4139 cErrors++;
4140 continue;
4141 }
4142# if 0 /** @todo sync access bit properly... */
4143 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4144 {
4145 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4146 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4147 cErrors++;
4148 }
4149 fIgnoreFlags |= X86_PTE_RW;
4150# else
4151 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4152# endif
4153 }
4154 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4155 {
4156 /* access bit emulation (not implemented). */
4157 if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
4158 {
4159 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4160 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4161 cErrors++;
4162 continue;
4163 }
4164 if (!PdeDst.n.u1Accessed)
4165 {
4166 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4167 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4168 cErrors++;
4169 }
4170 fIgnoreFlags |= X86_PTE_P;
4171 }
4172
4173 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4174 {
4175 AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4176 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4177 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4178 cErrors++;
4179 }
4180
4181 /* iterate the page table. */
4182 for (unsigned iPT = 0, off = 0;
4183 iPT < RT_ELEMENTS(pPTDst->a);
4184 iPT++, off += PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + PAGE_SIZE))
4185 {
4186 const SHWPTE PteDst = pPTDst->a[iPT];
4187
4188 if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4189 {
4190 AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4191 GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4192 cErrors++;
4193 }
4194
4195 /* skip not-present entries. */
4196 if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4197 continue;
4198
4199 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
4200
4201 /* match the physical addresses */
4202 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4203
4204# ifdef IN_RING3
4205 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4206 if (RT_FAILURE(rc))
4207 {
4208 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4209 {
4210 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4211 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4212 cErrors++;
4213 }
4214 }
4215 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4216 {
4217 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4218 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4219 cErrors++;
4220 continue;
4221 }
4222# endif
4223 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4224 if (!pPhysPage)
4225 {
4226# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4227 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4228 {
4229 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4230 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4231 cErrors++;
4232 continue;
4233 }
4234# endif
4235 if (SHW_PTE_IS_RW(PteDst))
4236 {
4237 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4238 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4239 cErrors++;
4240 }
4241 fIgnoreFlags |= X86_PTE_RW;
4242 }
4243 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4244 {
4245 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4246 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4247 cErrors++;
4248 continue;
4249 }
4250
4251 /* flags */
4252 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4253 {
4254 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4255 {
4256 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4257 {
4258 if (SHW_PTE_IS_RW(PteDst))
4259 {
4260 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4261 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4262 cErrors++;
4263 continue;
4264 }
4265 fIgnoreFlags |= X86_PTE_RW;
4266 }
4267 }
4268 else
4269 {
4270 if ( SHW_PTE_IS_P(PteDst)
4271# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
4272 && !PGM_PAGE_IS_MMIO(pPhysPage)
4273# endif
4274 )
4275 {
4276 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4277 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4278 cErrors++;
4279 continue;
4280 }
4281 fIgnoreFlags |= X86_PTE_P;
4282 }
4283 }
4284
4285 if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4286 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4287 )
4288 {
4289 AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4290 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4291 fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4292 cErrors++;
4293 continue;
4294 }
4295 } /* for each PTE */
4296 }
4297 }
4298 /* not present */
4299
4300 } /* for each PDE */
4301
4302 } /* for each PDPTE */
4303
4304 } /* for each PML4E */
4305
4306# ifdef DEBUG
4307 if (cErrors)
4308 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4309# endif
4310# endif /* GST is in {32BIT, PAE, AMD64} */
4311 return cErrors;
4312#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
4313}
4314#endif /* VBOX_STRICT */
4315
4316
4317/**
4318 * Sets up the CR3 for shadow paging
4319 *
4320 * @returns Strict VBox status code.
4321 * @retval VINF_SUCCESS.
4322 *
4323 * @param pVCpu The cross context virtual CPU structure.
4324 * @param GCPhysCR3 The physical address in the CR3 register. (A20
4325 * mask already applied.)
4326 */
4327PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
4328{
4329 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4330
4331 /* Update guest paging info. */
4332#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4333 || PGM_GST_TYPE == PGM_TYPE_PAE \
4334 || PGM_GST_TYPE == PGM_TYPE_AMD64
4335
4336 LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4337 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4338
4339 /*
4340 * Map the page CR3 points at.
4341 */
4342 RTHCPTR HCPtrGuestCR3;
4343 pgmLock(pVM);
4344 PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
4345 AssertReturn(pPageCR3, VERR_PGM_INVALID_CR3_ADDR);
4346 /** @todo this needs some reworking wrt. locking? */
4347# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4348 HCPtrGuestCR3 = NIL_RTHCPTR;
4349 int rc = VINF_SUCCESS;
4350# else
4351 int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3); /** @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4352# endif
4353 pgmUnlock(pVM);
4354 if (RT_SUCCESS(rc))
4355 {
4356# if PGM_GST_TYPE == PGM_TYPE_32BIT
4357# ifdef VBOX_WITH_RAM_IN_KERNEL
4358# ifdef IN_RING3
4359 pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
4360 pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
4361# else
4362 pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
4363 pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
4364# endif
4365# else
4366 pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4367# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4368 pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4369# endif
4370# endif
4371
4372# elif PGM_GST_TYPE == PGM_TYPE_PAE
4373# ifdef VBOX_WITH_RAM_IN_KERNEL
4374# ifdef IN_RING3
4375 pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
4376 pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
4377# else
4378 pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
4379 pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
4380# endif
4381# else
4382 pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4383# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4384 pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4385# endif
4386# endif
4387
4388 /*
4389 * Map the 4 PDs too.
4390 */
4391 PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4392 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4393 {
4394 pVCpu->pgm.s.aGstPaePdpeRegs[i].u = pGuestPDPT->a[i].u;
4395 if (pGuestPDPT->a[i].n.u1Present)
4396 {
4397 RTHCPTR HCPtr;
4398 RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, pGuestPDPT->a[i].u & X86_PDPE_PG_MASK);
4399 pgmLock(pVM);
4400 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4401 AssertReturn(pPage, VERR_PGM_INVALID_PDPE_ADDR);
4402# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4403 HCPtr = NIL_RTHCPTR;
4404 int rc2 = VINF_SUCCESS;
4405# else
4406 int rc2 = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
4407# endif
4408 pgmUnlock(pVM);
4409 if (RT_SUCCESS(rc2))
4410 {
4411# ifdef VBOX_WITH_RAM_IN_KERNEL
4412# ifdef IN_RING3
4413 pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
4414 pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
4415# else
4416 pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
4417 pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
4418# endif
4419# else
4420 pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4421# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4422 pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4423# endif
4424# endif
4425 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4426 continue;
4427 }
4428 AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4429 }
4430
4431 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4432# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4433 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4434# endif
4435 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4436 }
4437
4438# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4439# ifdef VBOX_WITH_RAM_IN_KERNEL
4440# ifdef IN_RING3
4441 pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
4442 pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
4443# else
4444 pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
4445 pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
4446# endif
4447# else
4448 pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4449# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4450 pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4451# endif
4452# endif
4453# endif
4454 }
4455 else
4456 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4457
4458#else /* prot/real stub */
4459 int rc = VINF_SUCCESS;
4460#endif
4461
4462 /*
4463 * Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
4464 */
4465# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4466 || PGM_SHW_TYPE == PGM_TYPE_PAE \
4467 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4468 && ( PGM_GST_TYPE != PGM_TYPE_REAL \
4469 && PGM_GST_TYPE != PGM_TYPE_PROT))
4470
4471 Assert(!pVM->pgm.s.fNestedPaging);
4472 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4473
4474 /*
4475 * Update the shadow root page as well since that's not fixed.
4476 */
4477 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4478 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4479 PPGMPOOLPAGE pNewShwPageCR3;
4480
4481 pgmLock(pVM);
4482
4483# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4484 if (pPool->cDirtyPages)
4485 pgmPoolResetDirtyPages(pVM);
4486# endif
4487
4488 Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4489 rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
4490 NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/,
4491 &pNewShwPageCR3);
4492 AssertFatalRC(rc);
4493 rc = VINF_SUCCESS;
4494
4495 pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4496# ifdef IN_RING0
4497 pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4498# else
4499 pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4500# endif
4501
4502# ifndef PGM_WITHOUT_MAPPINGS
4503 /*
4504 * Apply all hypervisor mappings to the new CR3.
4505 * Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4506 * make sure we check for conflicts in the new CR3 root.
4507 */
4508# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4509 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4510# endif
4511 rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4512 AssertRCReturn(rc, rc);
4513# endif
4514
4515 /* Set the current hypervisor CR3. */
4516 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4517
4518 /* Clean up the old CR3 root. */
4519 if ( pOldShwPageCR3
4520 && pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4521 {
4522 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4523# ifndef PGM_WITHOUT_MAPPINGS
4524 /* Remove the hypervisor mappings from the shadow page table. */
4525 pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4526# endif
4527 /* Mark the page as unlocked; allow flushing again. */
4528 pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4529
4530 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4531 }
4532 pgmUnlock(pVM);
4533# else
4534 NOREF(GCPhysCR3);
4535# endif
4536
4537 return rc;
4538}
4539
4540/**
4541 * Unmaps the shadow CR3.
4542 *
4543 * @returns VBox status, no specials.
4544 * @param pVCpu The cross context virtual CPU structure.
4545 */
4546PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
4547{
4548 LogFlow(("UnmapCR3\n"));
4549
4550 int rc = VINF_SUCCESS;
4551 PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4552
4553 /*
4554 * Update guest paging info.
4555 */
4556#if PGM_GST_TYPE == PGM_TYPE_32BIT
4557 pVCpu->pgm.s.pGst32BitPdR3 = 0;
4558# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4559 pVCpu->pgm.s.pGst32BitPdR0 = 0;
4560# endif
4561
4562#elif PGM_GST_TYPE == PGM_TYPE_PAE
4563 pVCpu->pgm.s.pGstPaePdptR3 = 0;
4564# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4565 pVCpu->pgm.s.pGstPaePdptR0 = 0;
4566# endif
4567 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4568 {
4569 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4570# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4571 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4572# endif
4573 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4574 }
4575
4576#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4577 pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4578# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4579 pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4580# endif
4581
4582#else /* prot/real mode stub */
4583 /* nothing to do */
4584#endif
4585
4586 /*
4587 * Update shadow paging info.
4588 */
4589#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4590 || PGM_SHW_TYPE == PGM_TYPE_PAE \
4591 || PGM_SHW_TYPE == PGM_TYPE_AMD64))
4592# if PGM_GST_TYPE != PGM_TYPE_REAL
4593 Assert(!pVM->pgm.s.fNestedPaging);
4594# endif
4595 pgmLock(pVM);
4596
4597# ifndef PGM_WITHOUT_MAPPINGS
4598 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4599 /* Remove the hypervisor mappings from the shadow page table. */
4600 pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4601# endif
4602
4603 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4604 {
4605 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4606
4607# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4608 if (pPool->cDirtyPages)
4609 pgmPoolResetDirtyPages(pVM);
4610# endif
4611
4612 /* Mark the page as unlocked; allow flushing again. */
4613 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4614
4615 pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4616 pVCpu->pgm.s.pShwPageCR3R3 = 0;
4617 pVCpu->pgm.s.pShwPageCR3R0 = 0;
4618 }
4619
4620 pgmUnlock(pVM);
4621#endif
4622
4623 return rc;
4624}
4625
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette