VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 45733

Last change on this file since 45733 was 45619, checked in by vboxsync, 12 years ago

PGMAllBth.h: Skip a path in Trap0eHandler that hooks into CSAM if CSAM is disabled, avoid some work (like disassembling the instruction).

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 204.9 KB
Line 
1/* $Id: PGMAllBth.h 45619 2013-04-18 19:54:22Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6 * {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7 * set to PGM_TYPE_PROT. Half of the code in this file is not
8 * exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9 *
10 * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11 * PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12 *
13 * @remarks This file is one big \#ifdef-orgy!
14 *
15 */
16
17/*
18 * Copyright (C) 2006-2013 Oracle Corporation
19 *
20 * This file is part of VirtualBox Open Source Edition (OSE), as
21 * available from http://www.virtualbox.org. This file is free software;
22 * you can redistribute it and/or modify it under the terms of the GNU
23 * General Public License (GPL) as published by the Free Software
24 * Foundation, in version 2 as it comes in the "COPYING" file of the
25 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27 */
28
29#ifdef _MSC_VER
30/** @todo we're generating unnecessary code in nested/ept shadow mode and for
31 * real/prot-guest+RC mode. */
32# pragma warning(disable: 4505)
33#endif
34
35/*******************************************************************************
36* Internal Functions *
37*******************************************************************************/
38RT_C_DECLS_BEGIN
39PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
40PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
42static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
43static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
44# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
45static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
46# else
47static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
48#endif
49PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
50PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
51PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
52#ifdef VBOX_STRICT
53PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
54#endif
55PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
56PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
57RT_C_DECLS_END
58
59
60/*
61 * Filter out some illegal combinations of guest and shadow paging, so we can
62 * remove redundant checks inside functions.
63 */
64#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
65# error "Invalid combination; PAE guest implies PAE shadow"
66#endif
67
68#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
69 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
70# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
71#endif
72
73#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
74 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
75# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
76#endif
77
78#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
79 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
80# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
81#endif
82
83#ifndef IN_RING3
84
85# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
86/**
87 * Deal with a guest page fault.
88 *
89 * @returns Strict VBox status code.
90 * @retval VINF_EM_RAW_GUEST_TRAP
91 * @retval VINF_EM_RAW_EMULATE_INSTR
92 *
93 * @param pVCpu The current CPU.
94 * @param pGstWalk The guest page table walk result.
95 * @param uErr The error code.
96 */
97PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
98{
99# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE)
100 /*
101 * Check for write conflicts with our hypervisor mapping.
102 *
103 * If the guest happens to access a non-present page, where our hypervisor
104 * is currently mapped, then we'll create a #PF storm in the guest.
105 */
106 if ( (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW)
107 && MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
108 {
109 /* Force a CR3 sync to check for conflicts and emulate the instruction. */
110 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
111 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
112 return VINF_EM_RAW_EMULATE_INSTR;
113 }
114# endif
115
116 /*
117 * Calc the error code for the guest trap.
118 */
119 uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
120 ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)
121 : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
122 if (pGstWalk->Core.fBadPhysAddr)
123 {
124 uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P;
125 Assert(!pGstWalk->Core.fNotPresent);
126 }
127 else if (!pGstWalk->Core.fNotPresent)
128 uNewErr |= X86_TRAP_PF_P;
129 TRPMSetErrorCode(pVCpu, uNewErr);
130
131 LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
132 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
133 return VINF_EM_RAW_GUEST_TRAP;
134}
135# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
136
137
138/**
139 * Deal with a guest page fault.
140 *
141 * The caller has taken the PGM lock.
142 *
143 * @returns Strict VBox status code.
144 *
145 * @param pVCpu The current CPU.
146 * @param uErr The error code.
147 * @param pRegFrame The register frame.
148 * @param pvFault The fault address.
149 * @param pPage The guest page at @a pvFault.
150 * @param pGstWalk The guest page table walk result.
151 * @param pfLockTaken PGM lock taken here or not (out). This is true
152 * when we're called.
153 */
154static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
155 RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
156# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
157 , PGSTPTWALK pGstWalk
158# endif
159 )
160{
161# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
162 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
163#endif
164 PVM pVM = pVCpu->CTX_SUFF(pVM);
165 int rc;
166
167 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
168 {
169 /*
170 * Physical page access handler.
171 */
172# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
173 const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
174# else
175 const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
176# endif
177 PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
178 if (pCur)
179 {
180# ifdef PGM_SYNC_N_PAGES
181 /*
182 * If the region is write protected and we got a page not present fault, then sync
183 * the pages. If the fault was caused by a read, then restart the instruction.
184 * In case of write access continue to the GC write handler.
185 *
186 * ASSUMES that there is only one handler per page or that they have similar write properties.
187 */
188 if ( !(uErr & X86_TRAP_PF_P)
189 && pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE)
190 {
191# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
192 rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
193# else
194 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
195# endif
196 if ( RT_FAILURE(rc)
197 || !(uErr & X86_TRAP_PF_RW)
198 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
199 {
200 AssertRC(rc);
201 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
202 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
203 return rc;
204 }
205 }
206# endif
207# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
208 /*
209 * If the access was not thru a #PF(RSVD|...) resync the page.
210 */
211 if ( !(uErr & X86_TRAP_PF_RSVD)
212 && pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
213# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
214 && pGstWalk->Core.fEffectiveRW
215 && !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
216# endif
217 )
218 {
219# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
220 rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
221# else
222 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
223# endif
224 if ( RT_FAILURE(rc)
225 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
226 {
227 AssertRC(rc);
228 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
229 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
230 return rc;
231 }
232 }
233# endif
234
235 AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
236 || (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
237 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
238 pvFault, GCPhysFault, pPage, uErr, pCur->enmType));
239 if (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE)
240 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
241 else
242 {
243 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
244 if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
245 }
246
247 if (pCur->CTX_SUFF(pfnHandler))
248 {
249 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
250 void *pvUser = pCur->CTX_SUFF(pvUser);
251# ifdef IN_RING0
252 PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
253# else
254 PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
255# endif
256
257 STAM_PROFILE_START(&pCur->Stat, h);
258 if (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler))
259 {
260 pgmUnlock(pVM);
261 *pfLockTaken = false;
262 }
263
264 rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
265
266# ifdef VBOX_WITH_STATISTICS
267 pgmLock(pVM);
268 pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
269 if (pCur)
270 STAM_PROFILE_STOP(&pCur->Stat, h);
271 pgmUnlock(pVM);
272# endif
273 }
274 else
275 rc = VINF_EM_RAW_EMULATE_INSTR;
276
277 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
278 return rc;
279 }
280 }
281# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
282 else
283 {
284# ifdef PGM_SYNC_N_PAGES
285 /*
286 * If the region is write protected and we got a page not present fault, then sync
287 * the pages. If the fault was caused by a read, then restart the instruction.
288 * In case of write access continue to the GC write handler.
289 */
290 if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
291 && !(uErr & X86_TRAP_PF_P))
292 {
293 rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
294 if ( RT_FAILURE(rc)
295 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
296 || !(uErr & X86_TRAP_PF_RW))
297 {
298 AssertRC(rc);
299 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
300 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
301 return rc;
302 }
303 }
304# endif
305 /*
306 * Ok, it's an virtual page access handler.
307 *
308 * Since it's faster to search by address, we'll do that first
309 * and then retry by GCPhys if that fails.
310 */
311 /** @todo r=bird: perhaps we should consider looking up by physical address directly now?
312 * r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
313 * out of sync, because the page was changed without us noticing it (not-present -> present
314 * without invlpg or mov cr3, xxx).
315 */
316 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
317 if (pCur)
318 {
319 AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
320 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
321 || !(uErr & X86_TRAP_PF_P)
322 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
323 ("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
324 pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCur->enmType));
325
326 if ( pvFault - pCur->Core.Key < pCur->cb
327 && ( uErr & X86_TRAP_PF_RW
328 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
329 {
330# ifdef IN_RC
331 STAM_PROFILE_START(&pCur->Stat, h);
332 RTGCPTR GCPtrStart = pCur->Core.Key;
333 CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
334 pgmUnlock(pVM);
335 *pfLockTaken = false;
336
337 rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, pvFault - GCPtrStart);
338
339# ifdef VBOX_WITH_STATISTICS
340 pgmLock(pVM);
341 pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
342 if (pCur)
343 STAM_PROFILE_STOP(&pCur->Stat, h);
344 pgmUnlock(pVM);
345# endif
346# else
347 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
348# endif
349 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
350 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
351 return rc;
352 }
353 /* Unhandled part of a monitored page */
354 Log(("Unhandled part of monitored page %RGv\n", pvFault));
355 }
356 else
357 {
358 /* Check by physical address. */
359 unsigned iPage;
360 rc = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &pCur, &iPage);
361 Assert(RT_SUCCESS(rc) || !pCur);
362 if ( pCur
363 && ( uErr & X86_TRAP_PF_RW
364 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
365 {
366 Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
367# ifdef IN_RC
368 STAM_PROFILE_START(&pCur->Stat, h);
369 RTGCPTR GCPtrStart = pCur->Core.Key;
370 CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
371 pgmUnlock(pVM);
372 *pfLockTaken = false;
373
374 RTGCPTR off = (iPage << PAGE_SHIFT)
375 + (pvFault & PAGE_OFFSET_MASK)
376 - (GCPtrStart & PAGE_OFFSET_MASK);
377 Assert(off < pCur->cb);
378 rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, off);
379
380# ifdef VBOX_WITH_STATISTICS
381 pgmLock(pVM);
382 pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
383 if (pCur)
384 STAM_PROFILE_STOP(&pCur->Stat, h);
385 pgmUnlock(pVM);
386# endif
387# else
388 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
389# endif
390 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
391 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
392 return rc;
393 }
394 }
395 }
396# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
397
398 /*
399 * There is a handled area of the page, but this fault doesn't belong to it.
400 * We must emulate the instruction.
401 *
402 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
403 * we first check if this was a page-not-present fault for a page with only
404 * write access handlers. Restart the instruction if it wasn't a write access.
405 */
406 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
407
408 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
409 && !(uErr & X86_TRAP_PF_P))
410 {
411# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
412 rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
413# else
414 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
415# endif
416 if ( RT_FAILURE(rc)
417 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
418 || !(uErr & X86_TRAP_PF_RW))
419 {
420 AssertRC(rc);
421 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
422 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
423 return rc;
424 }
425 }
426
427 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
428 * It's writing to an unhandled part of the LDT page several million times.
429 */
430 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
431 LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
432 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
433 return rc;
434} /* if any kind of handler */
435
436
437/**
438 * #PF Handler for raw-mode guest execution.
439 *
440 * @returns VBox status code (appropriate for trap handling and GC return).
441 *
442 * @param pVCpu Pointer to the VMCPU.
443 * @param uErr The trap error code.
444 * @param pRegFrame Trap register frame.
445 * @param pvFault The fault address.
446 * @param pfLockTaken PGM lock taken here or not (out)
447 */
448PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
449{
450 PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
451
452 *pfLockTaken = false;
453
454# if ( PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \
455 || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
456 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
457 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
458 int rc;
459
460# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
461 /*
462 * Walk the guest page translation tables and check if it's a guest fault.
463 */
464 GSTPTWALK GstWalk;
465 rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
466 if (RT_FAILURE_NP(rc))
467 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
468
469 /* assert some GstWalk sanity. */
470# if PGM_GST_TYPE == PGM_TYPE_AMD64
471 /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. */
472# endif
473# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
474 /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */
475# endif
476 /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */
477 /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */
478 Assert(GstWalk.Core.fSucceeded);
479
480 if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
481 {
482 if ( ( (uErr & X86_TRAP_PF_RW)
483 && !GstWalk.Core.fEffectiveRW
484 && ( (uErr & X86_TRAP_PF_US)
485 || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
486 || ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
487 || ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
488 )
489 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
490 }
491
492 /*
493 * Set the accessed and dirty flags.
494 */
495# if PGM_GST_TYPE == PGM_TYPE_AMD64
496 GstWalk.Pml4e.u |= X86_PML4E_A;
497 GstWalk.pPml4e->u |= X86_PML4E_A;
498 GstWalk.Pdpe.u |= X86_PDPE_A;
499 GstWalk.pPdpe->u |= X86_PDPE_A;
500# endif
501 if (GstWalk.Core.fBigPage)
502 {
503 Assert(GstWalk.Pde.b.u1Size);
504 if (uErr & X86_TRAP_PF_RW)
505 {
506 GstWalk.Pde.u |= X86_PDE4M_A | X86_PDE4M_D;
507 GstWalk.pPde->u |= X86_PDE4M_A | X86_PDE4M_D;
508 }
509 else
510 {
511 GstWalk.Pde.u |= X86_PDE4M_A;
512 GstWalk.pPde->u |= X86_PDE4M_A;
513 }
514 }
515 else
516 {
517 Assert(!GstWalk.Pde.b.u1Size);
518 GstWalk.Pde.u |= X86_PDE_A;
519 GstWalk.pPde->u |= X86_PDE_A;
520 if (uErr & X86_TRAP_PF_RW)
521 {
522# ifdef VBOX_WITH_STATISTICS
523 if (!GstWalk.Pte.n.u1Dirty)
524 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
525 else
526 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
527# endif
528 GstWalk.Pte.u |= X86_PTE_A | X86_PTE_D;
529 GstWalk.pPte->u |= X86_PTE_A | X86_PTE_D;
530 }
531 else
532 {
533 GstWalk.Pte.u |= X86_PTE_A;
534 GstWalk.pPte->u |= X86_PTE_A;
535 }
536 Assert(GstWalk.Pte.u == GstWalk.pPte->u);
537 }
538 AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u,
539 ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
540# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
541 GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */
542# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
543
544 /* Take the big lock now. */
545 *pfLockTaken = true;
546 pgmLock(pVM);
547
548# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
549 /*
550 * If it is a reserved bit fault we know that it is an MMIO (access
551 * handler) related fault and can skip some 200 lines of code.
552 */
553 if (uErr & X86_TRAP_PF_RSVD)
554 {
555 Assert(uErr & X86_TRAP_PF_P);
556 PPGMPAGE pPage;
557# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
558 rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
559 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
560 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
561 pfLockTaken, &GstWalk));
562 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
563# else
564 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
565 if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
566 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
567 pfLockTaken));
568 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
569# endif
570 AssertRC(rc);
571 PGM_INVL_PG(pVCpu, pvFault);
572 return rc; /* Restart with the corrected entry. */
573 }
574# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
575
576 /*
577 * Fetch the guest PDE, PDPE and PML4E.
578 */
579# if PGM_SHW_TYPE == PGM_TYPE_32BIT
580 const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
581 PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
582
583# elif PGM_SHW_TYPE == PGM_TYPE_PAE
584 const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
585 PX86PDPAE pPDDst;
586# if PGM_GST_TYPE == PGM_TYPE_PAE
587 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
588# else
589 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
590# endif
591 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
592
593# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
594 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
595 PX86PDPAE pPDDst;
596# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
597 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A,
598 X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst);
599# else
600 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
601# endif
602 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
603
604# elif PGM_SHW_TYPE == PGM_TYPE_EPT
605 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
606 PEPTPD pPDDst;
607 rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
608 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
609# endif
610 Assert(pPDDst);
611
612# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
613 /*
614 * Dirty page handling.
615 *
616 * If we successfully correct the write protection fault due to dirty bit
617 * tracking, then return immediately.
618 */
619 if (uErr & X86_TRAP_PF_RW) /* write fault? */
620 {
621 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
622 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
623 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
624 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
625 {
626 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
627 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
628 ? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
629 : &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
630 LogBird(("Trap0eHandler: returns VINF_SUCCESS\n"));
631 return VINF_SUCCESS;
632 }
633 //AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - triggers with smp w7 guests.
634 //AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto.
635 }
636
637# if 0 /* rarely useful; leave for debugging. */
638 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
639# endif
640# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
641
642 /*
643 * A common case is the not-present error caused by lazy page table syncing.
644 *
645 * It is IMPORTANT that we weed out any access to non-present shadow PDEs
646 * here so we can safely assume that the shadow PT is present when calling
647 * SyncPage later.
648 *
649 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
650 * of mapping conflict and defer to SyncCR3 in R3.
651 * (Again, we do NOT support access handlers for non-present guest pages.)
652 *
653 */
654# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
655 Assert(GstWalk.Pde.n.u1Present);
656# endif
657 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
658 && !pPDDst->a[iPDDst].n.u1Present)
659 {
660 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
661# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
662 LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
663 rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
664# else
665 LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
666 rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
667# endif
668 if (RT_SUCCESS(rc))
669 return rc;
670 Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
671 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
672 return VINF_PGM_SYNC_CR3;
673 }
674
675# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
676 /*
677 * Check if this address is within any of our mappings.
678 *
679 * This is *very* fast and it's gonna save us a bit of effort below and prevent
680 * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
681 * (BTW, it's impossible to have physical access handlers in a mapping.)
682 */
683 if (pgmMapAreMappingsEnabled(pVM))
684 {
685 PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
686 for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
687 {
688 if (pvFault < pMapping->GCPtr)
689 break;
690 if (pvFault - pMapping->GCPtr < pMapping->cb)
691 {
692 /*
693 * The first thing we check is if we've got an undetected conflict.
694 */
695 if (pgmMapAreMappingsFloating(pVM))
696 {
697 unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
698 while (iPT-- > 0)
699 if (GstWalk.pPde[iPT].n.u1Present)
700 {
701 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
702 Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
703 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
704 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
705 return VINF_PGM_SYNC_CR3;
706 }
707 }
708
709 /*
710 * Check if the fault address is in a virtual page access handler range.
711 */
712 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
713 if ( pCur
714 && pvFault - pCur->Core.Key < pCur->cb
715 && uErr & X86_TRAP_PF_RW)
716 {
717# ifdef IN_RC
718 STAM_PROFILE_START(&pCur->Stat, h);
719 pgmUnlock(pVM);
720 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
721 pgmLock(pVM);
722 STAM_PROFILE_STOP(&pCur->Stat, h);
723# else
724 AssertFailed();
725 rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
726# endif
727 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
728 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
729 return rc;
730 }
731
732 /*
733 * Pretend we're not here and let the guest handle the trap.
734 */
735 TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
736 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
737 LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
738 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
739 return VINF_EM_RAW_GUEST_TRAP;
740 }
741 }
742 } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
743# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
744
745 /*
746 * Check if this fault address is flagged for special treatment,
747 * which means we'll have to figure out the physical address and
748 * check flags associated with it.
749 *
750 * ASSUME that we can limit any special access handling to pages
751 * in page tables which the guest believes to be present.
752 */
753# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
754 RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
755# else
756 RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK);
757# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
758 PPGMPAGE pPage;
759 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
760 if (RT_FAILURE(rc))
761 {
762 /*
763 * When the guest accesses invalid physical memory (e.g. probing
764 * of RAM or accessing a remapped MMIO range), then we'll fall
765 * back to the recompiler to emulate the instruction.
766 */
767 LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
768 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
769 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
770 return VINF_EM_RAW_EMULATE_INSTR;
771 }
772
773 /*
774 * Any handlers for this page?
775 */
776 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
777# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
778 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
779 &GstWalk));
780# else
781 return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
782# endif
783
784 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTimeOutOfSync, c);
785
786# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
787 if (uErr & X86_TRAP_PF_P)
788 {
789 /*
790 * The page isn't marked, but it might still be monitored by a virtual page access handler.
791 * (ASSUMES no temporary disabling of virtual handlers.)
792 */
793 /** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
794 * we should correct both the shadow page table and physical memory flags, and not only check for
795 * accesses within the handler region but for access to pages with virtual handlers. */
796 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
797 if (pCur)
798 {
799 AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
800 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
801 || !(uErr & X86_TRAP_PF_P)
802 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
803 ("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
804
805 if ( pvFault - pCur->Core.Key < pCur->cb
806 && ( uErr & X86_TRAP_PF_RW
807 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
808 {
809# ifdef IN_RC
810 STAM_PROFILE_START(&pCur->Stat, h);
811 pgmUnlock(pVM);
812 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
813 pgmLock(pVM);
814 STAM_PROFILE_STOP(&pCur->Stat, h);
815# else
816 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
817# endif
818 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
819 return rc;
820 }
821 }
822 }
823# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
824
825 /*
826 * We are here only if page is present in Guest page tables and
827 * trap is not handled by our handlers.
828 *
829 * Check it for page out-of-sync situation.
830 */
831 if (!(uErr & X86_TRAP_PF_P))
832 {
833 /*
834 * Page is not present in our page tables. Try to sync it!
835 */
836 if (uErr & X86_TRAP_PF_US)
837 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
838 else /* supervisor */
839 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
840
841 if (PGM_PAGE_IS_BALLOONED(pPage))
842 {
843 /* Emulate reads from ballooned pages as they are not present in
844 our shadow page tables. (Required for e.g. Solaris guests; soft
845 ecc, random nr generator.) */
846 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
847 LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
848 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
849 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
850 return rc;
851 }
852
853# if defined(LOG_ENABLED) && !defined(IN_RING0)
854 RTGCPHYS GCPhys2;
855 uint64_t fPageGst2;
856 PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
857# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
858 Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
859 pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
860# else
861 Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
862 pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
863# endif
864# endif /* LOG_ENABLED */
865
866# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
867 if ( !GstWalk.Core.fEffectiveUS
868 && CSAMIsEnabled(pVM)
869 && CPUMGetGuestCPL(pVCpu) == 0)
870 {
871 /* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
872 if ( pvFault == (RTGCPTR)pRegFrame->eip
873 || pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
874# ifdef CSAM_DETECT_NEW_CODE_PAGES
875 || ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
876 && CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
877# endif /* CSAM_DETECT_NEW_CODE_PAGES */
878 )
879 {
880 LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
881 rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
882 if (rc != VINF_SUCCESS)
883 {
884 /*
885 * CSAM needs to perform a job in ring 3.
886 *
887 * Sync the page before going to the host context; otherwise we'll end up in a loop if
888 * CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
889 */
890 LogFlow(("CSAM ring 3 job\n"));
891 int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
892 AssertRC(rc2);
893
894 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
895 return rc;
896 }
897 }
898# ifdef CSAM_DETECT_NEW_CODE_PAGES
899 else if ( uErr == X86_TRAP_PF_RW
900 && pRegFrame->ecx >= 0x100 /* early check for movswd count */
901 && pRegFrame->ecx < 0x10000)
902 {
903 /* In case of a write to a non-present supervisor shadow page, we'll take special precautions
904 * to detect loading of new code pages.
905 */
906
907 /*
908 * Decode the instruction.
909 */
910 PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
911 uint32_t cbOp;
912 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
913
914 /* For now we'll restrict this to rep movsw/d instructions */
915 if ( rc == VINF_SUCCESS
916 && pDis->pCurInstr->opcode == OP_MOVSWD
917 && (pDis->prefix & DISPREFIX_REP))
918 {
919 CSAMMarkPossibleCodePage(pVM, pvFault);
920 }
921 }
922# endif /* CSAM_DETECT_NEW_CODE_PAGES */
923
924 /*
925 * Mark this page as safe.
926 */
927 /** @todo not correct for pages that contain both code and data!! */
928 Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
929 CSAMMarkPage(pVM, pvFault, true);
930 }
931# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
932# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
933 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
934# else
935 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
936# endif
937 if (RT_SUCCESS(rc))
938 {
939 /* The page was successfully synced, return to the guest. */
940 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
941 return VINF_SUCCESS;
942 }
943 }
944 else /* uErr & X86_TRAP_PF_P: */
945 {
946 /*
947 * Write protected pages are made writable when the guest makes the
948 * first write to it. This happens for pages that are shared, write
949 * monitored or not yet allocated.
950 *
951 * We may also end up here when CR0.WP=0 in the guest.
952 *
953 * Also, a side effect of not flushing global PDEs are out of sync
954 * pages due to physical monitored regions, that are no longer valid.
955 * Assume for now it only applies to the read/write flag.
956 */
957 if (uErr & X86_TRAP_PF_RW)
958 {
959 /*
960 * Check if it is a read-only page.
961 */
962 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
963 {
964 Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
965 Assert(!PGM_PAGE_IS_ZERO(pPage));
966 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
967 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
968
969 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
970 if (rc != VINF_SUCCESS)
971 {
972 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
973 return rc;
974 }
975 if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
976 return VINF_EM_NO_MEMORY;
977 }
978
979# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
980 /*
981 * Check to see if we need to emulate the instruction if CR0.WP=0.
982 */
983 if ( !GstWalk.Core.fEffectiveRW
984 && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
985 && CPUMGetGuestCPL(pVCpu) == 0)
986 {
987 Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P));
988 rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
989 if (RT_SUCCESS(rc))
990 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
991 else
992 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
993 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
994 return rc;
995 }
996# endif
997 /// @todo count the above case; else
998 if (uErr & X86_TRAP_PF_US)
999 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
1000 else /* supervisor */
1001 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1002
1003 /*
1004 * Sync the page.
1005 *
1006 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1007 * page is not present, which is not true in this case.
1008 */
1009# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1010 rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1011# else
1012 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
1013# endif
1014 if (RT_SUCCESS(rc))
1015 {
1016 /*
1017 * Page was successfully synced, return to guest but invalidate
1018 * the TLB first as the page is very likely to be in it.
1019 */
1020# if PGM_SHW_TYPE == PGM_TYPE_EPT
1021 HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
1022# else
1023 PGM_INVL_PG(pVCpu, pvFault);
1024# endif
1025# ifdef VBOX_STRICT
1026 RTGCPHYS GCPhys2;
1027 uint64_t fPageGst;
1028 if (!pVM->pgm.s.fNestedPaging)
1029 {
1030 rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
1031 AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
1032 LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
1033 }
1034 uint64_t fPageShw;
1035 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1036 AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
1037 ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
1038# endif /* VBOX_STRICT */
1039 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
1040 return VINF_SUCCESS;
1041 }
1042 }
1043 /** @todo else: why are we here? */
1044
1045# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1046 /*
1047 * Check for VMM page flags vs. Guest page flags consistency.
1048 * Currently only for debug purposes.
1049 */
1050 if (RT_SUCCESS(rc))
1051 {
1052 /* Get guest page flags. */
1053 uint64_t fPageGst;
1054 rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1055 if (RT_SUCCESS(rc))
1056 {
1057 uint64_t fPageShw;
1058 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1059
1060 /*
1061 * Compare page flags.
1062 * Note: we have AVL, A, D bits desynced.
1063 */
1064 AssertMsg( (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
1065 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)),
1066 ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n",
1067 pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
1068 }
1069 else
1070 AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1071 }
1072 else
1073 AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1074# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1075 }
1076
1077
1078 /*
1079 * If we get here it is because something failed above, i.e. most like guru
1080 * meditiation time.
1081 */
1082 LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1083 __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
1084 return rc;
1085
1086# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1087 NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
1088 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1089 return VERR_PGM_NOT_USED_IN_MODE;
1090# endif
1091}
1092#endif /* !IN_RING3 */
1093
1094
1095/**
1096 * Emulation of the invlpg instruction.
1097 *
1098 *
1099 * @returns VBox status code.
1100 *
1101 * @param pVCpu Pointer to the VMCPU.
1102 * @param GCPtrPage Page to invalidate.
1103 *
1104 * @remark ASSUMES that the guest is updating before invalidating. This order
1105 * isn't required by the CPU, so this is speculative and could cause
1106 * trouble.
1107 * @remark No TLB shootdown is done on any other VCPU as we assume that
1108 * invlpg emulation is the *only* reason for calling this function.
1109 * (The guest has to shoot down TLB entries on other CPUs itself)
1110 * Currently true, but keep in mind!
1111 *
1112 * @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1113 * Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1114 */
1115PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1116{
1117#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1118 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
1119 && PGM_SHW_TYPE != PGM_TYPE_EPT
1120 int rc;
1121 PVM pVM = pVCpu->CTX_SUFF(pVM);
1122 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1123
1124 PGM_LOCK_ASSERT_OWNER(pVM);
1125
1126 LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1127
1128 /*
1129 * Get the shadow PD entry and skip out if this PD isn't present.
1130 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1131 */
1132# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1133 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1134 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1135
1136 /* Fetch the pgm pool shadow descriptor. */
1137 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1138 Assert(pShwPde);
1139
1140# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1141 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1142 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1143
1144 /* If the shadow PDPE isn't present, then skip the invalidate. */
1145 if (!pPdptDst->a[iPdpt].n.u1Present)
1146 {
1147 Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1148 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1149 return VINF_SUCCESS;
1150 }
1151
1152 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1153 PPGMPOOLPAGE pShwPde = NULL;
1154 PX86PDPAE pPDDst;
1155
1156 /* Fetch the pgm pool shadow descriptor. */
1157 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1158 AssertRCSuccessReturn(rc, rc);
1159 Assert(pShwPde);
1160
1161 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1162 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1163
1164# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1165 /* PML4 */
1166 /*const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/
1167 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1168 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1169 PX86PDPAE pPDDst;
1170 PX86PDPT pPdptDst;
1171 PX86PML4E pPml4eDst;
1172 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1173 if (rc != VINF_SUCCESS)
1174 {
1175 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1176 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1177 return VINF_SUCCESS;
1178 }
1179 Assert(pPDDst);
1180
1181 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1182 PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1183
1184 if (!pPdpeDst->n.u1Present)
1185 {
1186 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1187 return VINF_SUCCESS;
1188 }
1189
1190 /* Fetch the pgm pool shadow descriptor. */
1191 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1192 Assert(pShwPde);
1193
1194# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1195
1196 const SHWPDE PdeDst = *pPdeDst;
1197 if (!PdeDst.n.u1Present)
1198 {
1199 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1200 PGM_INVL_PG(pVCpu, GCPtrPage);
1201 return VINF_SUCCESS;
1202 }
1203
1204 /*
1205 * Get the guest PD entry and calc big page.
1206 */
1207# if PGM_GST_TYPE == PGM_TYPE_32BIT
1208 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1209 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1210 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1211# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1212 unsigned iPDSrc = 0;
1213# if PGM_GST_TYPE == PGM_TYPE_PAE
1214 X86PDPE PdpeSrcIgn;
1215 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1216# else /* AMD64 */
1217 PX86PML4E pPml4eSrcIgn;
1218 X86PDPE PdpeSrcIgn;
1219 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1220# endif
1221 GSTPDE PdeSrc;
1222
1223 if (pPDSrc)
1224 PdeSrc = pPDSrc->a[iPDSrc];
1225 else
1226 PdeSrc.u = 0;
1227# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1228 const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1229
1230# ifdef IN_RING3
1231 /*
1232 * If a CR3 Sync is pending we may ignore the invalidate page operation
1233 * depending on the kind of sync and if it's a global page or not.
1234 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
1235 */
1236# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1237 if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1238 || ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1239 && fIsBigPage
1240 && PdeSrc.b.u1Global
1241 )
1242 )
1243# else
1244 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1245# endif
1246 {
1247 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1248 return VINF_SUCCESS;
1249 }
1250# endif /* IN_RING3 */
1251
1252 /*
1253 * Deal with the Guest PDE.
1254 */
1255 rc = VINF_SUCCESS;
1256 if (PdeSrc.n.u1Present)
1257 {
1258 Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1259 && (PdeSrc.n.u1Write || !PdeDst.n.u1Write));
1260# ifndef PGM_WITHOUT_MAPPING
1261 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1262 {
1263 /*
1264 * Conflict - Let SyncPT deal with it to avoid duplicate code.
1265 */
1266 Assert(pgmMapAreMappingsEnabled(pVM));
1267 Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1268 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1269 }
1270 else
1271# endif /* !PGM_WITHOUT_MAPPING */
1272 if (!fIsBigPage)
1273 {
1274 /*
1275 * 4KB - page.
1276 */
1277 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1278 RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1279
1280# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1281 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1282 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
1283# endif
1284 if (pShwPage->GCPhys == GCPhys)
1285 {
1286 /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1287 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1288
1289 PGSTPT pPTSrc;
1290 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1291 if (RT_SUCCESS(rc))
1292 {
1293 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1294 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1295 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1296 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1297 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1298 GCPtrPage, PteSrc.n.u1Present,
1299 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1300 PteSrc.n.u1User & PdeSrc.n.u1User,
1301 (uint64_t)PteSrc.u,
1302 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1303 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1304 }
1305 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1306 PGM_INVL_PG(pVCpu, GCPtrPage);
1307 }
1308 else
1309 {
1310 /*
1311 * The page table address changed.
1312 */
1313 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1314 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1315 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1316 ASMAtomicWriteSize(pPdeDst, 0);
1317 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1318 PGM_INVL_VCPU_TLBS(pVCpu);
1319 }
1320 }
1321 else
1322 {
1323 /*
1324 * 2/4MB - page.
1325 */
1326 /* Before freeing the page, check if anything really changed. */
1327 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1328 RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1329# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1330 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1331 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1332# endif
1333 if ( pShwPage->GCPhys == GCPhys
1334 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1335 {
1336 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1337 /** @todo This test is wrong as it cannot check the G bit!
1338 * FIXME */
1339 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1340 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
1341 && ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1342 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1343 {
1344 LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1345 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1346 return VINF_SUCCESS;
1347 }
1348 }
1349
1350 /*
1351 * Ok, the page table is present and it's been changed in the guest.
1352 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1353 * We could do this for some flushes in GC too, but we need an algorithm for
1354 * deciding which 4MB pages containing code likely to be executed very soon.
1355 */
1356 LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1357 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1358 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1359 ASMAtomicWriteSize(pPdeDst, 0);
1360 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1361 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1362 }
1363 }
1364 else
1365 {
1366 /*
1367 * Page directory is not present, mark shadow PDE not present.
1368 */
1369 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1370 {
1371 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1372 ASMAtomicWriteSize(pPdeDst, 0);
1373 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1374 PGM_INVL_PG(pVCpu, GCPtrPage);
1375 }
1376 else
1377 {
1378 Assert(pgmMapAreMappingsEnabled(pVM));
1379 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1380 }
1381 }
1382 return rc;
1383
1384#else /* guest real and protected mode */
1385 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1386 NOREF(pVCpu); NOREF(GCPtrPage);
1387 return VINF_SUCCESS;
1388#endif
1389}
1390
1391
1392/**
1393 * Update the tracking of shadowed pages.
1394 *
1395 * @param pVCpu Pointer to the VMCPU.
1396 * @param pShwPage The shadow page.
1397 * @param HCPhys The physical page we is being dereferenced.
1398 * @param iPte Shadow PTE index
1399 * @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1400 */
1401DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1402 RTGCPHYS GCPhysPage)
1403{
1404 PVM pVM = pVCpu->CTX_SUFF(pVM);
1405
1406# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1407 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1408 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1409
1410 /* Use the hint we retrieved from the cached guest PT. */
1411 if (pShwPage->fDirty)
1412 {
1413 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1414
1415 Assert(pShwPage->cPresent);
1416 Assert(pPool->cPresent);
1417 pShwPage->cPresent--;
1418 pPool->cPresent--;
1419
1420 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1421 AssertRelease(pPhysPage);
1422 pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1423 return;
1424 }
1425# else
1426 NOREF(GCPhysPage);
1427# endif
1428
1429 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1430 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1431
1432 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1433 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1434 * 2. write protect all shadowed pages. I.e. implement caching.
1435 */
1436 /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1437
1438 /*
1439 * Find the guest address.
1440 */
1441 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1442 pRam;
1443 pRam = pRam->CTX_SUFF(pNext))
1444 {
1445 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1446 while (iPage-- > 0)
1447 {
1448 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1449 {
1450 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1451
1452 Assert(pShwPage->cPresent);
1453 Assert(pPool->cPresent);
1454 pShwPage->cPresent--;
1455 pPool->cPresent--;
1456
1457 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1458 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1459 return;
1460 }
1461 }
1462 }
1463
1464 for (;;)
1465 AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1466}
1467
1468
1469/**
1470 * Update the tracking of shadowed pages.
1471 *
1472 * @param pVCpu Pointer to the VMCPU.
1473 * @param pShwPage The shadow page.
1474 * @param u16 The top 16-bit of the pPage->HCPhys.
1475 * @param pPage Pointer to the guest page. this will be modified.
1476 * @param iPTDst The index into the shadow table.
1477 */
1478DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1479{
1480 PVM pVM = pVCpu->CTX_SUFF(pVM);
1481
1482 /*
1483 * Just deal with the simple first time here.
1484 */
1485 if (!u16)
1486 {
1487 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1488 u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1489 /* Save the page table index. */
1490 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1491 }
1492 else
1493 u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1494
1495 /* write back */
1496 Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1497 PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1498
1499 /* update statistics. */
1500 pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1501 pShwPage->cPresent++;
1502 if (pShwPage->iFirstPresent > iPTDst)
1503 pShwPage->iFirstPresent = iPTDst;
1504}
1505
1506
1507/**
1508 * Modifies a shadow PTE to account for access handlers.
1509 *
1510 * @param pVM Pointer to the VM.
1511 * @param pPage The page in question.
1512 * @param fPteSrc The shadowed flags of the source PTE. Must include the
1513 * A (accessed) bit so it can be emulated correctly.
1514 * @param pPteDst The shadow PTE (output). This is temporary storage and
1515 * does not need to be set atomically.
1516 */
1517DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1518{
1519 NOREF(pVM);
1520 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1521 * Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1522 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1523 {
1524 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1525#if PGM_SHW_TYPE == PGM_TYPE_EPT
1526 pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1527 pPteDst->n.u1Present = 1;
1528 pPteDst->n.u1Execute = 1;
1529 pPteDst->n.u1IgnorePAT = 1;
1530 pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1531 /* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1532#else
1533 if (fPteSrc & X86_PTE_A)
1534 {
1535 SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage));
1536 SHW_PTE_SET_RO(*pPteDst);
1537 }
1538 else
1539 SHW_PTE_SET(*pPteDst, 0);
1540#endif
1541 }
1542#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1543# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
1544 else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1545 && ( BTH_IS_NP_ACTIVE(pVM)
1546 || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1547# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1548 && pVM->pgm.s.fLessThan52PhysicalAddressBits
1549# endif
1550 )
1551 {
1552 LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1553# if PGM_SHW_TYPE == PGM_TYPE_EPT
1554 /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1555 pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1556 /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1557 pPteDst->n.u1Present = 0;
1558 pPteDst->n.u1Write = 1;
1559 pPteDst->n.u1Execute = 0;
1560 /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1561 pPteDst->n.u3EMT = 7;
1562# else
1563 /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1564 SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P);
1565# endif
1566 }
1567# endif
1568#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1569 else
1570 {
1571 LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1572 SHW_PTE_SET(*pPteDst, 0);
1573 }
1574 /** @todo count these kinds of entries. */
1575}
1576
1577
1578/**
1579 * Creates a 4K shadow page for a guest page.
1580 *
1581 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1582 * physical address. The PdeSrc argument only the flags are used. No page
1583 * structured will be mapped in this function.
1584 *
1585 * @param pVCpu Pointer to the VMCPU.
1586 * @param pPteDst Destination page table entry.
1587 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1588 * Can safely assume that only the flags are being used.
1589 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1590 * @param pShwPage Pointer to the shadow page.
1591 * @param iPTDst The index into the shadow table.
1592 *
1593 * @remark Not used for 2/4MB pages!
1594 */
1595#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1596static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1597 PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1598#else
1599static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1600#endif
1601{
1602 PVM pVM = pVCpu->CTX_SUFF(pVM);
1603 RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1604
1605#if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1606 && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1607 && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1608
1609 if (pShwPage->fDirty)
1610 {
1611 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1612 PGSTPT pGstPT;
1613
1614 /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1615 pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1616 GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1617 pGstPT->a[iPTDst].u = PteSrc.u;
1618 }
1619#else
1620 Assert(!pShwPage->fDirty);
1621#endif
1622
1623#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1624 if ( PteSrc.n.u1Present
1625 && GST_IS_PTE_VALID(pVCpu, PteSrc))
1626#endif
1627 {
1628# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1629 RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1630# endif
1631 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1632
1633 /*
1634 * Find the ram range.
1635 */
1636 PPGMPAGE pPage;
1637 int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1638 if (RT_SUCCESS(rc))
1639 {
1640 /* Ignore ballooned pages.
1641 Don't return errors or use a fatal assert here as part of a
1642 shadow sync range might included ballooned pages. */
1643 if (PGM_PAGE_IS_BALLOONED(pPage))
1644 {
1645 Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */
1646 return;
1647 }
1648
1649#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1650 /* Make the page writable if necessary. */
1651 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1652 && ( PGM_PAGE_IS_ZERO(pPage)
1653# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1654 || ( PteSrc.n.u1Write
1655# else
1656 || ( 1
1657# endif
1658 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1659# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1660 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1661# endif
1662# ifdef VBOX_WITH_PAGE_SHARING
1663 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1664# endif
1665 )
1666 )
1667 )
1668 {
1669 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1670 AssertRC(rc);
1671 }
1672#endif
1673
1674 /*
1675 * Make page table entry.
1676 */
1677 SHWPTE PteDst;
1678# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1679 uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1680# else
1681 uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D;
1682# endif
1683 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1684 PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1685 else
1686 {
1687#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1688 /*
1689 * If the page or page directory entry is not marked accessed,
1690 * we mark the page not present.
1691 */
1692 if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
1693 {
1694 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1695 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1696 SHW_PTE_SET(PteDst, 0);
1697 }
1698 /*
1699 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1700 * when the page is modified.
1701 */
1702 else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1703 {
1704 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1705 SHW_PTE_SET(PteDst,
1706 fGstShwPteFlags
1707 | PGM_PAGE_GET_HCPHYS(pPage)
1708 | PGM_PTFLAGS_TRACK_DIRTY);
1709 SHW_PTE_SET_RO(PteDst);
1710 }
1711 else
1712#endif
1713 {
1714 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1715#if PGM_SHW_TYPE == PGM_TYPE_EPT
1716 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1717 PteDst.n.u1Present = 1;
1718 PteDst.n.u1Write = 1;
1719 PteDst.n.u1Execute = 1;
1720 PteDst.n.u1IgnorePAT = 1;
1721 PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1722 /* PteDst.n.u1Size = 0 */
1723#else
1724 SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage));
1725#endif
1726 }
1727
1728 /*
1729 * Make sure only allocated pages are mapped writable.
1730 */
1731 if ( SHW_PTE_IS_P_RW(PteDst)
1732 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1733 {
1734 /* Still applies to shared pages. */
1735 Assert(!PGM_PAGE_IS_ZERO(pPage));
1736 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1737 Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1738 }
1739 }
1740
1741 /*
1742 * Keep user track up to date.
1743 */
1744 if (SHW_PTE_IS_P(PteDst))
1745 {
1746 if (!SHW_PTE_IS_P(*pPteDst))
1747 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1748 else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1749 {
1750 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst)));
1751 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1752 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1753 }
1754 }
1755 else if (SHW_PTE_IS_P(*pPteDst))
1756 {
1757 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
1758 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1759 }
1760
1761 /*
1762 * Update statistics and commit the entry.
1763 */
1764#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1765 if (!PteSrc.n.u1Global)
1766 pShwPage->fSeenNonGlobal = true;
1767#endif
1768 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1769 return;
1770 }
1771
1772/** @todo count these three different kinds. */
1773 Log2(("SyncPageWorker: invalid address in Pte\n"));
1774 }
1775#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1776 else if (!PteSrc.n.u1Present)
1777 Log2(("SyncPageWorker: page not present in Pte\n"));
1778 else
1779 Log2(("SyncPageWorker: invalid Pte\n"));
1780#endif
1781
1782 /*
1783 * The page is not present or the PTE is bad. Replace the shadow PTE by
1784 * an empty entry, making sure to keep the user tracking up to date.
1785 */
1786 if (SHW_PTE_IS_P(*pPteDst))
1787 {
1788 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
1789 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1790 }
1791 SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1792}
1793
1794
1795/**
1796 * Syncs a guest OS page.
1797 *
1798 * There are no conflicts at this point, neither is there any need for
1799 * page table allocations.
1800 *
1801 * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1802 * When called in AMD64 guest mode, the guest PML4E shall be valid.
1803 *
1804 * @returns VBox status code.
1805 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1806 * @param pVCpu Pointer to the VMCPU.
1807 * @param PdeSrc Page directory entry of the guest.
1808 * @param GCPtrPage Guest context page address.
1809 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1810 * @param uErr Fault error (X86_TRAP_PF_*).
1811 */
1812static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1813{
1814 PVM pVM = pVCpu->CTX_SUFF(pVM);
1815 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
1816 LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1817
1818 PGM_LOCK_ASSERT_OWNER(pVM);
1819
1820#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1821 || PGM_GST_TYPE == PGM_TYPE_PAE \
1822 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
1823 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
1824 && PGM_SHW_TYPE != PGM_TYPE_EPT
1825
1826 /*
1827 * Assert preconditions.
1828 */
1829 Assert(PdeSrc.n.u1Present);
1830 Assert(cPages);
1831# if 0 /* rarely useful; leave for debugging. */
1832 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1833# endif
1834
1835 /*
1836 * Get the shadow PDE, find the shadow page table in the pool.
1837 */
1838# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1839 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1840 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1841
1842 /* Fetch the pgm pool shadow descriptor. */
1843 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1844 Assert(pShwPde);
1845
1846# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1847 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1848 PPGMPOOLPAGE pShwPde = NULL;
1849 PX86PDPAE pPDDst;
1850
1851 /* Fetch the pgm pool shadow descriptor. */
1852 int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1853 AssertRCSuccessReturn(rc2, rc2);
1854 Assert(pShwPde);
1855
1856 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1857 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1858
1859# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1860 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1861 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1862 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1863 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1864
1865 int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1866 AssertRCSuccessReturn(rc2, rc2);
1867 Assert(pPDDst && pPdptDst);
1868 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1869# endif
1870 SHWPDE PdeDst = *pPdeDst;
1871
1872 /*
1873 * - In the guest SMP case we could have blocked while another VCPU reused
1874 * this page table.
1875 * - With W7-64 we may also take this path when the A bit is cleared on
1876 * higher level tables (PDPE/PML4E). The guest does not invalidate the
1877 * relevant TLB entries. If we're write monitoring any page mapped by
1878 * the modified entry, we may end up here with a "stale" TLB entry.
1879 */
1880 if (!PdeDst.n.u1Present)
1881 {
1882 Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1883 AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW),
1884 ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1885 if (uErr & X86_TRAP_PF_P)
1886 PGM_INVL_PG(pVCpu, GCPtrPage);
1887 return VINF_SUCCESS; /* force the instruction to be executed again. */
1888 }
1889
1890 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1891 Assert(pShwPage);
1892
1893# if PGM_GST_TYPE == PGM_TYPE_AMD64
1894 /* Fetch the pgm pool shadow descriptor. */
1895 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1896 Assert(pShwPde);
1897# endif
1898
1899 /*
1900 * Check that the page is present and that the shadow PDE isn't out of sync.
1901 */
1902 const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1903 const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1904 RTGCPHYS GCPhys;
1905 if (!fBigPage)
1906 {
1907 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1908# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1909 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1910 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
1911# endif
1912 }
1913 else
1914 {
1915 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1916# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1917 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1918 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1919# endif
1920 }
1921 /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
1922 if ( fPdeValid
1923 && pShwPage->GCPhys == GCPhys
1924 && PdeSrc.n.u1Present
1925 && PdeSrc.n.u1User == PdeDst.n.u1User
1926 && (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
1927# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1928 && (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute || !GST_IS_NX_ACTIVE(pVCpu))
1929# endif
1930 )
1931 {
1932 /*
1933 * Check that the PDE is marked accessed already.
1934 * Since we set the accessed bit *before* getting here on a #PF, this
1935 * check is only meant for dealing with non-#PF'ing paths.
1936 */
1937 if (PdeSrc.n.u1Accessed)
1938 {
1939 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1940 if (!fBigPage)
1941 {
1942 /*
1943 * 4KB Page - Map the guest page table.
1944 */
1945 PGSTPT pPTSrc;
1946 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1947 if (RT_SUCCESS(rc))
1948 {
1949# ifdef PGM_SYNC_N_PAGES
1950 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1951 if ( cPages > 1
1952 && !(uErr & X86_TRAP_PF_P)
1953 && !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1954 {
1955 /*
1956 * This code path is currently only taken when the caller is PGMTrap0eHandler
1957 * for non-present pages!
1958 *
1959 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1960 * deal with locality.
1961 */
1962 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1963# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1964 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1965 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1966# else
1967 const unsigned offPTSrc = 0;
1968# endif
1969 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1970 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1971 iPTDst = 0;
1972 else
1973 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1974
1975 for (; iPTDst < iPTDstEnd; iPTDst++)
1976 {
1977 const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
1978
1979 if ( pPteSrc->n.u1Present
1980 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
1981 {
1982 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1983 NOREF(GCPtrCurPage);
1984# ifdef VBOX_WITH_RAW_MODE_NOT_R0
1985 /*
1986 * Assuming kernel code will be marked as supervisor - and not as user level
1987 * and executed using a conforming code selector - And marked as readonly.
1988 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
1989 */
1990 PPGMPAGE pPage;
1991 if ( ((PdeSrc.u & pPteSrc->u) & (X86_PTE_RW | X86_PTE_US))
1992 || iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1993 || !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
1994 || ( (pPage = pgmPhysGetPage(pVM, pPteSrc->u & GST_PTE_PG_MASK))
1995 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1996 )
1997# endif /* else: CSAM not active */
1998 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
1999 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2000 GCPtrCurPage, pPteSrc->n.u1Present,
2001 pPteSrc->n.u1Write & PdeSrc.n.u1Write,
2002 pPteSrc->n.u1User & PdeSrc.n.u1User,
2003 (uint64_t)pPteSrc->u,
2004 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2005 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2006 }
2007 }
2008 }
2009 else
2010# endif /* PGM_SYNC_N_PAGES */
2011 {
2012 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2013 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2014 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2015 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2016 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2017 GCPtrPage, PteSrc.n.u1Present,
2018 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2019 PteSrc.n.u1User & PdeSrc.n.u1User,
2020 (uint64_t)PteSrc.u,
2021 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2022 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2023 }
2024 }
2025 else /* MMIO or invalid page: emulated in #PF handler. */
2026 {
2027 LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2028 Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2029 }
2030 }
2031 else
2032 {
2033 /*
2034 * 4/2MB page - lazy syncing shadow 4K pages.
2035 * (There are many causes of getting here, it's no longer only CSAM.)
2036 */
2037 /* Calculate the GC physical address of this 4KB shadow page. */
2038 GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2039 /* Find ram range. */
2040 PPGMPAGE pPage;
2041 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2042 if (RT_SUCCESS(rc))
2043 {
2044 AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2045
2046# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2047 /* Try to make the page writable if necessary. */
2048 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2049 && ( PGM_PAGE_IS_ZERO(pPage)
2050 || ( PdeSrc.n.u1Write
2051 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2052# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2053 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2054# endif
2055# ifdef VBOX_WITH_PAGE_SHARING
2056 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2057# endif
2058 )
2059 )
2060 )
2061 {
2062 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2063 AssertRC(rc);
2064 }
2065# endif
2066
2067 /*
2068 * Make shadow PTE entry.
2069 */
2070 SHWPTE PteDst;
2071 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2072 PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2073 else
2074 SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
2075
2076 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2077 if ( SHW_PTE_IS_P(PteDst)
2078 && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2079 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2080
2081 /* Make sure only allocated pages are mapped writable. */
2082 if ( SHW_PTE_IS_P_RW(PteDst)
2083 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2084 {
2085 /* Still applies to shared pages. */
2086 Assert(!PGM_PAGE_IS_ZERO(pPage));
2087 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2088 Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2089 }
2090
2091 SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2092
2093 /*
2094 * If the page is not flagged as dirty and is writable, then make it read-only
2095 * at PD level, so we can set the dirty bit when the page is modified.
2096 *
2097 * ASSUMES that page access handlers are implemented on page table entry level.
2098 * Thus we will first catch the dirty access and set PDE.D and restart. If
2099 * there is an access handler, we'll trap again and let it work on the problem.
2100 */
2101 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2102 * As for invlpg, it simply frees the whole shadow PT.
2103 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2104 if ( !PdeSrc.b.u1Dirty
2105 && PdeSrc.b.u1Write)
2106 {
2107 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2108 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2109 PdeDst.n.u1Write = 0;
2110 }
2111 else
2112 {
2113 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2114 PdeDst.n.u1Write = PdeSrc.n.u1Write;
2115 }
2116 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2117 Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2118 GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2119 PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2120 }
2121 else
2122 {
2123 LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2124 /** @todo must wipe the shadow page table entry in this
2125 * case. */
2126 }
2127 }
2128 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2129 return VINF_SUCCESS;
2130 }
2131
2132 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2133 }
2134 else if (fPdeValid)
2135 {
2136 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2137 Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2138 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2139 }
2140 else
2141 {
2142/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2143 Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2144 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2145 }
2146
2147 /*
2148 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2149 * Yea, I'm lazy.
2150 */
2151 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2152 ASMAtomicWriteSize(pPdeDst, 0);
2153
2154 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2155 PGM_INVL_VCPU_TLBS(pVCpu);
2156 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2157
2158
2159#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2160 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
2161 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
2162 && !defined(IN_RC)
2163 NOREF(PdeSrc);
2164
2165# ifdef PGM_SYNC_N_PAGES
2166 /*
2167 * Get the shadow PDE, find the shadow page table in the pool.
2168 */
2169# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2170 X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2171
2172# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2173 X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2174
2175# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2176 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2177 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2178 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2179 X86PDEPAE PdeDst;
2180 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2181
2182 int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2183 AssertRCSuccessReturn(rc, rc);
2184 Assert(pPDDst && pPdptDst);
2185 PdeDst = pPDDst->a[iPDDst];
2186# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2187 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2188 PEPTPD pPDDst;
2189 EPTPDE PdeDst;
2190
2191 int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2192 if (rc != VINF_SUCCESS)
2193 {
2194 AssertRC(rc);
2195 return rc;
2196 }
2197 Assert(pPDDst);
2198 PdeDst = pPDDst->a[iPDDst];
2199# endif
2200 /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2201 if (!PdeDst.n.u1Present)
2202 {
2203 AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2204 Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2205 return VINF_SUCCESS; /* force the instruction to be executed again. */
2206 }
2207
2208 /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2209 if (PdeDst.n.u1Size)
2210 {
2211 Assert(pVM->pgm.s.fNestedPaging);
2212 Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2213 return VINF_SUCCESS;
2214 }
2215
2216 /* Mask away the page offset. */
2217 GCPtrPage &= ~((RTGCPTR)0xfff);
2218
2219 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2220 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2221
2222 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
2223 if ( cPages > 1
2224 && !(uErr & X86_TRAP_PF_P)
2225 && !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2226 {
2227 /*
2228 * This code path is currently only taken when the caller is PGMTrap0eHandler
2229 * for non-present pages!
2230 *
2231 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2232 * deal with locality.
2233 */
2234 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2235 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2236 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2237 iPTDst = 0;
2238 else
2239 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2240 for (; iPTDst < iPTDstEnd; iPTDst++)
2241 {
2242 if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2243 {
2244 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2245 | (iPTDst << PAGE_SHIFT));
2246
2247 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2248 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2249 GCPtrCurPage,
2250 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2251 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2252
2253 if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2254 break;
2255 }
2256 else
2257 Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2258 }
2259 }
2260 else
2261# endif /* PGM_SYNC_N_PAGES */
2262 {
2263 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2264 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2265 | (iPTDst << PAGE_SHIFT));
2266
2267 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2268
2269 Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2270 GCPtrPage,
2271 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2272 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2273 }
2274 return VINF_SUCCESS;
2275
2276#else
2277 NOREF(PdeSrc);
2278 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2279 return VERR_PGM_NOT_USED_IN_MODE;
2280#endif
2281}
2282
2283
2284#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2285
2286/**
2287 * CheckPageFault helper for returning a page fault indicating a non-present
2288 * (NP) entry in the page translation structures.
2289 *
2290 * @returns VINF_EM_RAW_GUEST_TRAP.
2291 * @param pVCpu Pointer to the VMCPU.
2292 * @param uErr The error code of the shadow fault. Corrections to
2293 * TRPM's copy will be made if necessary.
2294 * @param GCPtrPage For logging.
2295 * @param uPageFaultLevel For logging.
2296 */
2297DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2298{
2299 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2300 AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2301 AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2302 if (uErr & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
2303 TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD | X86_TRAP_PF_P));
2304
2305 Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2306 return VINF_EM_RAW_GUEST_TRAP;
2307}
2308
2309
2310/**
2311 * CheckPageFault helper for returning a page fault indicating a reserved bit
2312 * (RSVD) error in the page translation structures.
2313 *
2314 * @returns VINF_EM_RAW_GUEST_TRAP.
2315 * @param pVCpu Pointer to the VMCPU.
2316 * @param uErr The error code of the shadow fault. Corrections to
2317 * TRPM's copy will be made if necessary.
2318 * @param GCPtrPage For logging.
2319 * @param uPageFaultLevel For logging.
2320 */
2321DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2322{
2323 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2324 if ((uErr & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
2325 TRPMSetErrorCode(pVCpu, uErr | X86_TRAP_PF_RSVD | X86_TRAP_PF_P);
2326
2327 Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2328 return VINF_EM_RAW_GUEST_TRAP;
2329}
2330
2331
2332/**
2333 * CheckPageFault helper for returning a page protection fault (P).
2334 *
2335 * @returns VINF_EM_RAW_GUEST_TRAP.
2336 * @param pVCpu Pointer to the VMCPU.
2337 * @param uErr The error code of the shadow fault. Corrections to
2338 * TRPM's copy will be made if necessary.
2339 * @param GCPtrPage For logging.
2340 * @param uPageFaultLevel For logging.
2341 */
2342DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2343{
2344 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2345 AssertMsg(uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID), ("%#x\n", uErr));
2346 if ((uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2347 TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) | X86_TRAP_PF_P);
2348
2349 Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2350 return VINF_EM_RAW_GUEST_TRAP;
2351}
2352
2353
2354/**
2355 * Handle dirty bit tracking faults.
2356 *
2357 * @returns VBox status code.
2358 * @param pVCpu Pointer to the VMCPU.
2359 * @param uErr Page fault error code.
2360 * @param pPdeSrc Guest page directory entry.
2361 * @param pPdeDst Shadow page directory entry.
2362 * @param GCPtrPage Guest context page address.
2363 */
2364static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2365 RTGCPTR GCPtrPage)
2366{
2367 PVM pVM = pVCpu->CTX_SUFF(pVM);
2368 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2369 NOREF(uErr);
2370
2371 PGM_LOCK_ASSERT_OWNER(pVM);
2372
2373 /*
2374 * Handle big page.
2375 */
2376 if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2377 {
2378 if ( pPdeDst->n.u1Present
2379 && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2380 {
2381 SHWPDE PdeDst = *pPdeDst;
2382
2383 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2384 Assert(pPdeSrc->b.u1Write);
2385
2386 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2387 * fault again and take this path to only invalidate the entry (see below).
2388 */
2389 PdeDst.n.u1Write = 1;
2390 PdeDst.n.u1Accessed = 1;
2391 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2392 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2393 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2394 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2395 }
2396
2397# ifdef IN_RING0
2398 /* Check for stale TLB entry; only applies to the SMP guest case. */
2399 if ( pVM->cCpus > 1
2400 && pPdeDst->n.u1Write
2401 && pPdeDst->n.u1Accessed)
2402 {
2403 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2404 if (pShwPage)
2405 {
2406 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2407 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2408 if (SHW_PTE_IS_P_RW(*pPteDst))
2409 {
2410 /* Stale TLB entry. */
2411 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2412 PGM_INVL_PG(pVCpu, GCPtrPage);
2413 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2414 }
2415 }
2416 }
2417# endif /* IN_RING0 */
2418 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2419 }
2420
2421 /*
2422 * Map the guest page table.
2423 */
2424 PGSTPT pPTSrc;
2425 int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2426 if (RT_FAILURE(rc))
2427 {
2428 AssertRC(rc);
2429 return rc;
2430 }
2431
2432 if (pPdeDst->n.u1Present)
2433 {
2434 GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2435 const GSTPTE PteSrc = *pPteSrc;
2436
2437#ifdef VBOX_WITH_RAW_MODE_NOT_R0
2438 /* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2439 * Our individual shadow handlers will provide more information and force a fatal exit.
2440 */
2441 if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2442 {
2443 LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2444 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2445 }
2446#endif
2447 /*
2448 * Map shadow page table.
2449 */
2450 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2451 if (pShwPage)
2452 {
2453 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2454 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2455 if (SHW_PTE_IS_P(*pPteDst)) /** @todo Optimize accessed bit emulation? */
2456 {
2457 if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2458 {
2459 PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2460 SHWPTE PteDst = *pPteDst;
2461
2462 LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2463 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2464
2465 Assert(PteSrc.n.u1Write);
2466
2467 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2468 * entry will not harm; write access will simply fault again and
2469 * take this path to only invalidate the entry.
2470 */
2471 if (RT_LIKELY(pPage))
2472 {
2473 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2474 {
2475 //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2476 Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2477 /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2478 SHW_PTE_SET_RO(PteDst);
2479 }
2480 else
2481 {
2482 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2483 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2484 {
2485 rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2486 AssertRC(rc);
2487 }
2488 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2489 SHW_PTE_SET_RW(PteDst);
2490 else
2491 {
2492 /* Still applies to shared pages. */
2493 Assert(!PGM_PAGE_IS_ZERO(pPage));
2494 SHW_PTE_SET_RO(PteDst);
2495 }
2496 }
2497 }
2498 else
2499 SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2500
2501 SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2502 SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2503 PGM_INVL_PG(pVCpu, GCPtrPage);
2504 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2505 }
2506
2507# ifdef IN_RING0
2508 /* Check for stale TLB entry; only applies to the SMP guest case. */
2509 if ( pVM->cCpus > 1
2510 && SHW_PTE_IS_RW(*pPteDst)
2511 && SHW_PTE_IS_A(*pPteDst))
2512 {
2513 /* Stale TLB entry. */
2514 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2515 PGM_INVL_PG(pVCpu, GCPtrPage);
2516 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2517 }
2518# endif
2519 }
2520 }
2521 else
2522 AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2523 }
2524
2525 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2526}
2527
2528#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2529
2530
2531/**
2532 * Sync a shadow page table.
2533 *
2534 * The shadow page table is not present in the shadow PDE.
2535 *
2536 * Handles mapping conflicts.
2537 *
2538 * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2539 * conflict), and Trap0eHandler.
2540 *
2541 * A precondition for this method is that the shadow PDE is not present. The
2542 * caller must take the PGM lock before checking this and continue to hold it
2543 * when calling this method.
2544 *
2545 * @returns VBox status code.
2546 * @param pVCpu Pointer to the VMCPU.
2547 * @param iPD Page directory index.
2548 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
2549 * Assume this is a temporary mapping.
2550 * @param GCPtrPage GC Pointer of the page that caused the fault
2551 */
2552static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2553{
2554 PVM pVM = pVCpu->CTX_SUFF(pVM);
2555 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2556
2557#if 0 /* rarely useful; leave for debugging. */
2558 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2559#endif
2560 LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2561
2562 PGM_LOCK_ASSERT_OWNER(pVM);
2563
2564#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2565 || PGM_GST_TYPE == PGM_TYPE_PAE \
2566 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2567 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
2568 && PGM_SHW_TYPE != PGM_TYPE_EPT
2569
2570 int rc = VINF_SUCCESS;
2571
2572 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2573
2574 /*
2575 * Some input validation first.
2576 */
2577 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2578
2579 /*
2580 * Get the relevant shadow PDE entry.
2581 */
2582# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2583 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2584 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2585
2586 /* Fetch the pgm pool shadow descriptor. */
2587 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2588 Assert(pShwPde);
2589
2590# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2591 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2592 PPGMPOOLPAGE pShwPde = NULL;
2593 PX86PDPAE pPDDst;
2594 PSHWPDE pPdeDst;
2595
2596 /* Fetch the pgm pool shadow descriptor. */
2597 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2598 AssertRCSuccessReturn(rc, rc);
2599 Assert(pShwPde);
2600
2601 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2602 pPdeDst = &pPDDst->a[iPDDst];
2603
2604# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2605 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2606 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2607 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2608 PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2609 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2610 AssertRCSuccessReturn(rc, rc);
2611 Assert(pPDDst);
2612 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2613# endif
2614 SHWPDE PdeDst = *pPdeDst;
2615
2616# if PGM_GST_TYPE == PGM_TYPE_AMD64
2617 /* Fetch the pgm pool shadow descriptor. */
2618 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2619 Assert(pShwPde);
2620# endif
2621
2622# ifndef PGM_WITHOUT_MAPPINGS
2623 /*
2624 * Check for conflicts.
2625 * RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2626 * R3: Simply resolve the conflict.
2627 */
2628 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2629 {
2630 Assert(pgmMapAreMappingsEnabled(pVM));
2631# ifndef IN_RING3
2632 Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2633 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2634 return VERR_ADDRESS_CONFLICT;
2635
2636# else /* IN_RING3 */
2637 PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2638 Assert(pMapping);
2639# if PGM_GST_TYPE == PGM_TYPE_32BIT
2640 rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2641# elif PGM_GST_TYPE == PGM_TYPE_PAE
2642 rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2643# else
2644 AssertFailed(); NOREF(pMapping); /* can't happen for amd64 */
2645# endif
2646 if (RT_FAILURE(rc))
2647 {
2648 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2649 return rc;
2650 }
2651 PdeDst = *pPdeDst;
2652# endif /* IN_RING3 */
2653 }
2654# endif /* !PGM_WITHOUT_MAPPINGS */
2655 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2656
2657 /*
2658 * Sync the page directory entry.
2659 */
2660 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2661 const bool fPageTable = !PdeSrc.b.u1Size || !GST_IS_PSE_ACTIVE(pVCpu);
2662 if ( PdeSrc.n.u1Present
2663 && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2664 {
2665 /*
2666 * Allocate & map the page table.
2667 */
2668 PSHWPT pPTDst;
2669 PPGMPOOLPAGE pShwPage;
2670 RTGCPHYS GCPhys;
2671 if (fPageTable)
2672 {
2673 GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2674# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2675 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2676 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (PAGE_SIZE / 2)));
2677# endif
2678 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2679 pShwPde->idx, iPDDst, false /*fLockPage*/,
2680 &pShwPage);
2681 }
2682 else
2683 {
2684 PGMPOOLACCESS enmAccess;
2685# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2686 const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2687# else
2688 const bool fNoExecute = false;
2689# endif
2690
2691 GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2692# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2693 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2694 GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2695# endif
2696 /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2697 if (PdeSrc.n.u1User)
2698 {
2699 if (PdeSrc.n.u1Write)
2700 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2701 else
2702 enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2703 }
2704 else
2705 {
2706 if (PdeSrc.n.u1Write)
2707 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2708 else
2709 enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2710 }
2711 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2712 pShwPde->idx, iPDDst, false /*fLockPage*/,
2713 &pShwPage);
2714 }
2715 if (rc == VINF_SUCCESS)
2716 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2717 else if (rc == VINF_PGM_CACHED_PAGE)
2718 {
2719 /*
2720 * The PT was cached, just hook it up.
2721 */
2722 if (fPageTable)
2723 PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2724 else
2725 {
2726 PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2727 /* (see explanation and assumptions further down.) */
2728 if ( !PdeSrc.b.u1Dirty
2729 && PdeSrc.b.u1Write)
2730 {
2731 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2732 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2733 PdeDst.b.u1Write = 0;
2734 }
2735 }
2736 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2737 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2738 return VINF_SUCCESS;
2739 }
2740 else if (rc == VERR_PGM_POOL_FLUSHED)
2741 {
2742 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2743 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2744 return VINF_PGM_SYNC_CR3;
2745 }
2746 else
2747 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2748 /** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2749 * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2750 * irrelevant at this point. */
2751 PdeDst.u &= X86_PDE_AVL_MASK;
2752 PdeDst.u |= pShwPage->Core.Key;
2753
2754 /*
2755 * Page directory has been accessed (this is a fault situation, remember).
2756 */
2757 /** @todo
2758 * Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2759 * fault situation. What's more, the Trap0eHandler has already set the
2760 * accessed bit. So, it's actually just VerifyAccessSyncPage which
2761 * might need setting the accessed flag.
2762 *
2763 * The best idea is to leave this change to the caller and add an
2764 * assertion that it's set already. */
2765 pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2766 if (fPageTable)
2767 {
2768 /*
2769 * Page table - 4KB.
2770 *
2771 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2772 */
2773 Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2774 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2775 PGSTPT pPTSrc;
2776 rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2777 if (RT_SUCCESS(rc))
2778 {
2779 /*
2780 * Start by syncing the page directory entry so CSAM's TLB trick works.
2781 */
2782 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
2783 | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2784 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2785 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2786
2787 /*
2788 * Directory/page user or supervisor privilege: (same goes for read/write)
2789 *
2790 * Directory Page Combined
2791 * U/S U/S U/S
2792 * 0 0 0
2793 * 0 1 0
2794 * 1 0 0
2795 * 1 1 1
2796 *
2797 * Simple AND operation. Table listed for completeness.
2798 *
2799 */
2800 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2801# ifdef PGM_SYNC_N_PAGES
2802 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2803 unsigned iPTDst = iPTBase;
2804 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2805 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2806 iPTDst = 0;
2807 else
2808 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2809# else /* !PGM_SYNC_N_PAGES */
2810 unsigned iPTDst = 0;
2811 const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2812# endif /* !PGM_SYNC_N_PAGES */
2813 RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2814 | ((RTGCPTR)iPTDst << PAGE_SHIFT);
2815# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2816 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2817 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2818# else
2819 const unsigned offPTSrc = 0;
2820# endif
2821 for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
2822 {
2823 const unsigned iPTSrc = iPTDst + offPTSrc;
2824 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2825
2826 if (PteSrc.n.u1Present)
2827 {
2828# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2829 /*
2830 * Assuming kernel code will be marked as supervisor - and not as user level
2831 * and executed using a conforming code selector - And marked as readonly.
2832 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
2833 */
2834 PPGMPAGE pPage;
2835 if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW | X86_PTE_US))
2836 || !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
2837 || ( (pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc)))
2838 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2839 )
2840# endif
2841 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2842 Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2843 GCPtrCur,
2844 PteSrc.n.u1Present,
2845 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2846 PteSrc.n.u1User & PdeSrc.n.u1User,
2847 (uint64_t)PteSrc.u,
2848 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2849 (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2850 }
2851 /* else: the page table was cleared by the pool */
2852 } /* for PTEs */
2853 }
2854 }
2855 else
2856 {
2857 /*
2858 * Big page - 2/4MB.
2859 *
2860 * We'll walk the ram range list in parallel and optimize lookups.
2861 * We will only sync one shadow page table at a time.
2862 */
2863 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2864
2865 /**
2866 * @todo It might be more efficient to sync only a part of the 4MB
2867 * page (similar to what we do for 4KB PDs).
2868 */
2869
2870 /*
2871 * Start by syncing the page directory entry.
2872 */
2873 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2874 | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2875
2876 /*
2877 * If the page is not flagged as dirty and is writable, then make it read-only
2878 * at PD level, so we can set the dirty bit when the page is modified.
2879 *
2880 * ASSUMES that page access handlers are implemented on page table entry level.
2881 * Thus we will first catch the dirty access and set PDE.D and restart. If
2882 * there is an access handler, we'll trap again and let it work on the problem.
2883 */
2884 /** @todo move the above stuff to a section in the PGM documentation. */
2885 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2886 if ( !PdeSrc.b.u1Dirty
2887 && PdeSrc.b.u1Write)
2888 {
2889 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2890 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2891 PdeDst.b.u1Write = 0;
2892 }
2893 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2894 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2895
2896 /*
2897 * Fill the shadow page table.
2898 */
2899 /* Get address and flags from the source PDE. */
2900 SHWPTE PteDstBase;
2901 SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
2902
2903 /* Loop thru the entries in the shadow PT. */
2904 const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2905 Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2906 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2907 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2908 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2909 unsigned iPTDst = 0;
2910 while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2911 && !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2912 {
2913 if (pRam && GCPhys >= pRam->GCPhys)
2914 {
2915# ifndef PGM_WITH_A20
2916 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2917# endif
2918 do
2919 {
2920 /* Make shadow PTE. */
2921# ifdef PGM_WITH_A20
2922 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2923# else
2924 PPGMPAGE pPage = &pRam->aPages[iHCPage];
2925# endif
2926 SHWPTE PteDst;
2927
2928# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2929 /* Try to make the page writable if necessary. */
2930 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2931 && ( PGM_PAGE_IS_ZERO(pPage)
2932 || ( SHW_PTE_IS_RW(PteDstBase)
2933 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2934# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2935 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2936# endif
2937# ifdef VBOX_WITH_PAGE_SHARING
2938 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2939# endif
2940 && !PGM_PAGE_IS_BALLOONED(pPage))
2941 )
2942 )
2943 {
2944 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2945 AssertRCReturn(rc, rc);
2946 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2947 break;
2948 }
2949# endif
2950
2951 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2952 PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
2953 else if (PGM_PAGE_IS_BALLOONED(pPage))
2954 SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
2955# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2956 /*
2957 * Assuming kernel code will be marked as supervisor and not as user level and executed
2958 * using a conforming code selector. Don't check for readonly, as that implies the whole
2959 * 4MB can be code or readonly data. Linux enables write access for its large pages.
2960 */
2961 else if ( !PdeSrc.n.u1User
2962 && CSAMDoesPageNeedScanning(pVM, GCPtr | (iPTDst << SHW_PT_SHIFT)))
2963 SHW_PTE_SET(PteDst, 0);
2964# endif
2965 else
2966 SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase));
2967
2968 /* Only map writable pages writable. */
2969 if ( SHW_PTE_IS_P_RW(PteDst)
2970 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2971 {
2972 /* Still applies to shared pages. */
2973 Assert(!PGM_PAGE_IS_ZERO(pPage));
2974 SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2975 Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
2976 }
2977
2978 if (SHW_PTE_IS_P(PteDst))
2979 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2980
2981 /* commit it (not atomic, new table) */
2982 pPTDst->a[iPTDst] = PteDst;
2983 Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2984 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
2985 SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
2986
2987 /* advance */
2988 GCPhys += PAGE_SIZE;
2989 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
2990# ifndef PGM_WITH_A20
2991 iHCPage++;
2992# endif
2993 iPTDst++;
2994 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2995 && GCPhys <= pRam->GCPhysLast);
2996
2997 /* Advance ram range list. */
2998 while (pRam && GCPhys > pRam->GCPhysLast)
2999 pRam = pRam->CTX_SUFF(pNext);
3000 }
3001 else if (pRam)
3002 {
3003 Log(("Invalid pages at %RGp\n", GCPhys));
3004 do
3005 {
3006 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3007 GCPhys += PAGE_SIZE;
3008 iPTDst++;
3009 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3010 && GCPhys < pRam->GCPhys);
3011 PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3012 }
3013 else
3014 {
3015 Log(("Invalid pages at %RGp (2)\n", GCPhys));
3016 for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3017 SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3018 }
3019 } /* while more PTEs */
3020 } /* 4KB / 4MB */
3021 }
3022 else
3023 AssertRelease(!PdeDst.n.u1Present);
3024
3025 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3026 if (RT_FAILURE(rc))
3027 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3028 return rc;
3029
3030#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
3031 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
3032 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
3033 && !defined(IN_RC)
3034 NOREF(iPDSrc); NOREF(pPDSrc);
3035
3036 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3037
3038 /*
3039 * Validate input a little bit.
3040 */
3041 int rc = VINF_SUCCESS;
3042# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3043 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3044 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3045
3046 /* Fetch the pgm pool shadow descriptor. */
3047 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3048 Assert(pShwPde);
3049
3050# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3051 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3052 PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3053 PX86PDPAE pPDDst;
3054 PSHWPDE pPdeDst;
3055
3056 /* Fetch the pgm pool shadow descriptor. */
3057 rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3058 AssertRCSuccessReturn(rc, rc);
3059 Assert(pShwPde);
3060
3061 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3062 pPdeDst = &pPDDst->a[iPDDst];
3063
3064# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3065 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3066 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3067 PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3068 PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3069 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3070 AssertRCSuccessReturn(rc, rc);
3071 Assert(pPDDst);
3072 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3073
3074 /* Fetch the pgm pool shadow descriptor. */
3075 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3076 Assert(pShwPde);
3077
3078# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3079 const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3080 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3081 PEPTPD pPDDst;
3082 PEPTPDPT pPdptDst;
3083
3084 rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3085 if (rc != VINF_SUCCESS)
3086 {
3087 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3088 AssertRC(rc);
3089 return rc;
3090 }
3091 Assert(pPDDst);
3092 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3093
3094 /* Fetch the pgm pool shadow descriptor. */
3095 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3096 Assert(pShwPde);
3097# endif
3098 SHWPDE PdeDst = *pPdeDst;
3099
3100 Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3101 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3102
3103# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3104 if (BTH_IS_NP_ACTIVE(pVM))
3105 {
3106 /* Check if we allocated a big page before for this 2 MB range. */
3107 PPGMPAGE pPage;
3108 rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3109 if (RT_SUCCESS(rc))
3110 {
3111 RTHCPHYS HCPhys = NIL_RTHCPHYS;
3112 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3113 {
3114 if (PGM_A20_IS_ENABLED(pVCpu))
3115 {
3116 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3117 AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3118 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3119 }
3120 else
3121 {
3122 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3123 pVM->pgm.s.cLargePagesDisabled++;
3124 }
3125 }
3126 else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3127 && PGM_A20_IS_ENABLED(pVCpu))
3128 {
3129 /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3130 rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3131 if (RT_SUCCESS(rc))
3132 {
3133 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3134 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3135 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3136 }
3137 }
3138 else if ( PGMIsUsingLargePages(pVM)
3139 && PGM_A20_IS_ENABLED(pVCpu))
3140 {
3141 rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3142 if (RT_SUCCESS(rc))
3143 {
3144 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3145 Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3146 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3147 }
3148 else
3149 LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3150 }
3151
3152 if (HCPhys != NIL_RTHCPHYS)
3153 {
3154 PdeDst.u &= X86_PDE_AVL_MASK;
3155 PdeDst.u |= HCPhys;
3156 PdeDst.n.u1Present = 1;
3157 PdeDst.n.u1Write = 1;
3158 PdeDst.b.u1Size = 1;
3159# if PGM_SHW_TYPE == PGM_TYPE_EPT
3160 PdeDst.n.u1Execute = 1;
3161 PdeDst.b.u1IgnorePAT = 1;
3162 PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3163# else
3164 PdeDst.n.u1User = 1;
3165# endif
3166 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3167
3168 Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3169 /* Add a reference to the first page only. */
3170 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3171
3172 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3173 return VINF_SUCCESS;
3174 }
3175 }
3176 }
3177# endif /* HC_ARCH_BITS == 64 */
3178
3179 /*
3180 * Allocate & map the page table.
3181 */
3182 PSHWPT pPTDst;
3183 PPGMPOOLPAGE pShwPage;
3184 RTGCPHYS GCPhys;
3185
3186 /* Virtual address = physical address */
3187 GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3188 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3189 PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/,
3190 &pShwPage);
3191 if ( rc == VINF_SUCCESS
3192 || rc == VINF_PGM_CACHED_PAGE)
3193 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3194 else
3195 {
3196 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3197 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3198 }
3199
3200 if (rc == VINF_SUCCESS)
3201 {
3202 /* New page table; fully set it up. */
3203 Assert(pPTDst);
3204
3205 /* Mask away the page offset. */
3206 GCPtrPage &= ~(RTGCPTR)PAGE_OFFSET_MASK;
3207
3208 for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3209 {
3210 RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3211 | (iPTDst << PAGE_SHIFT));
3212
3213 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3214 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3215 GCPtrCurPage,
3216 SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3217 SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3218
3219 if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
3220 break;
3221 }
3222 }
3223 else
3224 rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3225
3226 /* Save the new PDE. */
3227 PdeDst.u &= X86_PDE_AVL_MASK;
3228 PdeDst.u |= pShwPage->Core.Key;
3229 PdeDst.n.u1Present = 1;
3230 PdeDst.n.u1Write = 1;
3231# if PGM_SHW_TYPE == PGM_TYPE_EPT
3232 PdeDst.n.u1Execute = 1;
3233# else
3234 PdeDst.n.u1User = 1;
3235 PdeDst.n.u1Accessed = 1;
3236# endif
3237 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3238
3239 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3240 if (RT_FAILURE(rc))
3241 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3242 return rc;
3243
3244#else
3245 NOREF(iPDSrc); NOREF(pPDSrc);
3246 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3247 return VERR_PGM_NOT_USED_IN_MODE;
3248#endif
3249}
3250
3251
3252
3253/**
3254 * Prefetch a page/set of pages.
3255 *
3256 * Typically used to sync commonly used pages before entering raw mode
3257 * after a CR3 reload.
3258 *
3259 * @returns VBox status code.
3260 * @param pVCpu Pointer to the VMCPU.
3261 * @param GCPtrPage Page to invalidate.
3262 */
3263PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3264{
3265#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3266 || PGM_GST_TYPE == PGM_TYPE_REAL \
3267 || PGM_GST_TYPE == PGM_TYPE_PROT \
3268 || PGM_GST_TYPE == PGM_TYPE_PAE \
3269 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3270 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
3271 && PGM_SHW_TYPE != PGM_TYPE_EPT
3272
3273 /*
3274 * Check that all Guest levels thru the PDE are present, getting the
3275 * PD and PDE in the processes.
3276 */
3277 int rc = VINF_SUCCESS;
3278# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3279# if PGM_GST_TYPE == PGM_TYPE_32BIT
3280 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3281 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3282# elif PGM_GST_TYPE == PGM_TYPE_PAE
3283 unsigned iPDSrc;
3284 X86PDPE PdpeSrc;
3285 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3286 if (!pPDSrc)
3287 return VINF_SUCCESS; /* not present */
3288# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3289 unsigned iPDSrc;
3290 PX86PML4E pPml4eSrc;
3291 X86PDPE PdpeSrc;
3292 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3293 if (!pPDSrc)
3294 return VINF_SUCCESS; /* not present */
3295# endif
3296 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3297# else
3298 PGSTPD pPDSrc = NULL;
3299 const unsigned iPDSrc = 0;
3300 GSTPDE PdeSrc;
3301
3302 PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3303 PdeSrc.n.u1Present = 1;
3304 PdeSrc.n.u1Write = 1;
3305 PdeSrc.n.u1Accessed = 1;
3306 PdeSrc.n.u1User = 1;
3307# endif
3308
3309 if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3310 {
3311 PVM pVM = pVCpu->CTX_SUFF(pVM);
3312 pgmLock(pVM);
3313
3314# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3315 const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3316# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3317 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3318 PX86PDPAE pPDDst;
3319 X86PDEPAE PdeDst;
3320# if PGM_GST_TYPE != PGM_TYPE_PAE
3321 X86PDPE PdpeSrc;
3322
3323 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3324 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3325# endif
3326 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3327 if (rc != VINF_SUCCESS)
3328 {
3329 pgmUnlock(pVM);
3330 AssertRC(rc);
3331 return rc;
3332 }
3333 Assert(pPDDst);
3334 PdeDst = pPDDst->a[iPDDst];
3335
3336# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3337 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3338 PX86PDPAE pPDDst;
3339 X86PDEPAE PdeDst;
3340
3341# if PGM_GST_TYPE == PGM_TYPE_PROT
3342 /* AMD-V nested paging */
3343 X86PML4E Pml4eSrc;
3344 X86PDPE PdpeSrc;
3345 PX86PML4E pPml4eSrc = &Pml4eSrc;
3346
3347 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3348 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
3349 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
3350# endif
3351
3352 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3353 if (rc != VINF_SUCCESS)
3354 {
3355 pgmUnlock(pVM);
3356 AssertRC(rc);
3357 return rc;
3358 }
3359 Assert(pPDDst);
3360 PdeDst = pPDDst->a[iPDDst];
3361# endif
3362 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3363 {
3364 if (!PdeDst.n.u1Present)
3365 {
3366 /** @todo r=bird: This guy will set the A bit on the PDE,
3367 * probably harmless. */
3368 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3369 }
3370 else
3371 {
3372 /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3373 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3374 * makes no sense to prefetch more than one page.
3375 */
3376 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3377 if (RT_SUCCESS(rc))
3378 rc = VINF_SUCCESS;
3379 }
3380 }
3381 pgmUnlock(pVM);
3382 }
3383 return rc;
3384
3385#elif PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3386 NOREF(pVCpu); NOREF(GCPtrPage);
3387 return VINF_SUCCESS; /* ignore */
3388#else
3389 AssertCompile(0);
3390#endif
3391}
3392
3393
3394
3395
3396/**
3397 * Syncs a page during a PGMVerifyAccess() call.
3398 *
3399 * @returns VBox status code (informational included).
3400 * @param pVCpu Pointer to the VMCPU.
3401 * @param GCPtrPage The address of the page to sync.
3402 * @param fPage The effective guest page flags.
3403 * @param uErr The trap error code.
3404 * @remarks This will normally never be called on invalid guest page
3405 * translation entries.
3406 */
3407PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3408{
3409 PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3410
3411 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3412
3413 Assert(!pVM->pgm.s.fNestedPaging);
3414#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3415 || PGM_GST_TYPE == PGM_TYPE_REAL \
3416 || PGM_GST_TYPE == PGM_TYPE_PROT \
3417 || PGM_GST_TYPE == PGM_TYPE_PAE \
3418 || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3419 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
3420 && PGM_SHW_TYPE != PGM_TYPE_EPT
3421
3422# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3423 if (!(fPage & X86_PTE_US))
3424 {
3425 /*
3426 * Mark this page as safe.
3427 */
3428 /** @todo not correct for pages that contain both code and data!! */
3429 Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3430 CSAMMarkPage(pVM, GCPtrPage, true);
3431 }
3432# endif
3433
3434 /*
3435 * Get guest PD and index.
3436 */
3437 /** @todo Performance: We've done all this a jiffy ago in the
3438 * PGMGstGetPage call. */
3439# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3440# if PGM_GST_TYPE == PGM_TYPE_32BIT
3441 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3442 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3443
3444# elif PGM_GST_TYPE == PGM_TYPE_PAE
3445 unsigned iPDSrc = 0;
3446 X86PDPE PdpeSrc;
3447 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3448 if (RT_UNLIKELY(!pPDSrc))
3449 {
3450 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3451 return VINF_EM_RAW_GUEST_TRAP;
3452 }
3453
3454# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3455 unsigned iPDSrc = 0; /* shut up gcc */
3456 PX86PML4E pPml4eSrc = NULL; /* ditto */
3457 X86PDPE PdpeSrc;
3458 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3459 if (RT_UNLIKELY(!pPDSrc))
3460 {
3461 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3462 return VINF_EM_RAW_GUEST_TRAP;
3463 }
3464# endif
3465
3466# else /* !PGM_WITH_PAGING */
3467 PGSTPD pPDSrc = NULL;
3468 const unsigned iPDSrc = 0;
3469# endif /* !PGM_WITH_PAGING */
3470 int rc = VINF_SUCCESS;
3471
3472 pgmLock(pVM);
3473
3474 /*
3475 * First check if the shadow pd is present.
3476 */
3477# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3478 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3479
3480# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3481 PX86PDEPAE pPdeDst;
3482 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3483 PX86PDPAE pPDDst;
3484# if PGM_GST_TYPE != PGM_TYPE_PAE
3485 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3486 X86PDPE PdpeSrc;
3487 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3488# endif
3489 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3490 if (rc != VINF_SUCCESS)
3491 {
3492 pgmUnlock(pVM);
3493 AssertRC(rc);
3494 return rc;
3495 }
3496 Assert(pPDDst);
3497 pPdeDst = &pPDDst->a[iPDDst];
3498
3499# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3500 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3501 PX86PDPAE pPDDst;
3502 PX86PDEPAE pPdeDst;
3503
3504# if PGM_GST_TYPE == PGM_TYPE_PROT
3505 /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3506 X86PML4E Pml4eSrc;
3507 X86PDPE PdpeSrc;
3508 PX86PML4E pPml4eSrc = &Pml4eSrc;
3509 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
3510 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
3511# endif
3512
3513 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3514 if (rc != VINF_SUCCESS)
3515 {
3516 pgmUnlock(pVM);
3517 AssertRC(rc);
3518 return rc;
3519 }
3520 Assert(pPDDst);
3521 pPdeDst = &pPDDst->a[iPDDst];
3522# endif
3523
3524 if (!pPdeDst->n.u1Present)
3525 {
3526 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3527 if (rc != VINF_SUCCESS)
3528 {
3529 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3530 pgmUnlock(pVM);
3531 AssertRC(rc);
3532 return rc;
3533 }
3534 }
3535
3536# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3537 /* Check for dirty bit fault */
3538 rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3539 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3540 Log(("PGMVerifyAccess: success (dirty)\n"));
3541 else
3542# endif
3543 {
3544# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3545 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3546# else
3547 GSTPDE PdeSrc;
3548 PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3549 PdeSrc.n.u1Present = 1;
3550 PdeSrc.n.u1Write = 1;
3551 PdeSrc.n.u1Accessed = 1;
3552 PdeSrc.n.u1User = 1;
3553# endif
3554
3555 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3556 if (uErr & X86_TRAP_PF_US)
3557 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3558 else /* supervisor */
3559 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3560
3561 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3562 if (RT_SUCCESS(rc))
3563 {
3564 /* Page was successfully synced */
3565 Log2(("PGMVerifyAccess: success (sync)\n"));
3566 rc = VINF_SUCCESS;
3567 }
3568 else
3569 {
3570 Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3571 rc = VINF_EM_RAW_GUEST_TRAP;
3572 }
3573 }
3574 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3575 pgmUnlock(pVM);
3576 return rc;
3577
3578#else /* PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_NESTED */
3579
3580 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3581 return VERR_PGM_NOT_USED_IN_MODE;
3582#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_NESTED */
3583}
3584
3585
3586/**
3587 * Syncs the paging hierarchy starting at CR3.
3588 *
3589 * @returns VBox status code, no specials.
3590 * @param pVCpu Pointer to the VMCPU.
3591 * @param cr0 Guest context CR0 register.
3592 * @param cr3 Guest context CR3 register. Not subjected to the A20
3593 * mask.
3594 * @param cr4 Guest context CR4 register.
3595 * @param fGlobal Including global page directories or not
3596 */
3597PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3598{
3599 PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3600 NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3601
3602 LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3603
3604#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3605
3606 pgmLock(pVM);
3607
3608# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3609 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3610 if (pPool->cDirtyPages)
3611 pgmPoolResetDirtyPages(pVM);
3612# endif
3613
3614 /*
3615 * Update page access handlers.
3616 * The virtual are always flushed, while the physical are only on demand.
3617 * WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3618 * have to look into that later because it will have a bad influence on the performance.
3619 * @note SvL: There's no need for that. Just invalidate the virtual range(s).
3620 * bird: Yes, but that won't work for aliases.
3621 */
3622 /** @todo this MUST go away. See @bugref{1557}. */
3623 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3624 PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3625 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3626 pgmUnlock(pVM);
3627#endif /* !NESTED && !EPT */
3628
3629#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3630 /*
3631 * Nested / EPT - almost no work.
3632 */
3633 Assert(!pgmMapAreMappingsEnabled(pVM));
3634 return VINF_SUCCESS;
3635
3636#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3637 /*
3638 * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3639 * out the shadow parts when the guest modifies its tables.
3640 */
3641 Assert(!pgmMapAreMappingsEnabled(pVM));
3642 return VINF_SUCCESS;
3643
3644#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3645
3646# ifndef PGM_WITHOUT_MAPPINGS
3647 /*
3648 * Check for and resolve conflicts with our guest mappings if they
3649 * are enabled and not fixed.
3650 */
3651 if (pgmMapAreMappingsFloating(pVM))
3652 {
3653 int rc = pgmMapResolveConflicts(pVM);
3654 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3);
3655 if (rc == VINF_PGM_SYNC_CR3)
3656 {
3657 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3658 return VINF_PGM_SYNC_CR3;
3659 }
3660 }
3661# else
3662 Assert(!pgmMapAreMappingsEnabled(pVM));
3663# endif
3664 return VINF_SUCCESS;
3665#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3666}
3667
3668
3669
3670
3671#ifdef VBOX_STRICT
3672# ifdef IN_RC
3673# undef AssertMsgFailed
3674# define AssertMsgFailed Log
3675# endif
3676
3677/**
3678 * Checks that the shadow page table is in sync with the guest one.
3679 *
3680 * @returns The number of errors.
3681 * @param pVM The virtual machine.
3682 * @param pVCpu Pointer to the VMCPU.
3683 * @param cr3 Guest context CR3 register.
3684 * @param cr4 Guest context CR4 register.
3685 * @param GCPtr Where to start. Defaults to 0.
3686 * @param cb How much to check. Defaults to everything.
3687 */
3688PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3689{
3690 NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3691#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3692 return 0;
3693#else
3694 unsigned cErrors = 0;
3695 PVM pVM = pVCpu->CTX_SUFF(pVM);
3696 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3697
3698# if PGM_GST_TYPE == PGM_TYPE_PAE
3699 /** @todo currently broken; crashes below somewhere */
3700 AssertFailed();
3701# endif
3702
3703# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3704 || PGM_GST_TYPE == PGM_TYPE_PAE \
3705 || PGM_GST_TYPE == PGM_TYPE_AMD64
3706
3707 bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3708 PPGMCPU pPGM = &pVCpu->pgm.s;
3709 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3710 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3711# ifndef IN_RING0
3712 RTHCPHYS HCPhys; /* general usage. */
3713# endif
3714 int rc;
3715
3716 /*
3717 * Check that the Guest CR3 and all its mappings are correct.
3718 */
3719 AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3720 ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3721 false);
3722# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3723# if PGM_GST_TYPE == PGM_TYPE_32BIT
3724 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3725# else
3726 rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3727# endif
3728 AssertRCReturn(rc, 1);
3729 HCPhys = NIL_RTHCPHYS;
3730 rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3731 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3732# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3733 pgmGstGet32bitPDPtr(pVCpu);
3734 RTGCPHYS GCPhys;
3735 rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3736 AssertRCReturn(rc, 1);
3737 AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3738# endif
3739# endif /* !IN_RING0 */
3740
3741 /*
3742 * Get and check the Shadow CR3.
3743 */
3744# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3745 unsigned cPDEs = X86_PG_ENTRIES;
3746 unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3747# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3748# if PGM_GST_TYPE == PGM_TYPE_32BIT
3749 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3750# else
3751 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3752# endif
3753 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3754# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3755 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3756 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3757# endif
3758 if (cb != ~(RTGCPTR)0)
3759 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3760
3761/** @todo call the other two PGMAssert*() functions. */
3762
3763# if PGM_GST_TYPE == PGM_TYPE_AMD64
3764 unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3765
3766 for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3767 {
3768 PPGMPOOLPAGE pShwPdpt = NULL;
3769 PX86PML4E pPml4eSrc;
3770 PX86PML4E pPml4eDst;
3771 RTGCPHYS GCPhysPdptSrc;
3772
3773 pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3774 pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3775
3776 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3777 if (!pPml4eDst->n.u1Present)
3778 {
3779 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3780 continue;
3781 }
3782
3783 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3784 GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
3785
3786 if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3787 {
3788 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3789 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3790 cErrors++;
3791 continue;
3792 }
3793
3794 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3795 {
3796 AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3797 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3798 cErrors++;
3799 continue;
3800 }
3801
3802 if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3803 || pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3804 || pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3805 {
3806 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3807 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3808 cErrors++;
3809 continue;
3810 }
3811# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3812 {
3813# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3814
3815# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
3816 /*
3817 * Check the PDPTEs too.
3818 */
3819 unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3820
3821 for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3822 {
3823 unsigned iPDSrc = 0; /* initialized to shut up gcc */
3824 PPGMPOOLPAGE pShwPde = NULL;
3825 PX86PDPE pPdpeDst;
3826 RTGCPHYS GCPhysPdeSrc;
3827 X86PDPE PdpeSrc;
3828 PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
3829# if PGM_GST_TYPE == PGM_TYPE_PAE
3830 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3831 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3832# else
3833 PX86PML4E pPml4eSrcIgn;
3834 PX86PDPT pPdptDst;
3835 PX86PDPAE pPDDst;
3836 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3837
3838 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3839 if (rc != VINF_SUCCESS)
3840 {
3841 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3842 GCPtr += 512 * _2M;
3843 continue; /* next PDPTE */
3844 }
3845 Assert(pPDDst);
3846# endif
3847 Assert(iPDSrc == 0);
3848
3849 pPdpeDst = &pPdptDst->a[iPdpt];
3850
3851 if (!pPdpeDst->n.u1Present)
3852 {
3853 GCPtr += 512 * _2M;
3854 continue; /* next PDPTE */
3855 }
3856
3857 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3858 GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
3859
3860 if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3861 {
3862 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3863 GCPtr += 512 * _2M;
3864 cErrors++;
3865 continue;
3866 }
3867
3868 if (GCPhysPdeSrc != pShwPde->GCPhys)
3869 {
3870# if PGM_GST_TYPE == PGM_TYPE_AMD64
3871 AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3872# else
3873 AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3874# endif
3875 GCPtr += 512 * _2M;
3876 cErrors++;
3877 continue;
3878 }
3879
3880# if PGM_GST_TYPE == PGM_TYPE_AMD64
3881 if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3882 || pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3883 || pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3884 {
3885 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3886 GCPtr += 512 * _2M;
3887 cErrors++;
3888 continue;
3889 }
3890# endif
3891
3892# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3893 {
3894# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3895# if PGM_GST_TYPE == PGM_TYPE_32BIT
3896 GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3897# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3898 PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3899# endif
3900# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3901 /*
3902 * Iterate the shadow page directory.
3903 */
3904 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3905 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3906
3907 for (;
3908 iPDDst < cPDEs;
3909 iPDDst++, GCPtr += cIncrement)
3910 {
3911# if PGM_SHW_TYPE == PGM_TYPE_PAE
3912 const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3913# else
3914 const SHWPDE PdeDst = pPDDst->a[iPDDst];
3915# endif
3916 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3917 {
3918 Assert(pgmMapAreMappingsEnabled(pVM));
3919 if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3920 {
3921 AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3922 cErrors++;
3923 continue;
3924 }
3925 }
3926 else if ( (PdeDst.u & X86_PDE_P)
3927 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3928 )
3929 {
3930 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3931 PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3932 if (!pPoolPage)
3933 {
3934 AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3935 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3936 cErrors++;
3937 continue;
3938 }
3939 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3940
3941 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
3942 {
3943 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3944 GCPtr, (uint64_t)PdeDst.u));
3945 cErrors++;
3946 }
3947
3948 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
3949 {
3950 AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3951 GCPtr, (uint64_t)PdeDst.u));
3952 cErrors++;
3953 }
3954
3955 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3956 if (!PdeSrc.n.u1Present)
3957 {
3958 AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3959 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3960 cErrors++;
3961 continue;
3962 }
3963
3964 if ( !PdeSrc.b.u1Size
3965 || !fBigPagesSupported)
3966 {
3967 GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
3968# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3969 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (PAGE_SIZE / 2)));
3970# endif
3971 }
3972 else
3973 {
3974# if PGM_GST_TYPE == PGM_TYPE_32BIT
3975 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3976 {
3977 AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3978 GCPtr, (uint64_t)PdeSrc.u));
3979 cErrors++;
3980 continue;
3981 }
3982# endif
3983 GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3984# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3985 GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
3986# endif
3987 }
3988
3989 if ( pPoolPage->enmKind
3990 != (!PdeSrc.b.u1Size || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3991 {
3992 AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3993 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3994 cErrors++;
3995 }
3996
3997 PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
3998 if (!pPhysPage)
3999 {
4000 AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4001 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4002 cErrors++;
4003 continue;
4004 }
4005
4006 if (GCPhysGst != pPoolPage->GCPhys)
4007 {
4008 AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4009 GCPhysGst, pPoolPage->GCPhys, GCPtr));
4010 cErrors++;
4011 continue;
4012 }
4013
4014 if ( !PdeSrc.b.u1Size
4015 || !fBigPagesSupported)
4016 {
4017 /*
4018 * Page Table.
4019 */
4020 const GSTPT *pPTSrc;
4021 rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1)),
4022 &pPTSrc);
4023 if (RT_FAILURE(rc))
4024 {
4025 AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4026 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4027 cErrors++;
4028 continue;
4029 }
4030 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
4031 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
4032 {
4033 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4034 // (This problem will go away when/if we shadow multiple CR3s.)
4035 AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4036 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4037 cErrors++;
4038 continue;
4039 }
4040 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4041 {
4042 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4043 GCPtr, (uint64_t)PdeDst.u));
4044 cErrors++;
4045 continue;
4046 }
4047
4048 /* iterate the page table. */
4049# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4050 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4051 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4052# else
4053 const unsigned offPTSrc = 0;
4054# endif
4055 for (unsigned iPT = 0, off = 0;
4056 iPT < RT_ELEMENTS(pPTDst->a);
4057 iPT++, off += PAGE_SIZE)
4058 {
4059 const SHWPTE PteDst = pPTDst->a[iPT];
4060
4061 /* skip not-present and dirty tracked entries. */
4062 if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4063 continue;
4064 Assert(SHW_PTE_IS_P(PteDst));
4065
4066 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4067 if (!PteSrc.n.u1Present)
4068 {
4069# ifdef IN_RING3
4070 PGMAssertHandlerAndFlagsInSync(pVM);
4071 DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE
4072 | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3,
4073 0, 0, UINT64_MAX, 99, NULL);
4074# endif
4075 AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4076 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4077 (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4078 cErrors++;
4079 continue;
4080 }
4081
4082 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
4083# if 1 /** @todo sync accessed bit properly... */
4084 fIgnoreFlags |= X86_PTE_A;
4085# endif
4086
4087 /* match the physical addresses */
4088 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4089 GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4090
4091# ifdef IN_RING3
4092 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4093 if (RT_FAILURE(rc))
4094 {
4095 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4096 {
4097 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4098 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4099 cErrors++;
4100 continue;
4101 }
4102 }
4103 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4104 {
4105 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4106 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4107 cErrors++;
4108 continue;
4109 }
4110# endif
4111
4112 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4113 if (!pPhysPage)
4114 {
4115# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4116 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4117 {
4118 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4119 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4120 cErrors++;
4121 continue;
4122 }
4123# endif
4124 if (SHW_PTE_IS_RW(PteDst))
4125 {
4126 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4127 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4128 cErrors++;
4129 }
4130 fIgnoreFlags |= X86_PTE_RW;
4131 }
4132 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4133 {
4134 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4135 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4136 cErrors++;
4137 continue;
4138 }
4139
4140 /* flags */
4141 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4142 {
4143 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4144 {
4145 if (SHW_PTE_IS_RW(PteDst))
4146 {
4147 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4148 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4149 cErrors++;
4150 continue;
4151 }
4152 fIgnoreFlags |= X86_PTE_RW;
4153 }
4154 else
4155 {
4156 if ( SHW_PTE_IS_P(PteDst)
4157# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
4158 && !PGM_PAGE_IS_MMIO(pPhysPage)
4159# endif
4160 )
4161 {
4162 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4163 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4164 cErrors++;
4165 continue;
4166 }
4167 fIgnoreFlags |= X86_PTE_P;
4168 }
4169 }
4170 else
4171 {
4172 if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4173 {
4174 if (SHW_PTE_IS_RW(PteDst))
4175 {
4176 AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4177 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4178 cErrors++;
4179 continue;
4180 }
4181 if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4182 {
4183 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4184 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4185 cErrors++;
4186 continue;
4187 }
4188 if (SHW_PTE_IS_D(PteDst))
4189 {
4190 AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4191 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4192 cErrors++;
4193 }
4194# if 0 /** @todo sync access bit properly... */
4195 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4196 {
4197 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4198 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4199 cErrors++;
4200 }
4201 fIgnoreFlags |= X86_PTE_RW;
4202# else
4203 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4204# endif
4205 }
4206 else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4207 {
4208 /* access bit emulation (not implemented). */
4209 if (PteSrc.n.u1Accessed || SHW_PTE_IS_P(PteDst))
4210 {
4211 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4212 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4213 cErrors++;
4214 continue;
4215 }
4216 if (!SHW_PTE_IS_A(PteDst))
4217 {
4218 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4219 GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4220 cErrors++;
4221 }
4222 fIgnoreFlags |= X86_PTE_P;
4223 }
4224# ifdef DEBUG_sandervl
4225 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4226# endif
4227 }
4228
4229 if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4230 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4231 )
4232 {
4233 AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4234 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4235 fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4236 cErrors++;
4237 continue;
4238 }
4239 } /* foreach PTE */
4240 }
4241 else
4242 {
4243 /*
4244 * Big Page.
4245 */
4246 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4247 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4248 {
4249 if (PdeDst.n.u1Write)
4250 {
4251 AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4252 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4253 cErrors++;
4254 continue;
4255 }
4256 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4257 {
4258 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4259 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4260 cErrors++;
4261 continue;
4262 }
4263# if 0 /** @todo sync access bit properly... */
4264 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4265 {
4266 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4267 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4268 cErrors++;
4269 }
4270 fIgnoreFlags |= X86_PTE_RW;
4271# else
4272 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4273# endif
4274 }
4275 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4276 {
4277 /* access bit emulation (not implemented). */
4278 if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
4279 {
4280 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4281 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4282 cErrors++;
4283 continue;
4284 }
4285 if (!PdeDst.n.u1Accessed)
4286 {
4287 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4288 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4289 cErrors++;
4290 }
4291 fIgnoreFlags |= X86_PTE_P;
4292 }
4293
4294 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4295 {
4296 AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4297 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4298 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4299 cErrors++;
4300 }
4301
4302 /* iterate the page table. */
4303 for (unsigned iPT = 0, off = 0;
4304 iPT < RT_ELEMENTS(pPTDst->a);
4305 iPT++, off += PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + PAGE_SIZE))
4306 {
4307 const SHWPTE PteDst = pPTDst->a[iPT];
4308
4309 if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4310 {
4311 AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4312 GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4313 cErrors++;
4314 }
4315
4316 /* skip not-present entries. */
4317 if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4318 continue;
4319
4320 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
4321
4322 /* match the physical addresses */
4323 HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4324
4325# ifdef IN_RING3
4326 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4327 if (RT_FAILURE(rc))
4328 {
4329 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4330 {
4331 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4332 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4333 cErrors++;
4334 }
4335 }
4336 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4337 {
4338 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4339 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4340 cErrors++;
4341 continue;
4342 }
4343# endif
4344 pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4345 if (!pPhysPage)
4346 {
4347# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4348 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4349 {
4350 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4351 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4352 cErrors++;
4353 continue;
4354 }
4355# endif
4356 if (SHW_PTE_IS_RW(PteDst))
4357 {
4358 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4359 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4360 cErrors++;
4361 }
4362 fIgnoreFlags |= X86_PTE_RW;
4363 }
4364 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4365 {
4366 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4367 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4368 cErrors++;
4369 continue;
4370 }
4371
4372 /* flags */
4373 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4374 {
4375 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4376 {
4377 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4378 {
4379 if (SHW_PTE_IS_RW(PteDst))
4380 {
4381 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4382 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4383 cErrors++;
4384 continue;
4385 }
4386 fIgnoreFlags |= X86_PTE_RW;
4387 }
4388 }
4389 else
4390 {
4391 if ( SHW_PTE_IS_P(PteDst)
4392# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
4393 && !PGM_PAGE_IS_MMIO(pPhysPage)
4394# endif
4395 )
4396 {
4397 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4398 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4399 cErrors++;
4400 continue;
4401 }
4402 fIgnoreFlags |= X86_PTE_P;
4403 }
4404 }
4405
4406 if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4407 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4408 )
4409 {
4410 AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4411 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4412 fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4413 cErrors++;
4414 continue;
4415 }
4416 } /* for each PTE */
4417 }
4418 }
4419 /* not present */
4420
4421 } /* for each PDE */
4422
4423 } /* for each PDPTE */
4424
4425 } /* for each PML4E */
4426
4427# ifdef DEBUG
4428 if (cErrors)
4429 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4430# endif
4431# endif /* GST is in {32BIT, PAE, AMD64} */
4432 return cErrors;
4433#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4434}
4435#endif /* VBOX_STRICT */
4436
4437
4438/**
4439 * Sets up the CR3 for shadow paging
4440 *
4441 * @returns Strict VBox status code.
4442 * @retval VINF_SUCCESS.
4443 *
4444 * @param pVCpu Pointer to the VMCPU.
4445 * @param GCPhysCR3 The physical address in the CR3 register. (A20
4446 * mask already applied.)
4447 */
4448PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4449{
4450 PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4451
4452 /* Update guest paging info. */
4453#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4454 || PGM_GST_TYPE == PGM_TYPE_PAE \
4455 || PGM_GST_TYPE == PGM_TYPE_AMD64
4456
4457 LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4458 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4459
4460 /*
4461 * Map the page CR3 points at.
4462 */
4463 RTHCPTR HCPtrGuestCR3;
4464 RTHCPHYS HCPhysGuestCR3;
4465 pgmLock(pVM);
4466 PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
4467 AssertReturn(pPageCR3, VERR_PGM_INVALID_CR3_ADDR);
4468 HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4469 /** @todo this needs some reworking wrt. locking? */
4470# if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4471 HCPtrGuestCR3 = NIL_RTHCPTR;
4472 int rc = VINF_SUCCESS;
4473# else
4474 int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3); /** @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4475# endif
4476 pgmUnlock(pVM);
4477 if (RT_SUCCESS(rc))
4478 {
4479 rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4480 if (RT_SUCCESS(rc))
4481 {
4482# ifdef IN_RC
4483 PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4484# endif
4485# if PGM_GST_TYPE == PGM_TYPE_32BIT
4486 pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4487# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4488 pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4489# endif
4490 pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4491
4492# elif PGM_GST_TYPE == PGM_TYPE_PAE
4493 unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4494 pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4495# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4496 pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4497# endif
4498 pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4499 LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4500
4501 /*
4502 * Map the 4 PDs too.
4503 */
4504 PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4505 RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4506 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4507 {
4508 pVCpu->pgm.s.aGstPaePdpeRegs[i].u = pGuestPDPT->a[i].u;
4509 if (pGuestPDPT->a[i].n.u1Present)
4510 {
4511 RTHCPTR HCPtr;
4512 RTHCPHYS HCPhys;
4513 RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, pGuestPDPT->a[i].u & X86_PDPE_PG_MASK);
4514 pgmLock(pVM);
4515 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4516 AssertReturn(pPage, VERR_PGM_INVALID_PDPE_ADDR);
4517 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4518# if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4519 HCPtr = NIL_RTHCPTR;
4520 int rc2 = VINF_SUCCESS;
4521# else
4522 int rc2 = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
4523# endif
4524 pgmUnlock(pVM);
4525 if (RT_SUCCESS(rc2))
4526 {
4527 rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4528 AssertRCReturn(rc, rc);
4529
4530 pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4531# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4532 pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4533# endif
4534 pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4535 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4536# ifdef IN_RC
4537 PGM_INVL_PG(pVCpu, GCPtr);
4538# endif
4539 continue;
4540 }
4541 AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4542 }
4543
4544 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4545# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4546 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4547# endif
4548 pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4549 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4550# ifdef IN_RC
4551 PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4552# endif
4553 }
4554
4555# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4556 pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4557# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4558 pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4559# endif
4560# endif
4561 }
4562 else
4563 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4564 }
4565 else
4566 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4567
4568#else /* prot/real stub */
4569 int rc = VINF_SUCCESS;
4570#endif
4571
4572 /* Update shadow paging info for guest modes with paging (32, pae, 64). */
4573# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4574 || PGM_SHW_TYPE == PGM_TYPE_PAE \
4575 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4576 && ( PGM_GST_TYPE != PGM_TYPE_REAL \
4577 && PGM_GST_TYPE != PGM_TYPE_PROT))
4578
4579 Assert(!pVM->pgm.s.fNestedPaging);
4580 PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4581
4582 /*
4583 * Update the shadow root page as well since that's not fixed.
4584 */
4585 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4586 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4587 PPGMPOOLPAGE pNewShwPageCR3;
4588
4589 pgmLock(pVM);
4590
4591# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4592 if (pPool->cDirtyPages)
4593 pgmPoolResetDirtyPages(pVM);
4594# endif
4595
4596 Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4597 rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
4598 NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/,
4599 &pNewShwPageCR3);
4600 AssertFatalRC(rc);
4601 rc = VINF_SUCCESS;
4602
4603# ifdef IN_RC
4604 /*
4605 * WARNING! We can't deal with jumps to ring 3 in the code below as the
4606 * state will be inconsistent! Flush important things now while
4607 * we still can and then make sure there are no ring-3 calls.
4608 */
4609# ifdef VBOX_WITH_REM
4610 REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4611# endif
4612 VMMRZCallRing3Disable(pVCpu);
4613# endif
4614
4615 pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4616# ifdef IN_RING0
4617 pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4618 pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4619# elif defined(IN_RC)
4620 pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4621 pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4622# else
4623 pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4624 pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4625# endif
4626
4627# ifndef PGM_WITHOUT_MAPPINGS
4628 /*
4629 * Apply all hypervisor mappings to the new CR3.
4630 * Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4631 * make sure we check for conflicts in the new CR3 root.
4632 */
4633# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4634 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) || VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4635# endif
4636 rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4637 AssertRCReturn(rc, rc);
4638# endif
4639
4640 /* Set the current hypervisor CR3. */
4641 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4642 SELMShadowCR3Changed(pVM, pVCpu);
4643
4644# ifdef IN_RC
4645 /* NOTE: The state is consistent again. */
4646 VMMRZCallRing3Enable(pVCpu);
4647# endif
4648
4649 /* Clean up the old CR3 root. */
4650 if ( pOldShwPageCR3
4651 && pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4652 {
4653 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4654# ifndef PGM_WITHOUT_MAPPINGS
4655 /* Remove the hypervisor mappings from the shadow page table. */
4656 pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4657# endif
4658 /* Mark the page as unlocked; allow flushing again. */
4659 pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4660
4661 pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4662 }
4663 pgmUnlock(pVM);
4664# else
4665 NOREF(GCPhysCR3);
4666# endif
4667
4668 return rc;
4669}
4670
4671/**
4672 * Unmaps the shadow CR3.
4673 *
4674 * @returns VBox status, no specials.
4675 * @param pVCpu Pointer to the VMCPU.
4676 */
4677PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4678{
4679 LogFlow(("UnmapCR3\n"));
4680
4681 int rc = VINF_SUCCESS;
4682 PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4683
4684 /*
4685 * Update guest paging info.
4686 */
4687#if PGM_GST_TYPE == PGM_TYPE_32BIT
4688 pVCpu->pgm.s.pGst32BitPdR3 = 0;
4689# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4690 pVCpu->pgm.s.pGst32BitPdR0 = 0;
4691# endif
4692 pVCpu->pgm.s.pGst32BitPdRC = 0;
4693
4694#elif PGM_GST_TYPE == PGM_TYPE_PAE
4695 pVCpu->pgm.s.pGstPaePdptR3 = 0;
4696# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4697 pVCpu->pgm.s.pGstPaePdptR0 = 0;
4698# endif
4699 pVCpu->pgm.s.pGstPaePdptRC = 0;
4700 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4701 {
4702 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4703# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4704 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4705# endif
4706 pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4707 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4708 }
4709
4710#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4711 pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4712# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4713 pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4714# endif
4715
4716#else /* prot/real mode stub */
4717 /* nothing to do */
4718#endif
4719
4720#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4721 /*
4722 * Update shadow paging info.
4723 */
4724# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4725 || PGM_SHW_TYPE == PGM_TYPE_PAE \
4726 || PGM_SHW_TYPE == PGM_TYPE_AMD64))
4727
4728# if PGM_GST_TYPE != PGM_TYPE_REAL
4729 Assert(!pVM->pgm.s.fNestedPaging);
4730# endif
4731
4732 pgmLock(pVM);
4733
4734# ifndef PGM_WITHOUT_MAPPINGS
4735 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4736 /* Remove the hypervisor mappings from the shadow page table. */
4737 pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4738# endif
4739
4740 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4741 {
4742 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4743
4744# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4745 if (pPool->cDirtyPages)
4746 pgmPoolResetDirtyPages(pVM);
4747# endif
4748
4749 /* Mark the page as unlocked; allow flushing again. */
4750 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4751
4752 pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4753 pVCpu->pgm.s.pShwPageCR3R3 = 0;
4754 pVCpu->pgm.s.pShwPageCR3R0 = 0;
4755 pVCpu->pgm.s.pShwPageCR3RC = 0;
4756 }
4757 pgmUnlock(pVM);
4758# endif
4759#endif /* !IN_RC*/
4760
4761 return rc;
4762}
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette