VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 13933

Last change on this file since 13933 was 13933, checked in by vboxsync, 16 years ago

PGM: PAE guest mappings, assert GCPtr being 32-bit.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 193.7 KB
Line 
1/* $Id: PGMAllBth.h 13933 2008-11-06 18:55:03Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * This file is a big challenge!
6 */
7
8/*
9 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
10 *
11 * This file is part of VirtualBox Open Source Edition (OSE), as
12 * available from http://www.virtualbox.org. This file is free software;
13 * you can redistribute it and/or modify it under the terms of the GNU
14 * General Public License (GPL) as published by the Free Software
15 * Foundation, in version 2 as it comes in the "COPYING" file of the
16 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20 * Clara, CA 95054 USA or visit http://www.sun.com if you need
21 * additional information or have any questions.
22 */
23
24/*******************************************************************************
25* Internal Functions *
26*******************************************************************************/
27__BEGIN_DECLS
28PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
33PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36#ifdef VBOX_STRICT
37PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38#endif
39#ifdef PGMPOOL_WITH_USER_TRACKING
40DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41#endif
42__END_DECLS
43
44
45/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
46#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
47# error "Invalid combination; PAE guest implies PAE shadow"
48#endif
49
50#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
51 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
52# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
53#endif
54
55#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
56 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
57# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
58#endif
59
60#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
61 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
62# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
63#endif
64
65#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
66# define PGM_WITHOUT_MAPPINGS
67#endif
68
69
70#ifndef IN_RING3
71/**
72 * #PF Handler for raw-mode guest execution.
73 *
74 * @returns VBox status code (appropriate for trap handling and GC return).
75 * @param pVM VM Handle.
76 * @param uErr The trap error code.
77 * @param pRegFrame Trap register frame.
78 * @param pvFault The fault address.
79 */
80PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
81{
82# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
83 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
84 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
85
86# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
87 /*
88 * Hide the instruction fetch trap indicator for now.
89 */
90 /** @todo NXE will change this and we must fix NXE in the switcher too! */
91 if (uErr & X86_TRAP_PF_ID)
92 {
93 uErr &= ~X86_TRAP_PF_ID;
94 TRPMSetErrorCode(pVM, uErr);
95 }
96# endif
97
98 /*
99 * Get PDs.
100 */
101 int rc;
102# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
103# if PGM_GST_TYPE == PGM_TYPE_32BIT
104 const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
105 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVM->pgm.s);
106
107# elif PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
108
109# if PGM_GST_TYPE == PGM_TYPE_PAE
110 unsigned iPDSrc;
111 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc, NULL);
112
113# elif PGM_GST_TYPE == PGM_TYPE_AMD64
114 unsigned iPDSrc;
115 PX86PML4E pPml4eSrc;
116 X86PDPE PdpeSrc;
117 PGSTPD pPDSrc;
118
119 pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
120 Assert(pPml4eSrc);
121# endif
122 /* Quick check for a valid guest trap. */
123 if (!pPDSrc)
124 {
125# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
126 LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)(((RTGCUINTPTR)pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVM) & X86_CR3_PAGE_MASK));
127# else
128 LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVM) & X86_CR3_PAGE_MASK));
129# endif
130 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2GuestTrap; });
131 TRPMSetErrorCode(pVM, uErr);
132 return VINF_EM_RAW_GUEST_TRAP;
133 }
134# endif
135
136# else /* !PGM_WITH_PAGING */
137 PGSTPD pPDSrc = NULL;
138 const unsigned iPDSrc = 0;
139# endif /* !PGM_WITH_PAGING */
140
141
142# if PGM_SHW_TYPE == PGM_TYPE_32BIT
143 const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
144 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
145
146# elif PGM_SHW_TYPE == PGM_TYPE_PAE
147 const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
148 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
149
150# if PGM_GST_TYPE == PGM_TYPE_PAE
151 /* Did we mark the PDPT as not present in SyncCR3? */
152 unsigned iPdpte = ((RTGCUINTPTR)pvFault >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
153 if (!pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPdpte].n.u1Present)
154 pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPdpte].n.u1Present = 1;
155
156# endif
157
158# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
159 const unsigned iPDDst = (((RTGCUINTPTR)pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
160 PX86PDPAE pPDDst;
161# if PGM_GST_TYPE == PGM_TYPE_PROT
162 /* AMD-V nested paging */
163 X86PML4E Pml4eSrc;
164 X86PDPE PdpeSrc;
165 PX86PML4E pPml4eSrc = &Pml4eSrc;
166
167 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
168 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
169 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
170# endif
171
172 rc = PGMShwSyncLongModePDPtr(pVM, (RTGCUINTPTR)pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
173 if (rc != VINF_SUCCESS)
174 {
175 AssertRC(rc);
176 return rc;
177 }
178 Assert(pPDDst);
179
180# elif PGM_SHW_TYPE == PGM_TYPE_EPT
181 const unsigned iPDDst = (((RTGCUINTPTR)pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
182 PEPTPD pPDDst;
183
184 rc = PGMShwGetEPTPDPtr(pVM, (RTGCUINTPTR)pvFault, NULL, &pPDDst);
185 if (rc != VINF_SUCCESS)
186 {
187 AssertRC(rc);
188 return rc;
189 }
190 Assert(pPDDst);
191# endif
192
193# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
194 /*
195 * If we successfully correct the write protection fault due to dirty bit
196 * tracking, or this page fault is a genuine one, then return immediately.
197 */
198 STAM_PROFILE_START(&pVM->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
199 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
200 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
201 if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
202 || rc == VINF_EM_RAW_GUEST_TRAP)
203 {
204 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
205 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVM->pgm.s.StatRZTrap0eTime2GuestTrap; });
206 LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
207 return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
208 }
209
210 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0ePD[iPDSrc]);
211# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
212
213 /*
214 * A common case is the not-present error caused by lazy page table syncing.
215 *
216 * It is IMPORTANT that we weed out any access to non-present shadow PDEs here
217 * so we can safely assume that the shadow PT is present when calling SyncPage later.
218 *
219 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
220 * of mapping conflict and defer to SyncCR3 in R3.
221 * (Again, we do NOT support access handlers for non-present guest pages.)
222 *
223 */
224# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
225 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
226# else
227 GSTPDE PdeSrc;
228 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
229 PdeSrc.n.u1Present = 1;
230 PdeSrc.n.u1Write = 1;
231 PdeSrc.n.u1Accessed = 1;
232 PdeSrc.n.u1User = 1;
233# endif
234 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
235 && !pPDDst->a[iPDDst].n.u1Present
236 && PdeSrc.n.u1Present
237 )
238
239 {
240 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2SyncPT; });
241 STAM_PROFILE_START(&pVM->pgm.s.StatRZTrap0eTimeSyncPT, f);
242 LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
243 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
244 if (RT_SUCCESS(rc))
245 {
246 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeSyncPT, f);
247 return rc;
248 }
249 Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
250 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
251 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeSyncPT, f);
252 return VINF_PGM_SYNC_CR3;
253 }
254
255# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
256 /*
257 * Check if this address is within any of our mappings.
258 *
259 * This is *very* fast and it's gonna save us a bit of effort below and prevent
260 * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
261 * (BTW, it's impossible to have physical access handlers in a mapping.)
262 */
263 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
264 {
265 STAM_PROFILE_START(&pVM->pgm.s.StatRZTrap0eTimeMapping, a);
266 PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
267 for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
268 {
269 if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
270 break;
271 if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
272 {
273 /*
274 * The first thing we check is if we've got an undetected conflict.
275 */
276 if (!pVM->pgm.s.fMappingsFixed)
277 {
278 unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
279 while (iPT-- > 0)
280 if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
281 {
282 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eConflicts);
283 Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
284 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
285 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeMapping, a);
286 return VINF_PGM_SYNC_CR3;
287 }
288 }
289
290 /*
291 * Check if the fault address is in a virtual page access handler range.
292 */
293 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
294 if ( pCur
295 && (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key < pCur->cb
296 && uErr & X86_TRAP_PF_RW)
297 {
298# ifdef IN_RC
299 STAM_PROFILE_START(&pCur->Stat, h);
300 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key);
301 STAM_PROFILE_STOP(&pCur->Stat, h);
302# else
303 AssertFailed();
304 rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
305# endif
306 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersMapping);
307 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeMapping, a);
308 return rc;
309 }
310
311 /*
312 * Pretend we're not here and let the guest handle the trap.
313 */
314 TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
315 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eGuestPFMapping);
316 LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
317 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeMapping, a);
318 return VINF_EM_RAW_GUEST_TRAP;
319 }
320 }
321 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeMapping, a);
322 } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
323# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
324
325 /*
326 * Check if this fault address is flagged for special treatment,
327 * which means we'll have to figure out the physical address and
328 * check flags associated with it.
329 *
330 * ASSUME that we can limit any special access handling to pages
331 * in page tables which the guest believes to be present.
332 */
333 if (PdeSrc.n.u1Present)
334 {
335 RTGCPHYS GCPhys = NIL_RTGCPHYS;
336
337# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
338# if PGM_GST_TYPE == PGM_TYPE_AMD64
339 bool fBigPagesSupported = true;
340# else
341 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
342# endif
343 if ( PdeSrc.b.u1Size
344 && fBigPagesSupported)
345 GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc)
346 | ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
347 else
348 {
349 PGSTPT pPTSrc;
350 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
351 if (RT_SUCCESS(rc))
352 {
353 unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
354 if (pPTSrc->a[iPTESrc].n.u1Present)
355 GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
356 }
357 }
358# else
359 /* No paging so the fault address is the physical address */
360 GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
361# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
362
363 /*
364 * If we have a GC address we'll check if it has any flags set.
365 */
366 if (GCPhys != NIL_RTGCPHYS)
367 {
368 STAM_PROFILE_START(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
369
370 PPGMPAGE pPage;
371 rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
372 if (RT_SUCCESS(rc))
373 {
374 if ( PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage)
375 || PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage))
376 {
377 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
378 {
379 /*
380 * Physical page access handler.
381 */
382 const RTGCPHYS GCPhysFault = GCPhys | ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
383 PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
384 if (pCur)
385 {
386# ifdef PGM_SYNC_N_PAGES
387 /*
388 * If the region is write protected and we got a page not present fault, then sync
389 * the pages. If the fault was caused by a read, then restart the instruction.
390 * In case of write access continue to the GC write handler.
391 *
392 * ASSUMES that there is only one handler per page or that they have similar write properties.
393 */
394 if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
395 && !(uErr & X86_TRAP_PF_P))
396 {
397 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
398 if ( RT_FAILURE(rc)
399 || !(uErr & X86_TRAP_PF_RW)
400 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
401 {
402 AssertRC(rc);
403 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersOutOfSync);
404 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
405 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
406 return rc;
407 }
408 }
409# endif
410
411 AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
412 || (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
413 ("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
414
415# if defined(IN_RC) || defined(IN_RING0)
416 if (pCur->CTX_SUFF(pfnHandler))
417 {
418 STAM_PROFILE_START(&pCur->Stat, h);
419 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pCur->CTX_SUFF(pvUser));
420 STAM_PROFILE_STOP(&pCur->Stat, h);
421 }
422 else
423# endif
424 rc = VINF_EM_RAW_EMULATE_INSTR;
425 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersPhysical);
426 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
427 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2HndPhys; });
428 return rc;
429 }
430 }
431# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
432 else
433 {
434# ifdef PGM_SYNC_N_PAGES
435 /*
436 * If the region is write protected and we got a page not present fault, then sync
437 * the pages. If the fault was caused by a read, then restart the instruction.
438 * In case of write access continue to the GC write handler.
439 */
440 if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
441 && !(uErr & X86_TRAP_PF_P))
442 {
443 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
444 if ( RT_FAILURE(rc)
445 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
446 || !(uErr & X86_TRAP_PF_RW))
447 {
448 AssertRC(rc);
449 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersOutOfSync);
450 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
451 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
452 return rc;
453 }
454 }
455# endif
456 /*
457 * Ok, it's an virtual page access handler.
458 *
459 * Since it's faster to search by address, we'll do that first
460 * and then retry by GCPhys if that fails.
461 */
462 /** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
463 /** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
464 * page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
465 */
466 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
467 if (pCur)
468 {
469 AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key < pCur->cb)
470 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
471 || !(uErr & X86_TRAP_PF_P)
472 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
473 ("Unexpected trap for virtual handler: %RGv (phys=%RGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
474
475 if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key < pCur->cb
476 && ( uErr & X86_TRAP_PF_RW
477 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
478 {
479# ifdef IN_RC
480 STAM_PROFILE_START(&pCur->Stat, h);
481 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key);
482 STAM_PROFILE_STOP(&pCur->Stat, h);
483# else
484 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
485# endif
486 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersVirtual);
487 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
488 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2HndVirt; });
489 return rc;
490 }
491 /* Unhandled part of a monitored page */
492 }
493 else
494 {
495 /* Check by physical address. */
496 PPGMVIRTHANDLER pCur;
497 unsigned iPage;
498 rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
499 &pCur, &iPage);
500 Assert(RT_SUCCESS(rc) || !pCur);
501 if ( pCur
502 && ( uErr & X86_TRAP_PF_RW
503 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
504 {
505 Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
506# ifdef IN_RC
507 RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->Core.Key & PAGE_OFFSET_MASK);
508 Assert(off < pCur->cb);
509 STAM_PROFILE_START(&pCur->Stat, h);
510 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
511 STAM_PROFILE_STOP(&pCur->Stat, h);
512# else
513 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
514# endif
515 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersVirtualByPhys);
516 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
517 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2HndVirt; });
518 return rc;
519 }
520 }
521 }
522# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
523
524 /*
525 * There is a handled area of the page, but this fault doesn't belong to it.
526 * We must emulate the instruction.
527 *
528 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
529 * we first check if this was a page-not-present fault for a page with only
530 * write access handlers. Restart the instruction if it wasn't a write access.
531 */
532 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersUnhandled);
533
534 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
535 && !(uErr & X86_TRAP_PF_P))
536 {
537 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
538 if ( RT_FAILURE(rc)
539 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
540 || !(uErr & X86_TRAP_PF_RW))
541 {
542 AssertRC(rc);
543 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersOutOfSync);
544 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
545 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
546 return rc;
547 }
548 }
549
550 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
551 * It's writing to an unhandled part of the LDT page several million times.
552 */
553 rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
554 LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%RHp%s%s\n",
555 rc, pPage->HCPhys,
556 PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ? " phys" : "",
557 PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ? " virt" : ""));
558 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
559 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2HndUnhandled; });
560 return rc;
561 } /* if any kind of handler */
562
563# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
564 if (uErr & X86_TRAP_PF_P)
565 {
566 /*
567 * The page isn't marked, but it might still be monitored by a virtual page access handler.
568 * (ASSUMES no temporary disabling of virtual handlers.)
569 */
570 /** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
571 * we should correct both the shadow page table and physical memory flags, and not only check for
572 * accesses within the handler region but for access to pages with virtual handlers. */
573 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
574 if (pCur)
575 {
576 AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key < pCur->cb)
577 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
578 || !(uErr & X86_TRAP_PF_P)
579 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
580 ("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
581
582 if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key < pCur->cb
583 && ( uErr & X86_TRAP_PF_RW
584 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
585 {
586# ifdef IN_RC
587 STAM_PROFILE_START(&pCur->Stat, h);
588 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->Core.Key);
589 STAM_PROFILE_STOP(&pCur->Stat, h);
590# else
591 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
592# endif
593 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersVirtualUnmarked);
594 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
595 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2HndVirt; });
596 return rc;
597 }
598 }
599 }
600# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
601 }
602 else
603 {
604 /* When the guest accesses invalid physical memory (e.g. probing of RAM or accessing a remapped MMIO range), then we'll fall
605 * back to the recompiler to emulate the instruction.
606 */
607 LogFlow(("pgmPhysGetPageEx %RGp failed with %Rrc\n", GCPhys, rc));
608 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eHandlersInvalid);
609 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
610 return VINF_EM_RAW_EMULATE_INSTR;
611 }
612
613 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeHandlers, b);
614
615# ifdef PGM_OUT_OF_SYNC_IN_GC
616 /*
617 * We are here only if page is present in Guest page tables and trap is not handled
618 * by our handlers.
619 * Check it for page out-of-sync situation.
620 */
621 STAM_PROFILE_START(&pVM->pgm.s.StatRZTrap0eTimeOutOfSync, c);
622
623 if (!(uErr & X86_TRAP_PF_P))
624 {
625 /*
626 * Page is not present in our page tables.
627 * Try to sync it!
628 * BTW, fPageShw is invalid in this branch!
629 */
630 if (uErr & X86_TRAP_PF_US)
631 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
632 else /* supervisor */
633 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
634
635# if defined(LOG_ENABLED) && !defined(IN_RING0)
636 RTGCPHYS GCPhys;
637 uint64_t fPageGst;
638 PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
639 Log(("Page out of sync: %RGv eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%RGp scan=%d\n",
640 pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
641# endif /* LOG_ENABLED */
642
643# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
644 if (CPUMGetGuestCPL(pVM, pRegFrame) == 0)
645 {
646 uint64_t fPageGst;
647 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
648 if ( RT_SUCCESS(rc)
649 && !(fPageGst & X86_PTE_US))
650 {
651 /* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
652 if ( pvFault == (RTGCPTR)pRegFrame->eip
653 || (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
654# ifdef CSAM_DETECT_NEW_CODE_PAGES
655 || ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
656 && CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
657# endif /* CSAM_DETECT_NEW_CODE_PAGES */
658 )
659 {
660 LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
661 rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
662 if (rc != VINF_SUCCESS)
663 {
664 /*
665 * CSAM needs to perform a job in ring 3.
666 *
667 * Sync the page before going to the host context; otherwise we'll end up in a loop if
668 * CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
669 */
670 LogFlow(("CSAM ring 3 job\n"));
671 int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
672 AssertRC(rc2);
673
674 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeOutOfSync, c);
675 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2CSAM; });
676 return rc;
677 }
678 }
679# ifdef CSAM_DETECT_NEW_CODE_PAGES
680 else if ( uErr == X86_TRAP_PF_RW
681 && pRegFrame->ecx >= 0x100 /* early check for movswd count */
682 && pRegFrame->ecx < 0x10000)
683 {
684 /* In case of a write to a non-present supervisor shadow page, we'll take special precautions
685 * to detect loading of new code pages.
686 */
687
688 /*
689 * Decode the instruction.
690 */
691 RTGCPTR PC;
692 rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
693 if (rc == VINF_SUCCESS)
694 {
695 DISCPUSTATE Cpu;
696 uint32_t cbOp;
697 rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
698
699 /* For now we'll restrict this to rep movsw/d instructions */
700 if ( rc == VINF_SUCCESS
701 && Cpu.pCurInstr->opcode == OP_MOVSWD
702 && (Cpu.prefix & PREFIX_REP))
703 {
704 CSAMMarkPossibleCodePage(pVM, pvFault);
705 }
706 }
707 }
708# endif /* CSAM_DETECT_NEW_CODE_PAGES */
709
710 /*
711 * Mark this page as safe.
712 */
713 /** @todo not correct for pages that contain both code and data!! */
714 Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
715 CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
716 }
717 }
718# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
719 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
720 if (RT_SUCCESS(rc))
721 {
722 /* The page was successfully synced, return to the guest. */
723 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeOutOfSync, c);
724 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2OutOfSync; });
725 return VINF_SUCCESS;
726 }
727 }
728 else
729 {
730 /*
731 * A side effect of not flushing global PDEs are out of sync pages due
732 * to physical monitored regions, that are no longer valid.
733 * Assume for now it only applies to the read/write flag
734 */
735 if (RT_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
736 {
737 if (uErr & X86_TRAP_PF_US)
738 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
739 else /* supervisor */
740 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
741
742
743 /*
744 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
745 */
746 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
747 if (RT_SUCCESS(rc))
748 {
749 /*
750 * Page was successfully synced, return to guest.
751 */
752# ifdef VBOX_STRICT
753 RTGCPHYS GCPhys;
754 uint64_t fPageGst;
755 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
756 Assert(RT_SUCCESS(rc) && fPageGst & X86_PTE_RW);
757 LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
758
759 uint64_t fPageShw;
760 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
761 AssertMsg(RT_SUCCESS(rc) && fPageShw & X86_PTE_RW, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
762# endif /* VBOX_STRICT */
763 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeOutOfSync, c);
764 STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
765 return VINF_SUCCESS;
766 }
767
768 /* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
769 if ( CPUMGetGuestCPL(pVM, pRegFrame) == 0
770 && ((CPUMGetGuestCR0(pVM) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG)
771 && (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P))
772 {
773 uint64_t fPageGst;
774 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
775 if ( RT_SUCCESS(rc)
776 && !(fPageGst & X86_PTE_RW))
777 {
778 rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
779 if (RT_SUCCESS(rc))
780 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eWPEmulInRZ);
781 else
782 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eWPEmulToR3);
783 return rc;
784 }
785 AssertMsgFailed(("Unexpected r/w page %RGv flag=%x rc=%Rrc\n", pvFault, (uint32_t)fPageGst, rc));
786 }
787 }
788
789# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
790# ifdef VBOX_STRICT
791 /*
792 * Check for VMM page flags vs. Guest page flags consistency.
793 * Currently only for debug purposes.
794 */
795 if (RT_SUCCESS(rc))
796 {
797 /* Get guest page flags. */
798 uint64_t fPageGst;
799 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
800 if (RT_SUCCESS(rc))
801 {
802 uint64_t fPageShw;
803 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
804
805 /*
806 * Compare page flags.
807 * Note: we have AVL, A, D bits desynched.
808 */
809 AssertMsg((fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)),
810 ("Page flags mismatch! pvFault=%RGv GCPhys=%RGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
811 }
812 else
813 AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
814 }
815 else
816 AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
817# endif /* VBOX_STRICT */
818# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
819 }
820 STAM_PROFILE_STOP(&pVM->pgm.s.StatRZTrap0eTimeOutOfSync, c);
821# endif /* PGM_OUT_OF_SYNC_IN_GC */
822 }
823 else
824 {
825 /*
826 * Page not present in Guest OS or invalid page table address.
827 * This is potential virtual page access handler food.
828 *
829 * For the present we'll say that our access handlers don't
830 * work for this case - we've already discarded the page table
831 * not present case which is identical to this.
832 *
833 * When we perchance find we need this, we will probably have AVL
834 * trees (offset based) to operate on and we can measure their speed
835 * agains mapping a page table and probably rearrange this handling
836 * a bit. (Like, searching virtual ranges before checking the
837 * physical address.)
838 */
839 }
840 }
841
842
843# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
844 /*
845 * Conclusion, this is a guest trap.
846 */
847 LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
848 STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eGuestPFUnh);
849 return VINF_EM_RAW_GUEST_TRAP;
850# else
851 /* present, but not a monitored page; perhaps the guest is probing physical memory */
852 return VINF_EM_RAW_EMULATE_INSTR;
853# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
854
855
856# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
857
858 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
859 return VERR_INTERNAL_ERROR;
860# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
861}
862#endif /* !IN_RING3 */
863
864
865/**
866 * Emulation of the invlpg instruction.
867 *
868 *
869 * @returns VBox status code.
870 *
871 * @param pVM VM handle.
872 * @param GCPtrPage Page to invalidate.
873 *
874 * @remark ASSUMES that the guest is updating before invalidating. This order
875 * isn't required by the CPU, so this is speculative and could cause
876 * trouble.
877 *
878 * @todo Flush page or page directory only if necessary!
879 * @todo Add a #define for simply invalidating the page.
880 */
881PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
882{
883#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
884 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
885 && PGM_SHW_TYPE != PGM_TYPE_EPT
886 int rc;
887
888 LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
889 /*
890 * Get the shadow PD entry and skip out if this PD isn't present.
891 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
892 */
893# if PGM_SHW_TYPE == PGM_TYPE_32BIT
894 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
895 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
896# elif PGM_SHW_TYPE == PGM_TYPE_PAE
897 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT; /* no mask; flat index into the 2048 entry array. */
898 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT); NOREF(iPdpte);
899 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
900 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT); NOREF(pPdptDst);
901
902 /* If the shadow PDPE isn't present, then skip the invalidate. */
903 if (!pPdptDst->a[iPdpte].n.u1Present)
904 {
905 Assert(!(pPdptDst->a[iPdpte].u & PGM_PLXFLAGS_MAPPING));
906 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
907 return VINF_SUCCESS;
908 }
909
910# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
911 /* PML4 */
912 AssertReturn(pVM->pgm.s.pHCPaePML4, VERR_INTERNAL_ERROR);
913
914 const unsigned iPml4e = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
915 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
916 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
917 PX86PDPAE pPDDst;
918 PX86PDPT pPdptDst;
919 PX86PML4E pPml4eDst = &pVM->pgm.s.pHCPaePML4->a[iPml4e];
920 rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
921 if (rc != VINF_SUCCESS)
922 {
923 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
924 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
925 if (!VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
926 PGM_INVL_GUEST_TLBS();
927 return VINF_SUCCESS;
928 }
929 Assert(pPDDst);
930
931 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
932 PX86PDPE pPdpeDst = &pPdptDst->a[iPdpte];
933
934 if (!pPdpeDst->n.u1Present)
935 {
936 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
937 if (!VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
938 PGM_INVL_GUEST_TLBS();
939 return VINF_SUCCESS;
940 }
941
942# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
943
944 const SHWPDE PdeDst = *pPdeDst;
945 if (!PdeDst.n.u1Present)
946 {
947 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
948 return VINF_SUCCESS;
949 }
950
951 /*
952 * Get the guest PD entry and calc big page.
953 */
954# if PGM_GST_TYPE == PGM_TYPE_32BIT
955 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVM->pgm.s);
956 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
957 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
958# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
959 unsigned iPDSrc;
960# if PGM_GST_TYPE == PGM_TYPE_PAE
961 X86PDPE PdpeSrc;
962 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
963# else /* AMD64 */
964 PX86PML4E pPml4eSrc;
965 X86PDPE PdpeSrc;
966 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
967# endif
968 GSTPDE PdeSrc;
969
970 if (pPDSrc)
971 PdeSrc = pPDSrc->a[iPDSrc];
972 else
973 PdeSrc.u = 0;
974# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
975
976# if PGM_GST_TYPE == PGM_TYPE_AMD64
977 const bool fIsBigPage = PdeSrc.b.u1Size;
978# else
979 const bool fIsBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
980# endif
981
982# ifdef IN_RING3
983 /*
984 * If a CR3 Sync is pending we may ignore the invalidate page operation
985 * depending on the kind of sync and if it's a global page or not.
986 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
987 */
988# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
989 if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
990 || ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
991 && fIsBigPage
992 && PdeSrc.b.u1Global
993 )
994 )
995# else
996 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
997# endif
998 {
999 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1000 return VINF_SUCCESS;
1001 }
1002# endif /* IN_RING3 */
1003
1004# if PGM_GST_TYPE == PGM_TYPE_AMD64
1005 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1006
1007 /* Fetch the pgm pool shadow descriptor. */
1008 PPGMPOOLPAGE pShwPdpt = pgmPoolGetPageByHCPhys(pVM, pPml4eDst->u & X86_PML4E_PG_MASK);
1009 Assert(pShwPdpt);
1010
1011 /* Fetch the pgm pool shadow descriptor. */
1012 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpte].u & SHW_PDPE_PG_MASK);
1013 Assert(pShwPde);
1014
1015 Assert(pPml4eDst->n.u1Present && (pPml4eDst->u & SHW_PDPT_MASK));
1016 RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1017
1018 if ( !pPml4eSrc->n.u1Present
1019 || pShwPdpt->GCPhys != GCPhysPdpt)
1020 {
1021 LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1022 GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1023 pgmPoolFreeByPage(pPool, pShwPdpt, pVM->pgm.s.pHCShwAmd64CR3->idx, iPml4e);
1024 pPml4eDst->u = 0;
1025 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1026 PGM_INVL_GUEST_TLBS();
1027 return VINF_SUCCESS;
1028 }
1029 if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
1030 || (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
1031 {
1032 /*
1033 * Mark not present so we can resync the PML4E when it's used.
1034 */
1035 LogFlow(("InvalidatePage: Out-of-sync PML4E at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1036 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1037 pgmPoolFreeByPage(pPool, pShwPdpt, pVM->pgm.s.pHCShwAmd64CR3->idx, iPml4e);
1038 pPml4eDst->u = 0;
1039 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1040 PGM_INVL_GUEST_TLBS();
1041 }
1042 else if (!pPml4eSrc->n.u1Accessed)
1043 {
1044 /*
1045 * Mark not present so we can set the accessed bit.
1046 */
1047 LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1048 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1049 pgmPoolFreeByPage(pPool, pShwPdpt, pVM->pgm.s.pHCShwAmd64CR3->idx, iPml4e);
1050 pPml4eDst->u = 0;
1051 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1052 PGM_INVL_GUEST_TLBS();
1053 }
1054
1055 /* Check if the PDPT entry has changed. */
1056 Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1057 RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1058 if ( !PdpeSrc.n.u1Present
1059 || pShwPde->GCPhys != GCPhysPd)
1060 {
1061 LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1062 GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1063 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpte);
1064 pPdpeDst->u = 0;
1065 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1066 PGM_INVL_GUEST_TLBS();
1067 return VINF_SUCCESS;
1068 }
1069 if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1070 || (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1071 {
1072 /*
1073 * Mark not present so we can resync the PDPTE when it's used.
1074 */
1075 LogFlow(("InvalidatePage: Out-of-sync PDPE at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1076 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1077 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpte);
1078 pPdpeDst->u = 0;
1079 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1080 PGM_INVL_GUEST_TLBS();
1081 }
1082 else if (!PdpeSrc.lm.u1Accessed)
1083 {
1084 /*
1085 * Mark not present so we can set the accessed bit.
1086 */
1087 LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1088 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1089 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpte);
1090 pPdpeDst->u = 0;
1091 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1092 PGM_INVL_GUEST_TLBS();
1093 }
1094# endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1095
1096# if PGM_GST_TYPE == PGM_TYPE_PAE
1097 /*
1098 * Update the shadow PDPE and free all the shadow PD entries if the PDPE is marked not present.
1099 * Note: This shouldn't actually be necessary as we monitor the PDPT page for changes.
1100 */
1101 if (!pPDSrc)
1102 {
1103 /* Guest PDPE not present */
1104 PX86PDPAE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* root of the 2048 PDE array */
1105 PX86PDEPAE pPDEDst = &pPDPAE->a[iPdpte * X86_PG_PAE_ENTRIES];
1106 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1107
1108 Assert(!PdpeSrc.n.u1Present);
1109 LogFlow(("InvalidatePage: guest PDPE %d not present; clear shw pdpe\n", iPdpte));
1110
1111 /* for each page directory entry */
1112 for (unsigned iPD = 0; iPD < X86_PG_PAE_ENTRIES; iPD++)
1113 {
1114 if ( pPDEDst[iPD].n.u1Present
1115 && !(pPDEDst[iPD].u & PGM_PDFLAGS_MAPPING))
1116 {
1117 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst[iPD].u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdpte * X86_PG_PAE_ENTRIES + iPD);
1118 pPDEDst[iPD].u = 0;
1119 }
1120 }
1121 if (!(pPdptDst->a[iPdpte].u & PGM_PLXFLAGS_MAPPING))
1122 pPdptDst->a[iPdpte].n.u1Present = 0;
1123 PGM_INVL_GUEST_TLBS();
1124 }
1125 AssertMsg(pVM->pgm.s.fMappingsFixed || (PdpeSrc.u & X86_PDPE_PG_MASK) == pVM->pgm.s.aGCPhysGstPaePDsMonitored[iPdpte], ("%RGp vs %RGp (mon)\n", (PdpeSrc.u & X86_PDPE_PG_MASK), pVM->pgm.s.aGCPhysGstPaePDsMonitored[iPdpte]));
1126# endif
1127
1128
1129 /*
1130 * Deal with the Guest PDE.
1131 */
1132 rc = VINF_SUCCESS;
1133 if (PdeSrc.n.u1Present)
1134 {
1135 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1136 {
1137 /*
1138 * Conflict - Let SyncPT deal with it to avoid duplicate code.
1139 */
1140 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1141 Assert(PGMGetGuestMode(pVM) <= PGMMODE_PAE);
1142 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
1143 }
1144 else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1145 || (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1146 {
1147 /*
1148 * Mark not present so we can resync the PDE when it's used.
1149 */
1150 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1151 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1152# if PGM_GST_TYPE == PGM_TYPE_AMD64
1153 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1154# else
1155 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1156# endif
1157 pPdeDst->u = 0;
1158 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1159 PGM_INVL_GUEST_TLBS();
1160 }
1161 else if (!PdeSrc.n.u1Accessed)
1162 {
1163 /*
1164 * Mark not present so we can set the accessed bit.
1165 */
1166 LogFlow(("InvalidatePage: Out-of-sync (A) at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1167 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1168# if PGM_GST_TYPE == PGM_TYPE_AMD64
1169 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1170# else
1171 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1172# endif
1173 pPdeDst->u = 0;
1174 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1175 PGM_INVL_GUEST_TLBS();
1176 }
1177 else if (!fIsBigPage)
1178 {
1179 /*
1180 * 4KB - page.
1181 */
1182 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1183 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1184# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1185 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1186 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1187# endif
1188 if (pShwPage->GCPhys == GCPhys)
1189 {
1190# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1191 const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1192 PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1193 if (pPT->a[iPTEDst].n.u1Present)
1194 {
1195# ifdef PGMPOOL_WITH_USER_TRACKING
1196 /* This is very unlikely with caching/monitoring enabled. */
1197 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1198# endif
1199 pPT->a[iPTEDst].u = 0;
1200 }
1201# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1202 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
1203 if (RT_SUCCESS(rc))
1204 rc = VINF_SUCCESS;
1205# endif
1206 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1207 PGM_INVL_PG(GCPtrPage);
1208 }
1209 else
1210 {
1211 /*
1212 * The page table address changed.
1213 */
1214 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1215 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1216# if PGM_GST_TYPE == PGM_TYPE_AMD64
1217 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1218# else
1219 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1220# endif
1221 pPdeDst->u = 0;
1222 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1223 PGM_INVL_GUEST_TLBS();
1224 }
1225 }
1226 else
1227 {
1228 /*
1229 * 2/4MB - page.
1230 */
1231 /* Before freeing the page, check if anything really changed. */
1232 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1233 RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1234# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1235 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1236 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1237# endif
1238 if ( pShwPage->GCPhys == GCPhys
1239 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1240 {
1241 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1242 /** @todo PAT */
1243 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1244 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1245 && ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1246 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1247 {
1248 LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1249 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1250 return VINF_SUCCESS;
1251 }
1252 }
1253
1254 /*
1255 * Ok, the page table is present and it's been changed in the guest.
1256 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1257 * We could do this for some flushes in GC too, but we need an algorithm for
1258 * deciding which 4MB pages containing code likely to be executed very soon.
1259 */
1260 LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1261 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1262# if PGM_GST_TYPE == PGM_TYPE_AMD64
1263 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1264# else
1265 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1266# endif
1267 pPdeDst->u = 0;
1268 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1269 PGM_INVL_BIG_PG(GCPtrPage);
1270 }
1271 }
1272 else
1273 {
1274 /*
1275 * Page directory is not present, mark shadow PDE not present.
1276 */
1277 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1278 {
1279# if PGM_GST_TYPE == PGM_TYPE_AMD64
1280 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1281# else
1282 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1283# endif
1284 pPdeDst->u = 0;
1285 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1286 PGM_INVL_PG(GCPtrPage);
1287 }
1288 else
1289 {
1290 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1291 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDMappings));
1292 }
1293 }
1294
1295 return rc;
1296
1297#else /* guest real and protected mode */
1298 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1299 return VINF_SUCCESS;
1300#endif
1301}
1302
1303
1304#ifdef PGMPOOL_WITH_USER_TRACKING
1305/**
1306 * Update the tracking of shadowed pages.
1307 *
1308 * @param pVM The VM handle.
1309 * @param pShwPage The shadow page.
1310 * @param HCPhys The physical page we is being dereferenced.
1311 */
1312DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1313{
1314# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1315 STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1316 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1317
1318 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1319 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1320 * 2. write protect all shadowed pages. I.e. implement caching.
1321 */
1322 /*
1323 * Find the guest address.
1324 */
1325 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1326 pRam;
1327 pRam = pRam->CTX_SUFF(pNext))
1328 {
1329 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1330 while (iPage-- > 0)
1331 {
1332 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1333 {
1334 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1335 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1336 pShwPage->cPresent--;
1337 pPool->cPresent--;
1338 STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1339 return;
1340 }
1341 }
1342 }
1343
1344 for (;;)
1345 AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1346# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1347 pShwPage->cPresent--;
1348 pVM->pgm.s.CTX_SUFF(pPool)->cPresent--;
1349# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1350}
1351
1352
1353/**
1354 * Update the tracking of shadowed pages.
1355 *
1356 * @param pVM The VM handle.
1357 * @param pShwPage The shadow page.
1358 * @param u16 The top 16-bit of the pPage->HCPhys.
1359 * @param pPage Pointer to the guest page. this will be modified.
1360 * @param iPTDst The index into the shadow table.
1361 */
1362DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1363{
1364# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1365 /*
1366 * We're making certain assumptions about the placement of cRef and idx.
1367 */
1368 Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1369 Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1370
1371 /*
1372 * Just deal with the simple first time here.
1373 */
1374 if (!u16)
1375 {
1376 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1377 u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) | pShwPage->idx;
1378 }
1379 else
1380 u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1381
1382 /* write back, trying to be clever... */
1383 Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%RHp->%RHp iPTDst=%#x\n",
1384 u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) | ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1385 *((uint16_t *)&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1386# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1387
1388 /* update statistics. */
1389 pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1390 pShwPage->cPresent++;
1391 if (pShwPage->iFirstPresent > iPTDst)
1392 pShwPage->iFirstPresent = iPTDst;
1393}
1394#endif /* PGMPOOL_WITH_USER_TRACKING */
1395
1396
1397/**
1398 * Creates a 4K shadow page for a guest page.
1399 *
1400 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1401 * physical address. The PdeSrc argument only the flags are used. No page structured
1402 * will be mapped in this function.
1403 *
1404 * @param pVM VM handle.
1405 * @param pPteDst Destination page table entry.
1406 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1407 * Can safely assume that only the flags are being used.
1408 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1409 * @param pShwPage Pointer to the shadow page.
1410 * @param iPTDst The index into the shadow table.
1411 *
1412 * @remark Not used for 2/4MB pages!
1413 */
1414DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1415{
1416 if (PteSrc.n.u1Present)
1417 {
1418 /*
1419 * Find the ram range.
1420 */
1421 PPGMPAGE pPage;
1422 int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1423 if (RT_SUCCESS(rc))
1424 {
1425 /** @todo investiage PWT, PCD and PAT. */
1426 /*
1427 * Make page table entry.
1428 */
1429 const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1430 SHWPTE PteDst;
1431 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1432 {
1433 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1434 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1435 {
1436#if PGM_SHW_TYPE == PGM_TYPE_EPT
1437 PteDst.u = (HCPhys & EPT_PTE_PG_MASK);
1438 PteDst.n.u1Present = 1;
1439 PteDst.n.u1Execute = 1;
1440 PteDst.n.u1IgnorePAT = 1;
1441 PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1442 /* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1443#else
1444 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1445 | (HCPhys & X86_PTE_PAE_PG_MASK);
1446#endif
1447 }
1448 else
1449 {
1450 LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", HCPhys));
1451 PteDst.u = 0;
1452 }
1453 /** @todo count these two kinds. */
1454 }
1455 else
1456 {
1457#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1458 /*
1459 * If the page or page directory entry is not marked accessed,
1460 * we mark the page not present.
1461 */
1462 if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
1463 {
1464 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1465 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,AccessedPage));
1466 PteDst.u = 0;
1467 }
1468 else
1469 /*
1470 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1471 * when the page is modified.
1472 */
1473 if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1474 {
1475 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyPage));
1476 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1477 | (HCPhys & X86_PTE_PAE_PG_MASK)
1478 | PGM_PTFLAGS_TRACK_DIRTY;
1479 }
1480 else
1481#endif
1482 {
1483 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyPageSkipped));
1484#if PGM_SHW_TYPE == PGM_TYPE_EPT
1485 PteDst.u = (HCPhys & EPT_PTE_PG_MASK);
1486 PteDst.n.u1Present = 1;
1487 PteDst.n.u1Write = 1;
1488 PteDst.n.u1Execute = 1;
1489 PteDst.n.u1IgnorePAT = 1;
1490 PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1491 /* PteDst.n.u1Size = 0 */
1492#else
1493 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1494 | (HCPhys & X86_PTE_PAE_PG_MASK);
1495#endif
1496 }
1497 }
1498
1499#ifdef PGMPOOL_WITH_USER_TRACKING
1500 /*
1501 * Keep user track up to date.
1502 */
1503 if (PteDst.n.u1Present)
1504 {
1505 if (!pPteDst->n.u1Present)
1506 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1507 else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1508 {
1509 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1510 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1511 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1512 }
1513 }
1514 else if (pPteDst->n.u1Present)
1515 {
1516 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1517 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1518 }
1519#endif /* PGMPOOL_WITH_USER_TRACKING */
1520
1521 /*
1522 * Update statistics and commit the entry.
1523 */
1524 if (!PteSrc.n.u1Global)
1525 pShwPage->fSeenNonGlobal = true;
1526 *pPteDst = PteDst;
1527 }
1528 /* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1529 /** @todo count these. */
1530 }
1531 else
1532 {
1533 /*
1534 * Page not-present.
1535 */
1536 LogFlow(("SyncPageWorker: page not present in Pte\n"));
1537#ifdef PGMPOOL_WITH_USER_TRACKING
1538 /* Keep user track up to date. */
1539 if (pPteDst->n.u1Present)
1540 {
1541 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1542 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1543 }
1544#endif /* PGMPOOL_WITH_USER_TRACKING */
1545 pPteDst->u = 0;
1546 /** @todo count these. */
1547 }
1548}
1549
1550
1551/**
1552 * Syncs a guest OS page.
1553 *
1554 * There are no conflicts at this point, neither is there any need for
1555 * page table allocations.
1556 *
1557 * @returns VBox status code.
1558 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1559 * @param pVM VM handle.
1560 * @param PdeSrc Page directory entry of the guest.
1561 * @param GCPtrPage Guest context page address.
1562 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1563 * @param uErr Fault error (X86_TRAP_PF_*).
1564 */
1565PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1566{
1567 LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1568
1569#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1570 || PGM_GST_TYPE == PGM_TYPE_PAE \
1571 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
1572 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
1573 && PGM_SHW_TYPE != PGM_TYPE_EPT
1574
1575# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1576 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1577# endif
1578
1579 /*
1580 * Assert preconditions.
1581 */
1582 Assert(PdeSrc.n.u1Present);
1583 Assert(cPages);
1584 STAM_COUNTER_INC(&pVM->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1585
1586 /*
1587 * Get the shadow PDE, find the shadow page table in the pool.
1588 */
1589# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1590 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1591 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1592# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1593 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1594 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT); NOREF(iPdpte); /* no mask; flat index into the 2048 entry array. */
1595 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT); NOREF(pPdptDst);
1596 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1597# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1598 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1599 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1600 PX86PDPAE pPDDst;
1601 X86PDEPAE PdeDst;
1602 PX86PDPT pPdptDst;
1603
1604 int rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
1605 AssertRCSuccessReturn(rc, rc);
1606 Assert(pPDDst && pPdptDst);
1607 PdeDst = pPDDst->a[iPDDst];
1608# endif
1609 Assert(PdeDst.n.u1Present);
1610 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1611
1612# if PGM_GST_TYPE == PGM_TYPE_AMD64
1613 /* Fetch the pgm pool shadow descriptor. */
1614 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpte].u & X86_PDPE_PG_MASK);
1615 Assert(pShwPde);
1616# endif
1617
1618 /*
1619 * Check that the page is present and that the shadow PDE isn't out of sync.
1620 */
1621# if PGM_GST_TYPE == PGM_TYPE_AMD64
1622 const bool fBigPage = PdeSrc.b.u1Size;
1623# else
1624 const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1625# endif
1626 RTGCPHYS GCPhys;
1627 if (!fBigPage)
1628 {
1629 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1630# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1631 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1632 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1633# endif
1634 }
1635 else
1636 {
1637 GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1638# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1639 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1640 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1641# endif
1642 }
1643 if ( pShwPage->GCPhys == GCPhys
1644 && PdeSrc.n.u1Present
1645 && (PdeSrc.n.u1User == PdeDst.n.u1User)
1646 && (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
1647# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1648 && (!fNoExecuteBitValid || PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1649# endif
1650 )
1651 {
1652 /*
1653 * Check that the PDE is marked accessed already.
1654 * Since we set the accessed bit *before* getting here on a #PF, this
1655 * check is only meant for dealing with non-#PF'ing paths.
1656 */
1657 if (PdeSrc.n.u1Accessed)
1658 {
1659 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1660 if (!fBigPage)
1661 {
1662 /*
1663 * 4KB Page - Map the guest page table.
1664 */
1665 PGSTPT pPTSrc;
1666 int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1667 if (RT_SUCCESS(rc))
1668 {
1669# ifdef PGM_SYNC_N_PAGES
1670 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1671 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1672 {
1673 /*
1674 * This code path is currently only taken when the caller is PGMTrap0eHandler
1675 * for non-present pages!
1676 *
1677 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1678 * deal with locality.
1679 */
1680 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1681# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1682 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1683 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1684# else
1685 const unsigned offPTSrc = 0;
1686# endif
1687 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1688 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1689 iPTDst = 0;
1690 else
1691 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1692 for (; iPTDst < iPTDstEnd; iPTDst++)
1693 {
1694 if (!pPTDst->a[iPTDst].n.u1Present)
1695 {
1696 GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1697 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1698 NOREF(GCPtrCurPage);
1699#ifndef IN_RING0
1700 /*
1701 * Assuming kernel code will be marked as supervisor - and not as user level
1702 * and executed using a conforming code selector - And marked as readonly.
1703 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
1704 */
1705 PPGMPAGE pPage;
1706 if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW | X86_PTE_US))
1707 || iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1708 || !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1709 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1710 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1711 )
1712#endif /* else: CSAM not active */
1713 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1714 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1715 GCPtrCurPage, PteSrc.n.u1Present,
1716 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1717 PteSrc.n.u1User & PdeSrc.n.u1User,
1718 (uint64_t)PteSrc.u,
1719 (uint64_t)pPTDst->a[iPTDst].u,
1720 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1721 }
1722 }
1723 }
1724 else
1725# endif /* PGM_SYNC_N_PAGES */
1726 {
1727 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1728 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1729 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1730 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1731 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1732 GCPtrPage, PteSrc.n.u1Present,
1733 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1734 PteSrc.n.u1User & PdeSrc.n.u1User,
1735 (uint64_t)PteSrc.u,
1736 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1737 }
1738 }
1739 else /* MMIO or invalid page: emulated in #PF handler. */
1740 {
1741 LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1742 Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1743 }
1744 }
1745 else
1746 {
1747 /*
1748 * 4/2MB page - lazy syncing shadow 4K pages.
1749 * (There are many causes of getting here, it's no longer only CSAM.)
1750 */
1751 /* Calculate the GC physical address of this 4KB shadow page. */
1752 RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc) | ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1753 /* Find ram range. */
1754 PPGMPAGE pPage;
1755 int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1756 if (RT_SUCCESS(rc))
1757 {
1758 /*
1759 * Make shadow PTE entry.
1760 */
1761 const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1762 SHWPTE PteDst;
1763 PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1764 | (HCPhys & X86_PTE_PAE_PG_MASK);
1765 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1766 {
1767 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1768 PteDst.n.u1Write = 0;
1769 else
1770 PteDst.u = 0;
1771 }
1772 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1773# ifdef PGMPOOL_WITH_USER_TRACKING
1774 if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1775 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1776# endif
1777 pPTDst->a[iPTDst] = PteDst;
1778
1779
1780 /*
1781 * If the page is not flagged as dirty and is writable, then make it read-only
1782 * at PD level, so we can set the dirty bit when the page is modified.
1783 *
1784 * ASSUMES that page access handlers are implemented on page table entry level.
1785 * Thus we will first catch the dirty access and set PDE.D and restart. If
1786 * there is an access handler, we'll trap again and let it work on the problem.
1787 */
1788 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1789 * As for invlpg, it simply frees the whole shadow PT.
1790 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1791 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1792 {
1793 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
1794 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
1795 PdeDst.n.u1Write = 0;
1796 }
1797 else
1798 {
1799 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1800 PdeDst.n.u1Write = PdeSrc.n.u1Write;
1801 }
1802# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1803 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1804# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1805 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1806# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1807 pPDDst->a[iPDDst] = PdeDst;
1808# endif
1809 Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1810 GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1811 PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1812 }
1813 else
1814 LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1815 }
1816 return VINF_SUCCESS;
1817 }
1818 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPagePDNAs));
1819 }
1820 else
1821 {
1822 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1823 Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1824 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1825 }
1826
1827 /*
1828 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1829 * Yea, I'm lazy.
1830 */
1831 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1832# if PGM_GST_TYPE == PGM_TYPE_AMD64
1833 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1834# else
1835 pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPDDst);
1836# endif
1837
1838# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1839 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1840# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1841 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1842# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1843 pPDDst->a[iPDDst].u = 0;
1844# endif
1845 PGM_INVL_GUEST_TLBS();
1846 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1847
1848#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
1849 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
1850 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
1851
1852# ifdef PGM_SYNC_N_PAGES
1853 /*
1854 * Get the shadow PDE, find the shadow page table in the pool.
1855 */
1856# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1857 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1858 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1859# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1860 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT; /* no mask; flat index into the 2048 entry array. */
1861 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1862# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1863 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1864 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpte);
1865 PX86PDPAE pPDDst;
1866 X86PDEPAE PdeDst;
1867 PX86PDPT pPdptDst;
1868
1869 int rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
1870 AssertRCSuccessReturn(rc, rc);
1871 Assert(pPDDst && pPdptDst);
1872 PdeDst = pPDDst->a[iPDDst];
1873# elif PGM_SHW_TYPE == PGM_TYPE_EPT
1874 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1875 PEPTPD pPDDst;
1876 EPTPDE PdeDst;
1877
1878 int rc = PGMShwGetEPTPDPtr(pVM, GCPtrPage, NULL, &pPDDst);
1879 if (rc != VINF_SUCCESS)
1880 {
1881 AssertRC(rc);
1882 return rc;
1883 }
1884 Assert(pPDDst);
1885 PdeDst = pPDDst->a[iPDDst];
1886# endif
1887 Assert(PdeDst.n.u1Present);
1888 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1889 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1890
1891 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1892 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1893 {
1894 /*
1895 * This code path is currently only taken when the caller is PGMTrap0eHandler
1896 * for non-present pages!
1897 *
1898 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1899 * deal with locality.
1900 */
1901 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1902 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1903 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1904 iPTDst = 0;
1905 else
1906 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1907 for (; iPTDst < iPTDstEnd; iPTDst++)
1908 {
1909 if (!pPTDst->a[iPTDst].n.u1Present)
1910 {
1911 GSTPTE PteSrc;
1912
1913 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT);
1914
1915 /* Fake the page table entry */
1916 PteSrc.u = GCPtrCurPage;
1917 PteSrc.n.u1Present = 1;
1918 PteSrc.n.u1Dirty = 1;
1919 PteSrc.n.u1Accessed = 1;
1920 PteSrc.n.u1Write = 1;
1921 PteSrc.n.u1User = 1;
1922
1923 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1924
1925 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1926 GCPtrCurPage, PteSrc.n.u1Present,
1927 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1928 PteSrc.n.u1User & PdeSrc.n.u1User,
1929 (uint64_t)PteSrc.u,
1930 (uint64_t)pPTDst->a[iPTDst].u,
1931 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1932 }
1933 else
1934 Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
1935 }
1936 }
1937 else
1938# endif /* PGM_SYNC_N_PAGES */
1939 {
1940 GSTPTE PteSrc;
1941 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1942 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT);
1943
1944 /* Fake the page table entry */
1945 PteSrc.u = GCPtrCurPage;
1946 PteSrc.n.u1Present = 1;
1947 PteSrc.n.u1Dirty = 1;
1948 PteSrc.n.u1Accessed = 1;
1949 PteSrc.n.u1Write = 1;
1950 PteSrc.n.u1User = 1;
1951 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1952
1953 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
1954 GCPtrPage, PteSrc.n.u1Present,
1955 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1956 PteSrc.n.u1User & PdeSrc.n.u1User,
1957 (uint64_t)PteSrc.u,
1958 (uint64_t)pPTDst->a[iPTDst].u,
1959 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1960 }
1961 return VINF_SUCCESS;
1962
1963#else
1964 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1965 return VERR_INTERNAL_ERROR;
1966#endif
1967}
1968
1969
1970#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1971/**
1972 * Investigate page fault and handle write protection page faults caused by
1973 * dirty bit tracking.
1974 *
1975 * @returns VBox status code.
1976 * @param pVM VM handle.
1977 * @param uErr Page fault error code.
1978 * @param pPdeDst Shadow page directory entry.
1979 * @param pPdeSrc Guest page directory entry.
1980 * @param GCPtrPage Guest context page address.
1981 */
1982PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1983{
1984 bool fWriteProtect = !!(CPUMGetGuestCR0(pVM) & X86_CR0_WP);
1985 bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
1986 bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
1987# if PGM_GST_TYPE == PGM_TYPE_AMD64
1988 bool fBigPagesSupported = true;
1989# else
1990 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1991# endif
1992# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1993 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1994# endif
1995 unsigned uPageFaultLevel;
1996 int rc;
1997
1998 STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
1999 LogFlow(("CheckPageFault: GCPtrPage=%RGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
2000
2001# if PGM_GST_TYPE == PGM_TYPE_PAE \
2002 || PGM_GST_TYPE == PGM_TYPE_AMD64
2003
2004# if PGM_GST_TYPE == PGM_TYPE_AMD64
2005 PX86PML4E pPml4eSrc;
2006 PX86PDPE pPdpeSrc;
2007
2008 pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc);
2009 Assert(pPml4eSrc);
2010
2011 /*
2012 * Real page fault? (PML4E level)
2013 */
2014 if ( (uErr & X86_TRAP_PF_RSVD)
2015 || !pPml4eSrc->n.u1Present
2016 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
2017 || (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
2018 || (fUserLevelFault && !pPml4eSrc->n.u1User)
2019 )
2020 {
2021 uPageFaultLevel = 0;
2022 goto l_UpperLevelPageFault;
2023 }
2024 Assert(pPdpeSrc);
2025
2026# else /* PAE */
2027 PX86PDPE pPdpeSrc = pgmGstGetPaePDPEPtr(&pVM->pgm.s, GCPtrPage);
2028# endif /* PAE */
2029
2030 /*
2031 * Real page fault? (PDPE level)
2032 */
2033 if ( (uErr & X86_TRAP_PF_RSVD)
2034 || !pPdpeSrc->n.u1Present
2035# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
2036 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
2037 || (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault || fWriteProtect))
2038 || (fUserLevelFault && !pPdpeSrc->lm.u1User)
2039# endif
2040 )
2041 {
2042 uPageFaultLevel = 1;
2043 goto l_UpperLevelPageFault;
2044 }
2045# endif
2046
2047 /*
2048 * Real page fault? (PDE level)
2049 */
2050 if ( (uErr & X86_TRAP_PF_RSVD)
2051 || !pPdeSrc->n.u1Present
2052# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2053 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
2054# endif
2055 || (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
2056 || (fUserLevelFault && !pPdeSrc->n.u1User) )
2057 {
2058 uPageFaultLevel = 2;
2059 goto l_UpperLevelPageFault;
2060 }
2061
2062 /*
2063 * First check the easy case where the page directory has been marked read-only to track
2064 * the dirty bit of an emulated BIG page
2065 */
2066 if (pPdeSrc->b.u1Size && fBigPagesSupported)
2067 {
2068 /* Mark guest page directory as accessed */
2069# if PGM_GST_TYPE == PGM_TYPE_AMD64
2070 pPml4eSrc->n.u1Accessed = 1;
2071 pPdpeSrc->lm.u1Accessed = 1;
2072# endif
2073 pPdeSrc->b.u1Accessed = 1;
2074
2075 /*
2076 * Only write protection page faults are relevant here.
2077 */
2078 if (fWriteFault)
2079 {
2080 /* Mark guest page directory as dirty (BIG page only). */
2081 pPdeSrc->b.u1Dirty = 1;
2082
2083 if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2084 {
2085 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2086
2087 Assert(pPdeSrc->b.u1Write);
2088
2089 pPdeDst->n.u1Write = 1;
2090 pPdeDst->n.u1Accessed = 1;
2091 pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2092 PGM_INVL_BIG_PG(GCPtrPage);
2093 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2094 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2095 }
2096 }
2097 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2098 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2099 }
2100 /* else: 4KB page table */
2101
2102 /*
2103 * Map the guest page table.
2104 */
2105 PGSTPT pPTSrc;
2106 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2107 if (RT_SUCCESS(rc))
2108 {
2109 /*
2110 * Real page fault?
2111 */
2112 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2113 const GSTPTE PteSrc = *pPteSrc;
2114 if ( !PteSrc.n.u1Present
2115# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2116 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2117# endif
2118 || (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault || fWriteProtect))
2119 || (fUserLevelFault && !PteSrc.n.u1User)
2120 )
2121 {
2122 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2123 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2124 LogFlow(("CheckPageFault: real page fault at %RGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2125
2126 /* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2127 * See the 2nd case above as well.
2128 */
2129 if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2130 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2131
2132 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2133 return VINF_EM_RAW_GUEST_TRAP;
2134 }
2135 LogFlow(("CheckPageFault: page fault at %RGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2136
2137 /*
2138 * Set the accessed bits in the page directory and the page table.
2139 */
2140# if PGM_GST_TYPE == PGM_TYPE_AMD64
2141 pPml4eSrc->n.u1Accessed = 1;
2142 pPdpeSrc->lm.u1Accessed = 1;
2143# endif
2144 pPdeSrc->n.u1Accessed = 1;
2145 pPteSrc->n.u1Accessed = 1;
2146
2147 /*
2148 * Only write protection page faults are relevant here.
2149 */
2150 if (fWriteFault)
2151 {
2152 /* Write access, so mark guest entry as dirty. */
2153# ifdef VBOX_WITH_STATISTICS
2154 if (!pPteSrc->n.u1Dirty)
2155 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtiedPage));
2156 else
2157 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageAlreadyDirty));
2158# endif
2159
2160 pPteSrc->n.u1Dirty = 1;
2161
2162 if (pPdeDst->n.u1Present)
2163 {
2164#ifndef IN_RING0
2165 /* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2166 * Our individual shadow handlers will provide more information and force a fatal exit.
2167 */
2168 if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2169 {
2170 LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2171 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2172 return VINF_SUCCESS;
2173 }
2174#endif
2175 /*
2176 * Map shadow page table.
2177 */
2178 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2179 if (pShwPage)
2180 {
2181 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2182 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2183 if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
2184 && (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
2185 {
2186 LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2187# ifdef VBOX_STRICT
2188 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2189 if (pPage)
2190 AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
2191 ("Unexpected dirty bit tracking on monitored page %RGv (phys %RGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
2192# endif
2193 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2194
2195 Assert(pPteSrc->n.u1Write);
2196
2197 pPteDst->n.u1Write = 1;
2198 pPteDst->n.u1Dirty = 1;
2199 pPteDst->n.u1Accessed = 1;
2200 pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2201 PGM_INVL_PG(GCPtrPage);
2202
2203 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2204 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2205 }
2206 }
2207 else
2208 AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2209 }
2210 }
2211/** @todo Optimize accessed bit emulation? */
2212# ifdef VBOX_STRICT
2213 /*
2214 * Sanity check.
2215 */
2216 else if ( !pPteSrc->n.u1Dirty
2217 && (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2218 && pPdeDst->n.u1Present)
2219 {
2220 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2221 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2222 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2223 if ( pPteDst->n.u1Present
2224 && pPteDst->n.u1Write)
2225 LogFlow(("Writable present page %RGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2226 }
2227# endif /* VBOX_STRICT */
2228 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2229 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2230 }
2231 AssertRC(rc);
2232 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2233 return rc;
2234
2235
2236l_UpperLevelPageFault:
2237 /*
2238 * Pagefault detected while checking the PML4E, PDPE or PDE.
2239 * Single exit handler to get rid of duplicate code paths.
2240 */
2241 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2242 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2243 Log(("CheckPageFault: real page fault at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2244
2245 if (
2246# if PGM_GST_TYPE == PGM_TYPE_AMD64
2247 pPml4eSrc->n.u1Present &&
2248# endif
2249# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
2250 pPdpeSrc->n.u1Present &&
2251# endif
2252 pPdeSrc->n.u1Present)
2253 {
2254 /* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2255 if (pPdeSrc->b.u1Size && fBigPagesSupported)
2256 {
2257 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2258 }
2259 else
2260 {
2261 /*
2262 * Map the guest page table.
2263 */
2264 PGSTPT pPTSrc;
2265 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2266 if (RT_SUCCESS(rc))
2267 {
2268 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2269 const GSTPTE PteSrc = *pPteSrc;
2270 if (pPteSrc->n.u1Present)
2271 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2272 }
2273 AssertRC(rc);
2274 }
2275 }
2276 return VINF_EM_RAW_GUEST_TRAP;
2277}
2278#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2279
2280
2281/**
2282 * Sync a shadow page table.
2283 *
2284 * The shadow page table is not present. This includes the case where
2285 * there is a conflict with a mapping.
2286 *
2287 * @returns VBox status code.
2288 * @param pVM VM handle.
2289 * @param iPD Page directory index.
2290 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
2291 * Assume this is a temporary mapping.
2292 * @param GCPtrPage GC Pointer of the page that caused the fault
2293 */
2294PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
2295{
2296 STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2297 STAM_COUNTER_INC(&pVM->pgm.s.StatSyncPtPD[iPDSrc]);
2298 LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2299
2300#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2301 || PGM_GST_TYPE == PGM_TYPE_PAE \
2302 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2303 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
2304 && PGM_SHW_TYPE != PGM_TYPE_EPT
2305
2306 int rc = VINF_SUCCESS;
2307
2308 /*
2309 * Validate input a little bit.
2310 */
2311 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2312# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2313 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2314 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2315# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2316 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT; /* no mask; flat index into the 2048 entry array. */
2317 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT); NOREF(iPdpte);
2318 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT); NOREF(pPdptDst);
2319 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2320# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2321 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2322 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2323 PX86PDPAE pPDDst;
2324 PX86PDPT pPdptDst;
2325 rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
2326 AssertRCSuccessReturn(rc, rc);
2327 Assert(pPDDst);
2328# endif
2329
2330 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2331 SHWPDE PdeDst = *pPdeDst;
2332
2333# if PGM_GST_TYPE == PGM_TYPE_AMD64
2334 /* Fetch the pgm pool shadow descriptor. */
2335 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpte].u & X86_PDPE_PG_MASK);
2336 Assert(pShwPde);
2337# endif
2338
2339# ifndef PGM_WITHOUT_MAPPINGS
2340 /*
2341 * Check for conflicts.
2342 * GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2343 * HC: Simply resolve the conflict.
2344 */
2345 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2346 {
2347 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2348# ifndef IN_RING3
2349 Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2350 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2351 return VERR_ADDRESS_CONFLICT;
2352# else
2353 PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2354 Assert(pMapping);
2355# if PGM_GST_TYPE == PGM_TYPE_32BIT
2356 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2357# elif PGM_GST_TYPE == PGM_TYPE_PAE
2358 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2359# else
2360 AssertFailed(); /* can't happen for amd64 */
2361# endif
2362 if (RT_FAILURE(rc))
2363 {
2364 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2365 return rc;
2366 }
2367 PdeDst = *pPdeDst;
2368# endif
2369 }
2370# else /* PGM_WITHOUT_MAPPINGS */
2371 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2372# endif /* PGM_WITHOUT_MAPPINGS */
2373 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2374
2375 /*
2376 * Sync page directory entry.
2377 */
2378 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2379 if (PdeSrc.n.u1Present)
2380 {
2381 /*
2382 * Allocate & map the page table.
2383 */
2384 PSHWPT pPTDst;
2385# if PGM_GST_TYPE == PGM_TYPE_AMD64
2386 const bool fPageTable = !PdeSrc.b.u1Size;
2387# else
2388 const bool fPageTable = !PdeSrc.b.u1Size || !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
2389# endif
2390 PPGMPOOLPAGE pShwPage;
2391 RTGCPHYS GCPhys;
2392 if (fPageTable)
2393 {
2394 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2395# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2396 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2397 GCPhys |= (iPDDst & 1) * (PAGE_SIZE / 2);
2398# endif
2399# if PGM_GST_TYPE == PGM_TYPE_AMD64
2400 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2401# else
2402 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2403# endif
2404 }
2405 else
2406 {
2407 GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
2408# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2409 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2410 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2411# endif
2412# if PGM_GST_TYPE == PGM_TYPE_AMD64
2413 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, pShwPde->idx, iPDDst, &pShwPage);
2414# else
2415 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2416# endif
2417 }
2418 if (rc == VINF_SUCCESS)
2419 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2420 else if (rc == VINF_PGM_CACHED_PAGE)
2421 {
2422 /*
2423 * The PT was cached, just hook it up.
2424 */
2425 if (fPageTable)
2426 PdeDst.u = pShwPage->Core.Key
2427 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2428 else
2429 {
2430 PdeDst.u = pShwPage->Core.Key
2431 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2432 /* (see explanation and assumptions further down.) */
2433 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2434 {
2435 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2436 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2437 PdeDst.b.u1Write = 0;
2438 }
2439 }
2440 *pPdeDst = PdeDst;
2441 return VINF_SUCCESS;
2442 }
2443 else if (rc == VERR_PGM_POOL_FLUSHED)
2444 {
2445 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
2446 return VINF_PGM_SYNC_CR3;
2447 }
2448 else
2449 AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2450 PdeDst.u &= X86_PDE_AVL_MASK;
2451 PdeDst.u |= pShwPage->Core.Key;
2452
2453 /*
2454 * Page directory has been accessed (this is a fault situation, remember).
2455 */
2456 pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2457 if (fPageTable)
2458 {
2459 /*
2460 * Page table - 4KB.
2461 *
2462 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2463 */
2464 Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2465 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2466 PGSTPT pPTSrc;
2467 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2468 if (RT_SUCCESS(rc))
2469 {
2470 /*
2471 * Start by syncing the page directory entry so CSAM's TLB trick works.
2472 */
2473 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
2474 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2475 *pPdeDst = PdeDst;
2476
2477 /*
2478 * Directory/page user or supervisor privilege: (same goes for read/write)
2479 *
2480 * Directory Page Combined
2481 * U/S U/S U/S
2482 * 0 0 0
2483 * 0 1 0
2484 * 1 0 0
2485 * 1 1 1
2486 *
2487 * Simple AND operation. Table listed for completeness.
2488 *
2489 */
2490 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT4K));
2491# ifdef PGM_SYNC_N_PAGES
2492 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2493 unsigned iPTDst = iPTBase;
2494 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2495 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2496 iPTDst = 0;
2497 else
2498 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2499# else /* !PGM_SYNC_N_PAGES */
2500 unsigned iPTDst = 0;
2501 const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2502# endif /* !PGM_SYNC_N_PAGES */
2503# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2504 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2505 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2506# else
2507 const unsigned offPTSrc = 0;
2508# endif
2509 for (; iPTDst < iPTDstEnd; iPTDst++)
2510 {
2511 const unsigned iPTSrc = iPTDst + offPTSrc;
2512 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2513
2514 if (PteSrc.n.u1Present) /* we've already cleared it above */
2515 {
2516# ifndef IN_RING0
2517 /*
2518 * Assuming kernel code will be marked as supervisor - and not as user level
2519 * and executed using a conforming code selector - And marked as readonly.
2520 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
2521 */
2522 PPGMPAGE pPage;
2523 if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW | X86_PTE_US))
2524 || !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)))
2525 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2526 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2527 )
2528# endif
2529 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2530 Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2531 (RTGCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)),
2532 PteSrc.n.u1Present,
2533 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2534 PteSrc.n.u1User & PdeSrc.n.u1User,
2535 (uint64_t)PteSrc.u,
2536 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2537 (RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2538 }
2539 } /* for PTEs */
2540 }
2541 }
2542 else
2543 {
2544 /*
2545 * Big page - 2/4MB.
2546 *
2547 * We'll walk the ram range list in parallel and optimize lookups.
2548 * We will only sync on shadow page table at a time.
2549 */
2550 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT4M));
2551
2552 /**
2553 * @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2554 */
2555
2556 /*
2557 * Start by syncing the page directory entry.
2558 */
2559 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2560 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2561
2562 /*
2563 * If the page is not flagged as dirty and is writable, then make it read-only
2564 * at PD level, so we can set the dirty bit when the page is modified.
2565 *
2566 * ASSUMES that page access handlers are implemented on page table entry level.
2567 * Thus we will first catch the dirty access and set PDE.D and restart. If
2568 * there is an access handler, we'll trap again and let it work on the problem.
2569 */
2570 /** @todo move the above stuff to a section in the PGM documentation. */
2571 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2572 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2573 {
2574 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2575 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2576 PdeDst.b.u1Write = 0;
2577 }
2578 *pPdeDst = PdeDst;
2579
2580 /*
2581 * Fill the shadow page table.
2582 */
2583 /* Get address and flags from the source PDE. */
2584 SHWPTE PteDstBase;
2585 PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT);
2586
2587 /* Loop thru the entries in the shadow PT. */
2588 const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2589 Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2590 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2591 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2592 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2593 unsigned iPTDst = 0;
2594 while (iPTDst < RT_ELEMENTS(pPTDst->a))
2595 {
2596 /* Advance ram range list. */
2597 while (pRam && GCPhys > pRam->GCPhysLast)
2598 pRam = pRam->CTX_SUFF(pNext);
2599 if (pRam && GCPhys >= pRam->GCPhys)
2600 {
2601 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2602 do
2603 {
2604 /* Make shadow PTE. */
2605 PPGMPAGE pPage = &pRam->aPages[iHCPage];
2606 SHWPTE PteDst;
2607
2608 /* Make sure the RAM has already been allocated. */
2609 if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2610 {
2611 if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2612 {
2613# ifdef IN_RING3
2614 int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2615# else
2616 int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2617# endif
2618 if (rc != VINF_SUCCESS)
2619 return rc;
2620 }
2621 }
2622
2623 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2624 {
2625 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2626 {
2627 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2628 PteDst.n.u1Write = 0;
2629 }
2630 else
2631 PteDst.u = 0;
2632 }
2633# ifndef IN_RING0
2634 /*
2635 * Assuming kernel code will be marked as supervisor and not as user level and executed
2636 * using a conforming code selector. Don't check for readonly, as that implies the whole
2637 * 4MB can be code or readonly data. Linux enables write access for its large pages.
2638 */
2639 else if ( !PdeSrc.n.u1User
2640 && CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))))
2641 PteDst.u = 0;
2642# endif
2643 else
2644 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2645# ifdef PGMPOOL_WITH_USER_TRACKING
2646 if (PteDst.n.u1Present)
2647 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2648# endif
2649 /* commit it */
2650 pPTDst->a[iPTDst] = PteDst;
2651 Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2652 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2653 PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2654
2655 /* advance */
2656 GCPhys += PAGE_SIZE;
2657 iHCPage++;
2658 iPTDst++;
2659 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2660 && GCPhys <= pRam->GCPhysLast);
2661 }
2662 else if (pRam)
2663 {
2664 Log(("Invalid pages at %RGp\n", GCPhys));
2665 do
2666 {
2667 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2668 GCPhys += PAGE_SIZE;
2669 iPTDst++;
2670 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2671 && GCPhys < pRam->GCPhys);
2672 }
2673 else
2674 {
2675 Log(("Invalid pages at %RGp (2)\n", GCPhys));
2676 for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2677 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2678 }
2679 } /* while more PTEs */
2680 } /* 4KB / 4MB */
2681 }
2682 else
2683 AssertRelease(!PdeDst.n.u1Present);
2684
2685 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2686 if (RT_FAILURE(rc))
2687 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPTFailed));
2688 return rc;
2689
2690#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2691 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
2692 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
2693
2694 int rc = VINF_SUCCESS;
2695
2696 /*
2697 * Validate input a little bit.
2698 */
2699# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2700 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2701 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2702# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2703 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT; /* no mask; flat index into the 2048 entry array. */
2704 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2705# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2706 const unsigned iPdpte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2707 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2708 PX86PDPAE pPDDst;
2709 PX86PDPT pPdptDst;
2710 rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
2711 AssertRCSuccessReturn(rc, rc);
2712 Assert(pPDDst);
2713
2714 /* Fetch the pgm pool shadow descriptor. */
2715 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpte].u & X86_PDPE_PG_MASK);
2716 Assert(pShwPde);
2717# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2718 const unsigned iPdpte = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2719 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2720 PEPTPD pPDDst;
2721 PEPTPDPT pPdptDst;
2722
2723 rc = PGMShwGetEPTPDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
2724 if (rc != VINF_SUCCESS)
2725 {
2726 AssertRC(rc);
2727 return rc;
2728 }
2729 Assert(pPDDst);
2730
2731 /* Fetch the pgm pool shadow descriptor. */
2732 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdpte].u & EPT_PDPTE_PG_MASK);
2733 Assert(pShwPde);
2734# endif
2735 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2736 SHWPDE PdeDst = *pPdeDst;
2737
2738 Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2739 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2740
2741 GSTPDE PdeSrc;
2742 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2743 PdeSrc.n.u1Present = 1;
2744 PdeSrc.n.u1Write = 1;
2745 PdeSrc.n.u1Accessed = 1;
2746 PdeSrc.n.u1User = 1;
2747
2748 /*
2749 * Allocate & map the page table.
2750 */
2751 PSHWPT pPTDst;
2752 PPGMPOOLPAGE pShwPage;
2753 RTGCPHYS GCPhys;
2754
2755 /* Virtual address = physical address */
2756 GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
2757# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_EPT
2758 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2759# else
2760 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2761# endif
2762
2763 if ( rc == VINF_SUCCESS
2764 || rc == VINF_PGM_CACHED_PAGE)
2765 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2766 else
2767 AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2768
2769 PdeDst.u &= X86_PDE_AVL_MASK;
2770 PdeDst.u |= pShwPage->Core.Key;
2771 PdeDst.n.u1Present = 1;
2772 PdeDst.n.u1Write = 1;
2773# if PGM_SHW_TYPE == PGM_TYPE_EPT
2774 PdeDst.n.u1Execute = 1;
2775# else
2776 PdeDst.n.u1User = 1;
2777 PdeDst.n.u1Accessed = 1;
2778# endif
2779 *pPdeDst = PdeDst;
2780
2781 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2782 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2783 return rc;
2784
2785#else
2786 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2787 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2788 return VERR_INTERNAL_ERROR;
2789#endif
2790}
2791
2792
2793
2794/**
2795 * Prefetch a page/set of pages.
2796 *
2797 * Typically used to sync commonly used pages before entering raw mode
2798 * after a CR3 reload.
2799 *
2800 * @returns VBox status code.
2801 * @param pVM VM handle.
2802 * @param GCPtrPage Page to invalidate.
2803 */
2804PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2805{
2806#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2807 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
2808 /*
2809 * Check that all Guest levels thru the PDE are present, getting the
2810 * PD and PDE in the processes.
2811 */
2812 int rc = VINF_SUCCESS;
2813# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2814# if PGM_GST_TYPE == PGM_TYPE_32BIT
2815 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2816 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVM->pgm.s);
2817# elif PGM_GST_TYPE == PGM_TYPE_PAE
2818 unsigned iPDSrc;
2819 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc, NULL);
2820 if (!pPDSrc)
2821 return VINF_SUCCESS; /* not present */
2822# elif PGM_GST_TYPE == PGM_TYPE_AMD64
2823 unsigned iPDSrc;
2824 PX86PML4E pPml4eSrc;
2825 X86PDPE PdpeSrc;
2826 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
2827 if (!pPDSrc)
2828 return VINF_SUCCESS; /* not present */
2829# endif
2830 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2831# else
2832 PGSTPD pPDSrc = NULL;
2833 const unsigned iPDSrc = 0;
2834 GSTPDE PdeSrc;
2835
2836 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2837 PdeSrc.n.u1Present = 1;
2838 PdeSrc.n.u1Write = 1;
2839 PdeSrc.n.u1Accessed = 1;
2840 PdeSrc.n.u1User = 1;
2841# endif
2842
2843 if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2844 {
2845# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2846 const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2847# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2848 const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2849# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2850 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2851 PX86PDPAE pPDDst;
2852 X86PDEPAE PdeDst;
2853
2854# if PGM_GST_TYPE == PGM_TYPE_PROT
2855 /* AMD-V nested paging */
2856 X86PML4E Pml4eSrc;
2857 X86PDPE PdpeSrc;
2858 PX86PML4E pPml4eSrc = &Pml4eSrc;
2859
2860 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
2861 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_NX | X86_PML4E_A;
2862 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_NX | X86_PDPE_A;
2863# endif
2864
2865 int rc = PGMShwSyncLongModePDPtr(pVM, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
2866 if (rc != VINF_SUCCESS)
2867 {
2868 AssertRC(rc);
2869 return rc;
2870 }
2871 Assert(pPDDst);
2872 PdeDst = pPDDst->a[iPDDst];
2873# endif
2874 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2875 {
2876 if (!PdeDst.n.u1Present)
2877 /** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2878 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2879 else
2880 {
2881 /** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2882 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2883 * makes no sense to prefetch more than one page.
2884 */
2885 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2886 if (RT_SUCCESS(rc))
2887 rc = VINF_SUCCESS;
2888 }
2889 }
2890 }
2891 return rc;
2892
2893#elif PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
2894 return VINF_SUCCESS; /* ignore */
2895#endif
2896}
2897
2898
2899
2900
2901/**
2902 * Syncs a page during a PGMVerifyAccess() call.
2903 *
2904 * @returns VBox status code (informational included).
2905 * @param GCPtrPage The address of the page to sync.
2906 * @param fPage The effective guest page flags.
2907 * @param uErr The trap error code.
2908 */
2909PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2910{
2911 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2912
2913 Assert(!HWACCMIsNestedPagingActive(pVM));
2914#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_TYPE_AMD64) \
2915 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
2916
2917# ifndef IN_RING0
2918 if (!(fPage & X86_PTE_US))
2919 {
2920 /*
2921 * Mark this page as safe.
2922 */
2923 /** @todo not correct for pages that contain both code and data!! */
2924 Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
2925 CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
2926 }
2927# endif
2928
2929 /*
2930 * Get guest PD and index.
2931 */
2932# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2933# if PGM_GST_TYPE == PGM_TYPE_32BIT
2934 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2935 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVM->pgm.s);
2936# elif PGM_GST_TYPE == PGM_TYPE_PAE
2937 unsigned iPDSrc;
2938 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc, NULL);
2939
2940 if (pPDSrc)
2941 {
2942 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
2943 return VINF_EM_RAW_GUEST_TRAP;
2944 }
2945# elif PGM_GST_TYPE == PGM_TYPE_AMD64
2946 unsigned iPDSrc;
2947 PX86PML4E pPml4eSrc;
2948 X86PDPE PdpeSrc;
2949 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
2950 if (!pPDSrc)
2951 {
2952 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
2953 return VINF_EM_RAW_GUEST_TRAP;
2954 }
2955# endif
2956# else
2957 PGSTPD pPDSrc = NULL;
2958 const unsigned iPDSrc = 0;
2959# endif
2960 int rc = VINF_SUCCESS;
2961
2962 /*
2963 * First check if the shadow pd is present.
2964 */
2965# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2966 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2967# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2968 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2969# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2970 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2971 PX86PDPAE pPDDst;
2972 PX86PDEPAE pPdeDst;
2973
2974# if PGM_GST_TYPE == PGM_TYPE_PROT
2975 /* AMD-V nested paging */
2976 X86PML4E Pml4eSrc;
2977 X86PDPE PdpeSrc;
2978 PX86PML4E pPml4eSrc = &Pml4eSrc;
2979
2980 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
2981 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_NX | X86_PML4E_A;
2982 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_NX | X86_PDPE_A;
2983# endif
2984
2985 rc = PGMShwSyncLongModePDPtr(pVM, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
2986 if (rc != VINF_SUCCESS)
2987 {
2988 AssertRC(rc);
2989 return rc;
2990 }
2991 Assert(pPDDst);
2992 pPdeDst = &pPDDst->a[iPDDst];
2993# endif
2994 if (!pPdeDst->n.u1Present)
2995 {
2996 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2997 AssertRC(rc);
2998 if (rc != VINF_SUCCESS)
2999 return rc;
3000 }
3001
3002# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3003 /* Check for dirty bit fault */
3004 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3005 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3006 Log(("PGMVerifyAccess: success (dirty)\n"));
3007 else
3008 {
3009 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3010#else
3011 {
3012 GSTPDE PdeSrc;
3013 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3014 PdeSrc.n.u1Present = 1;
3015 PdeSrc.n.u1Write = 1;
3016 PdeSrc.n.u1Accessed = 1;
3017 PdeSrc.n.u1User = 1;
3018
3019#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
3020 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3021 if (uErr & X86_TRAP_PF_US)
3022 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
3023 else /* supervisor */
3024 STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3025
3026 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
3027 if (RT_SUCCESS(rc))
3028 {
3029 /* Page was successfully synced */
3030 Log2(("PGMVerifyAccess: success (sync)\n"));
3031 rc = VINF_SUCCESS;
3032 }
3033 else
3034 {
3035 Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", GCPtrPage, rc));
3036 return VINF_EM_RAW_GUEST_TRAP;
3037 }
3038 }
3039 return rc;
3040
3041#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3042
3043 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3044 return VERR_INTERNAL_ERROR;
3045#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3046}
3047
3048
3049#if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3050# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
3051/**
3052 * Figures out which kind of shadow page this guest PDE warrants.
3053 *
3054 * @returns Shadow page kind.
3055 * @param pPdeSrc The guest PDE in question.
3056 * @param cr4 The current guest cr4 value.
3057 */
3058DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
3059{
3060# if PMG_GST_TYPE == PGM_TYPE_AMD64
3061 if (!pPdeSrc->n.u1Size)
3062# else
3063 if (!pPdeSrc->n.u1Size || !(cr4 & X86_CR4_PSE))
3064# endif
3065 return BTH_PGMPOOLKIND_PT_FOR_PT;
3066 //switch (pPdeSrc->u & (X86_PDE4M_RW | X86_PDE4M_US /*| X86_PDE4M_PAE_NX*/))
3067 //{
3068 // case 0:
3069 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
3070 // case X86_PDE4M_RW:
3071 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
3072 // case X86_PDE4M_US:
3073 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
3074 // case X86_PDE4M_RW | X86_PDE4M_US:
3075 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
3076# if 0
3077 // case X86_PDE4M_PAE_NX:
3078 // return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
3079 // case X86_PDE4M_RW | X86_PDE4M_PAE_NX:
3080 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
3081 // case X86_PDE4M_US | X86_PDE4M_PAE_NX:
3082 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
3083 // case X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PAE_NX:
3084 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
3085# endif
3086 return BTH_PGMPOOLKIND_PT_FOR_BIG;
3087 //}
3088}
3089# endif
3090#endif
3091
3092#undef MY_STAM_COUNTER_INC
3093#define MY_STAM_COUNTER_INC(a) do { } while (0)
3094
3095
3096/**
3097 * Syncs the paging hierarchy starting at CR3.
3098 *
3099 * @returns VBox status code, no specials.
3100 * @param pVM The virtual machine.
3101 * @param cr0 Guest context CR0 register
3102 * @param cr3 Guest context CR3 register
3103 * @param cr4 Guest context CR4 register
3104 * @param fGlobal Including global page directories or not
3105 */
3106PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3107{
3108 if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
3109 fGlobal = true; /* Change this CR3 reload to be a global one. */
3110
3111#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3112 /*
3113 * Update page access handlers.
3114 * The virtual are always flushed, while the physical are only on demand.
3115 * WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3116 * have to look into that later because it will have a bad influence on the performance.
3117 * @note SvL: There's no need for that. Just invalidate the virtual range(s).
3118 * bird: Yes, but that won't work for aliases.
3119 */
3120 /** @todo this MUST go away. See #1557. */
3121 STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3122 PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3123 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3124#endif
3125
3126#ifdef PGMPOOL_WITH_MONITORING
3127 int rc = pgmPoolSyncCR3(pVM);
3128 if (rc != VINF_SUCCESS)
3129 return rc;
3130#endif
3131
3132#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3133 /** @todo check if this is really necessary */
3134 HWACCMFlushTLB(pVM);
3135 return VINF_SUCCESS;
3136
3137#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3138 /* No need to check all paging levels; we zero out the shadow parts when the guest modifies its tables. */
3139 return VINF_SUCCESS;
3140#else
3141
3142 Assert(fGlobal || (cr4 & X86_CR4_PGE));
3143 MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3Global) : &pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3NotGlobal));
3144
3145# if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3146# if PGM_GST_TYPE == PGM_TYPE_AMD64
3147 bool fBigPagesSupported = true;
3148# else
3149 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
3150# endif
3151
3152 /*
3153 * Get page directory addresses.
3154 */
3155# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3156 PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
3157# else /* PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64*/
3158# if PGM_GST_TYPE == PGM_TYPE_32BIT
3159 PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
3160# endif
3161# endif
3162
3163# if PGM_GST_TYPE == PGM_TYPE_32BIT
3164 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVM->pgm.s);
3165 Assert(pPDSrc);
3166# ifndef IN_RC
3167 Assert(PGMPhysGCPhys2HCPtrAssert(pVM, (RTGCPHYS)(cr3 & GST_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
3168# endif
3169# endif
3170
3171 /*
3172 * Iterate the page directory.
3173 */
3174 PPGMMAPPING pMapping;
3175 unsigned iPdNoMapping;
3176 const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
3177 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3178
3179 /* Only check mappings if they are supposed to be put into the shadow page table. */
3180 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
3181 {
3182 pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
3183 iPdNoMapping = (pMapping) ? (pMapping->GCPtr >> GST_PD_SHIFT) : ~0U;
3184 }
3185 else
3186 {
3187 pMapping = 0;
3188 iPdNoMapping = ~0U;
3189 }
3190# if PGM_GST_TYPE == PGM_TYPE_AMD64
3191 for (uint64_t iPml4e = 0; iPml4e < X86_PG_PAE_ENTRIES; iPml4e++)
3192 {
3193 PPGMPOOLPAGE pShwPdpt = NULL;
3194 PX86PML4E pPml4eSrc, pPml4eDst;
3195 RTGCPHYS GCPhysPdptSrc;
3196
3197 pPml4eSrc = &pVM->pgm.s.CTXSUFF(pGstPaePML4)->a[iPml4e];
3198 pPml4eDst = &pVM->pgm.s.CTXMID(p,PaePML4)->a[iPml4e];
3199
3200 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3201 if (!pPml4eDst->n.u1Present)
3202 continue;
3203 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3204
3205 GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3206
3207 /* Anything significant changed? */
3208 if ( pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present
3209 || GCPhysPdptSrc != pShwPdpt->GCPhys)
3210 {
3211 /* Free it. */
3212 LogFlow(("SyncCR3: Out-of-sync PML4E (GCPhys) GCPtr=%RX64 %RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
3213 (uint64_t)iPml4e << X86_PML4_SHIFT, pShwPdpt->GCPhys, GCPhysPdptSrc, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
3214 pgmPoolFreeByPage(pPool, pShwPdpt, pVM->pgm.s.pHCShwAmd64CR3->idx, iPml4e);
3215 pPml4eDst->u = 0;
3216 continue;
3217 }
3218 /* Force an attribute sync. */
3219 pPml4eDst->n.u1User = pPml4eSrc->n.u1User;
3220 pPml4eDst->n.u1Write = pPml4eSrc->n.u1Write;
3221 pPml4eDst->n.u1NoExecute = pPml4eSrc->n.u1NoExecute;
3222
3223# else
3224 {
3225# endif
3226# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3227 for (uint64_t iPdpte = 0; iPdpte < GST_PDPE_ENTRIES; iPdpte++)
3228 {
3229 unsigned iPDSrc;
3230# if PGM_GST_TYPE == PGM_TYPE_PAE
3231 X86PDPE PdpeSrc;
3232 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, iPdpte << X86_PDPT_SHIFT, &iPDSrc, &PdpeSrc);
3233 PX86PDPAE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
3234 PX86PDEPAE pPDEDst = &pPDPAE->a[iPdpte * X86_PG_PAE_ENTRIES];
3235 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT); NOREF(pPdptDst);
3236
3237 if (pPDSrc == NULL)
3238 {
3239 /* PDPE not present */
3240 if (pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPdpte].n.u1Present)
3241 {
3242 LogFlow(("SyncCR3: guest PDPE %d not present; clear shw pdpe\n", iPdpte));
3243 /* for each page directory entry */
3244 for (unsigned iPD = 0; iPD < RT_ELEMENTS(pPDSrc->a); iPD++)
3245 {
3246 if ( pPDEDst[iPD].n.u1Present
3247 && !(pPDEDst[iPD].u & PGM_PDFLAGS_MAPPING))
3248 {
3249 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst[iPD].u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdpte * X86_PG_PAE_ENTRIES + iPD);
3250 pPDEDst[iPD].u = 0;
3251 }
3252 }
3253 }
3254 if (!(pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPdpte].u & PGM_PLXFLAGS_MAPPING))
3255 pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPdpte].n.u1Present = 0;
3256 continue;
3257 }
3258# else /* PGM_GST_TYPE != PGM_TYPE_PAE */
3259 PPGMPOOLPAGE pShwPde = NULL;
3260 RTGCPHYS GCPhysPdeSrc;
3261 PX86PDPE pPdpeDst;
3262 PX86PML4E pPml4eSrc;
3263 X86PDPE PdpeSrc;
3264 PX86PDPT pPdptDst;
3265 PX86PDPAE pPDDst;
3266 PX86PDEPAE pPDEDst;
3267 RTGCUINTPTR GCPtr = (iPml4e << X86_PML4_SHIFT) || (iPdpte << X86_PDPT_SHIFT);
3268 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3269
3270 int rc = PGMShwGetLongModePDPtr(pVM, GCPtr, &pPdptDst, &pPDDst);
3271 if (rc != VINF_SUCCESS)
3272 {
3273 if (rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
3274 break; /* next PML4E */
3275
3276 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3277 continue; /* next PDPTE */
3278 }
3279 Assert(pPDDst);
3280 pPDEDst = &pPDDst->a[0];
3281 Assert(iPDSrc == 0);
3282
3283 pPdpeDst = &pPdptDst->a[iPdpte];
3284
3285 /* Fetch the pgm pool shadow descriptor if the shadow pdpte is present. */
3286 if (!pPdpeDst->n.u1Present)
3287 continue; /* next PDPTE */
3288
3289 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3290 GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3291
3292 /* Anything significant changed? */
3293 if ( PdpeSrc.n.u1Present != pPdpeDst->n.u1Present
3294 || GCPhysPdeSrc != pShwPde->GCPhys)
3295 {
3296 /* Free it. */
3297 LogFlow(("SyncCR3: Out-of-sync PDPE (GCPhys) GCPtr=%RX64 %RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
3298 ((uint64_t)iPml4e << X86_PML4_SHIFT) + ((uint64_t)iPdpte << X86_PDPT_SHIFT), pShwPde->GCPhys, GCPhysPdeSrc, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
3299
3300 /* Mark it as not present if there's no hypervisor mapping present. (bit flipped at the top of Trap0eHandler) */
3301 Assert(!(pPdpeDst->u & PGM_PLXFLAGS_MAPPING));
3302 pgmPoolFreeByPage(pPool, pShwPde, pShwPde->idx, iPdpte);
3303 pPdpeDst->u = 0;
3304 continue; /* next guest PDPTE */
3305 }
3306 /* Force an attribute sync. */
3307 pPdpeDst->lm.u1User = PdpeSrc.lm.u1User;
3308 pPdpeDst->lm.u1Write = PdpeSrc.lm.u1Write;
3309 pPdpeDst->lm.u1NoExecute = PdpeSrc.lm.u1NoExecute;
3310# endif /* PGM_GST_TYPE != PGM_TYPE_PAE */
3311
3312# else /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
3313 {
3314# endif /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
3315 for (unsigned iPD = 0; iPD < RT_ELEMENTS(pPDSrc->a); iPD++)
3316 {
3317# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3318 Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
3319# elif PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3320 AssertMsg(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst, ("%p vs %p\n", &pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512], pPDEDst));
3321# endif
3322 GSTPDE PdeSrc = pPDSrc->a[iPD];
3323 if ( PdeSrc.n.u1Present
3324 && (PdeSrc.n.u1User || fRawR0Enabled))
3325 {
3326# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3327 || PGM_GST_TYPE == PGM_TYPE_PAE) \
3328 && !defined(PGM_WITHOUT_MAPPINGS)
3329
3330 /*
3331 * Check for conflicts with GC mappings.
3332 */
3333# if PGM_GST_TYPE == PGM_TYPE_PAE
3334 if (iPD + iPdpte * X86_PG_PAE_ENTRIES == iPdNoMapping)
3335# else
3336 if (iPD == iPdNoMapping)
3337# endif
3338 {
3339 if (pVM->pgm.s.fMappingsFixed)
3340 {
3341 /* It's fixed, just skip the mapping. */
3342 const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
3343 iPD += cPTs - 1;
3344 pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3345 pMapping = pMapping->CTX_SUFF(pNext);
3346 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3347 continue;
3348 }
3349# ifdef IN_RING3
3350# if PGM_GST_TYPE == PGM_TYPE_32BIT
3351 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
3352# elif PGM_GST_TYPE == PGM_TYPE_PAE
3353 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPdpte << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
3354# endif
3355 if (RT_FAILURE(rc))
3356 return rc;
3357
3358 /*
3359 * Update iPdNoMapping and pMapping.
3360 */
3361 pMapping = pVM->pgm.s.pMappingsR3;
3362 while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
3363 pMapping = pMapping->pNextR3;
3364 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3365# else
3366 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3367 return VINF_PGM_SYNC_CR3;
3368# endif
3369 }
3370# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3371 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3372# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3373
3374 /*
3375 * Sync page directory entry.
3376 *
3377 * The current approach is to allocated the page table but to set
3378 * the entry to not-present and postpone the page table synching till
3379 * it's actually used.
3380 */
3381# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3382 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
3383# elif PGM_GST_TYPE == PGM_TYPE_PAE
3384 const unsigned iPdShw = iPD + iPdpte * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
3385# else
3386 const unsigned iPdShw = iPD; NOREF(iPdShw);
3387# endif
3388 {
3389 SHWPDE PdeDst = *pPDEDst;
3390 if (PdeDst.n.u1Present)
3391 {
3392 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
3393 RTGCPHYS GCPhys;
3394 if ( !PdeSrc.b.u1Size
3395 || !fBigPagesSupported)
3396 {
3397 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
3398# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3399 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3400 GCPhys |= i * (PAGE_SIZE / 2);
3401# endif
3402 }
3403 else
3404 {
3405 GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3406# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3407 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
3408 GCPhys |= i * X86_PAGE_2M_SIZE;
3409# endif
3410 }
3411
3412 if ( pShwPage->GCPhys == GCPhys
3413 && pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
3414 && ( pShwPage->fCached
3415 || ( !fGlobal
3416 && ( false
3417# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
3418 || ( (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
3419# if PGM_GST_TYPE == PGM_TYPE_AMD64
3420 && (cr4 & X86_CR4_PGE)) /* global 2/4MB page. */
3421# else
3422 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE)) /* global 2/4MB page. */
3423# endif
3424 || ( !pShwPage->fSeenNonGlobal
3425 && (cr4 & X86_CR4_PGE))
3426# endif
3427 )
3428 )
3429 )
3430 && ( (PdeSrc.u & (X86_PDE_US | X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US | X86_PDE_RW))
3431 || ( fBigPagesSupported
3432 && ((PdeSrc.u & (X86_PDE_US | X86_PDE4M_PS | X86_PDE4M_D)) | PGM_PDFLAGS_TRACK_DIRTY)
3433 == ((PdeDst.u & (X86_PDE_US | X86_PDE_RW | PGM_PDFLAGS_TRACK_DIRTY)) | X86_PDE4M_PS))
3434 )
3435 )
3436 {
3437# ifdef VBOX_WITH_STATISTICS
3438 if ( !fGlobal
3439 && (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
3440# if PGM_GST_TYPE == PGM_TYPE_AMD64
3441 && (cr4 & X86_CR4_PGE)) /* global 2/4MB page. */
3442# else
3443 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE))
3444# endif
3445 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3DstSkippedGlobalPD));
3446 else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
3447 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3DstSkippedGlobalPT));
3448 else
3449 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3DstCacheHit));
3450# endif /* VBOX_WITH_STATISTICS */
3451 /** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
3452 * The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
3453 //# ifdef PGMPOOL_WITH_CACHE
3454 // pgmPoolCacheUsed(pPool, pShwPage);
3455 //# endif
3456 }
3457 else
3458 {
3459# if PGM_GST_TYPE == PGM_TYPE_AMD64
3460 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPdShw);
3461# else
3462 pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
3463# endif
3464 pPDEDst->u = 0;
3465 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3DstFreed));
3466 }
3467 }
3468 else
3469 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3DstNotPresent));
3470 pPDEDst++;
3471 }
3472 }
3473# if PGM_GST_TYPE == PGM_TYPE_PAE
3474 else if (iPD + iPdpte * X86_PG_PAE_ENTRIES != iPdNoMapping)
3475# else
3476 else if (iPD != iPdNoMapping)
3477# endif
3478 {
3479 /*
3480 * Check if there is any page directory to mark not present here.
3481 */
3482# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3483 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
3484# elif PGM_GST_TYPE == PGM_TYPE_PAE
3485 const unsigned iPdShw = iPD + iPdpte * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
3486# else
3487 const unsigned iPdShw = iPD; NOREF(iPdShw);
3488# endif
3489 {
3490 if (pPDEDst->n.u1Present)
3491 {
3492# if PGM_GST_TYPE == PGM_TYPE_AMD64
3493 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), pShwPde->idx, iPdShw);
3494# else
3495 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
3496# endif
3497 pPDEDst->u = 0;
3498 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3DstFreedSrcNP));
3499 }
3500 pPDEDst++;
3501 }
3502 }
3503 else
3504 {
3505# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3506 || PGM_GST_TYPE == PGM_TYPE_PAE) \
3507 && !defined(PGM_WITHOUT_MAPPINGS)
3508
3509 const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
3510
3511 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3512 if (pVM->pgm.s.fMappingsFixed)
3513 {
3514 /* It's fixed, just skip the mapping. */
3515 pMapping = pMapping->CTX_SUFF(pNext);
3516 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3517 }
3518 else
3519 {
3520 /*
3521 * Check for conflicts for subsequent pagetables
3522 * and advance to the next mapping.
3523 */
3524 iPdNoMapping = ~0U;
3525 unsigned iPT = cPTs;
3526 while (iPT-- > 1)
3527 {
3528 if ( pPDSrc->a[iPD + iPT].n.u1Present
3529 && (pPDSrc->a[iPD + iPT].n.u1User || fRawR0Enabled))
3530 {
3531# ifdef IN_RING3
3532# if PGM_GST_TYPE == PGM_TYPE_32BIT
3533 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
3534# elif PGM_GST_TYPE == PGM_TYPE_PAE
3535 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPdpte << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
3536# endif
3537 if (RT_FAILURE(rc))
3538 return rc;
3539
3540 /*
3541 * Update iPdNoMapping and pMapping.
3542 */
3543 pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
3544 while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
3545 pMapping = pMapping->CTX_SUFF(pNext);
3546 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3547 break;
3548# else
3549 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3550 return VINF_PGM_SYNC_CR3;
3551# endif
3552 }
3553 }
3554 if (iPdNoMapping == ~0U && pMapping)
3555 {
3556 pMapping = pMapping->CTX_SUFF(pNext);
3557 if (pMapping)
3558 iPdNoMapping = pMapping->GCPtr >> GST_PD_SHIFT;
3559 }
3560 }
3561
3562 /* advance. */
3563 iPD += cPTs - 1;
3564 pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3565# if PGM_GST_TYPE != PGM_SHW_TYPE
3566 AssertCompile(PGM_GST_TYPE == PGM_TYPE_32BIT && PGM_SHW_TYPE == PGM_TYPE_PAE);
3567# endif
3568# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3569 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3570# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3571 }
3572
3573 } /* for iPD */
3574 } /* for each PDPTE (PAE) */
3575 } /* for each page map level 4 entry (amd64) */
3576 return VINF_SUCCESS;
3577
3578# else /* guest real and protected mode */
3579 return VINF_SUCCESS;
3580# endif
3581#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
3582}
3583
3584
3585
3586
3587#ifdef VBOX_STRICT
3588#ifdef IN_RC
3589# undef AssertMsgFailed
3590# define AssertMsgFailed Log
3591#endif
3592#ifdef IN_RING3
3593# include <VBox/dbgf.h>
3594
3595/**
3596 * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3597 *
3598 * @returns VBox status code (VINF_SUCCESS).
3599 * @param pVM The VM handle.
3600 * @param cr3 The root of the hierarchy.
3601 * @param crr The cr4, only PAE and PSE is currently used.
3602 * @param fLongMode Set if long mode, false if not long mode.
3603 * @param cMaxDepth Number of levels to dump.
3604 * @param pHlp Pointer to the output functions.
3605 */
3606__BEGIN_DECLS
3607VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3608__END_DECLS
3609
3610#endif
3611
3612/**
3613 * Checks that the shadow page table is in sync with the guest one.
3614 *
3615 * @returns The number of errors.
3616 * @param pVM The virtual machine.
3617 * @param cr3 Guest context CR3 register
3618 * @param cr4 Guest context CR4 register
3619 * @param GCPtr Where to start. Defaults to 0.
3620 * @param cb How much to check. Defaults to everything.
3621 */
3622PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
3623{
3624#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3625 return 0;
3626#else
3627 unsigned cErrors = 0;
3628
3629#if PGM_GST_TYPE == PGM_TYPE_PAE
3630 /** @todo currently broken; crashes below somewhere */
3631 AssertFailed();
3632#endif
3633
3634#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3635 || PGM_GST_TYPE == PGM_TYPE_PAE \
3636 || PGM_GST_TYPE == PGM_TYPE_AMD64
3637
3638# if PGM_GST_TYPE == PGM_TYPE_AMD64
3639 bool fBigPagesSupported = true;
3640# else
3641 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
3642# endif
3643 PPGM pPGM = &pVM->pgm.s;
3644 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3645 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3646# ifndef IN_RING0
3647 RTHCPHYS HCPhys; /* general usage. */
3648# endif
3649 int rc;
3650
3651 /*
3652 * Check that the Guest CR3 and all its mappings are correct.
3653 */
3654 AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3655 ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3656 false);
3657# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3658# if PGM_GST_TYPE == PGM_TYPE_32BIT
3659 rc = PGMShwGetPage(pVM, (RTGCPTR)pPGM->pGuestPDRC, NULL, &HCPhysShw);
3660# else
3661 rc = PGMShwGetPage(pVM, (RTGCPTR)pPGM->pGstPaePDPTRC, NULL, &HCPhysShw);
3662# endif
3663 AssertRCReturn(rc, 1);
3664 HCPhys = NIL_RTHCPHYS;
3665 rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3666 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3667# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3668 RTGCPHYS GCPhys;
3669 rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGuestPDR3, &GCPhys);
3670 AssertRCReturn(rc, 1);
3671 AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3672# endif
3673#endif /* !IN_RING0 */
3674
3675 /*
3676 * Get and check the Shadow CR3.
3677 */
3678# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3679 unsigned cPDEs = X86_PG_ENTRIES;
3680 unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3681# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3682# if PGM_GST_TYPE == PGM_TYPE_32BIT
3683 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3684# else
3685 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3686# endif
3687 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3688# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3689 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3690 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3691# endif
3692 if (cb != ~(RTGCUINTPTR)0)
3693 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3694
3695/** @todo call the other two PGMAssert*() functions. */
3696
3697# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
3698 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3699# endif
3700
3701# if PGM_GST_TYPE == PGM_TYPE_AMD64
3702 unsigned iPml4e = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3703
3704 for (; iPml4e < X86_PG_PAE_ENTRIES; iPml4e++)
3705 {
3706 PPGMPOOLPAGE pShwPdpt = NULL;
3707 PX86PML4E pPml4eSrc;
3708 PX86PML4E pPml4eDst;
3709 RTGCPHYS GCPhysPdptSrc;
3710
3711 pPml4eSrc = &pVM->pgm.s.CTXSUFF(pGstPaePML4)->a[iPml4e];
3712 pPml4eDst = &pVM->pgm.s.CTXMID(p,PaePML4)->a[iPml4e];
3713
3714 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3715 if (!pPml4eDst->n.u1Present)
3716 {
3717 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3718 continue;
3719 }
3720
3721# if PGM_GST_TYPE == PGM_TYPE_PAE
3722 /* not correct to call pgmPoolGetPage */
3723 AssertFailed();
3724# endif
3725 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3726 GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3727
3728 if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3729 {
3730 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3731 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3732 cErrors++;
3733 continue;
3734 }
3735
3736 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3737 {
3738 AssertMsgFailed(("Physical address doesn't match! iPml4e %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4e, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3739 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3740 cErrors++;
3741 continue;
3742 }
3743
3744 if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3745 || pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3746 || pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3747 {
3748 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3749 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3750 cErrors++;
3751 continue;
3752 }
3753# else
3754 {
3755# endif
3756
3757# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
3758 /*
3759 * Check the PDPTEs too.
3760 */
3761 unsigned iPdpte = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3762
3763 for (;iPdpte <= SHW_PDPT_MASK; iPdpte++)
3764 {
3765 unsigned iPDSrc;
3766 PPGMPOOLPAGE pShwPde = NULL;
3767 PX86PDPE pPdpeDst;
3768 RTGCPHYS GCPhysPdeSrc;
3769# if PGM_GST_TYPE == PGM_TYPE_PAE
3770 X86PDPE PdpeSrc;
3771 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtr, &iPDSrc, &PdpeSrc);
3772 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
3773 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT);
3774# else
3775 PX86PML4E pPml4eSrc;
3776 X86PDPE PdpeSrc;
3777 PX86PDPT pPdptDst;
3778 PX86PDPAE pPDDst;
3779 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3780
3781 rc = PGMShwGetLongModePDPtr(pVM, GCPtr, &pPdptDst, &pPDDst);
3782 if (rc != VINF_SUCCESS)
3783 {
3784 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3785 GCPtr += 512 * _2M;
3786 continue; /* next PDPTE */
3787 }
3788 Assert(pPDDst);
3789# endif
3790 Assert(iPDSrc == 0);
3791
3792 pPdpeDst = &pPdptDst->a[iPdpte];
3793
3794 if (!pPdpeDst->n.u1Present)
3795 {
3796 GCPtr += 512 * _2M;
3797 continue; /* next PDPTE */
3798 }
3799
3800 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3801 GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3802
3803 if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3804 {
3805 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3806 GCPtr += 512 * _2M;
3807 cErrors++;
3808 continue;
3809 }
3810
3811 if (GCPhysPdeSrc != pShwPde->GCPhys)
3812 {
3813# if PGM_GST_TYPE == PGM_TYPE_AMD64
3814 AssertMsgFailed(("Physical address doesn't match! iPml4e %d iPdpte %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4e, iPdpte, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3815# else
3816 AssertMsgFailed(("Physical address doesn't match! iPdpte %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpte, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3817# endif
3818 GCPtr += 512 * _2M;
3819 cErrors++;
3820 continue;
3821 }
3822
3823# if PGM_GST_TYPE == PGM_TYPE_AMD64
3824 if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3825 || pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3826 || pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3827 {
3828 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3829 GCPtr += 512 * _2M;
3830 cErrors++;
3831 continue;
3832 }
3833# endif
3834
3835# else
3836 {
3837# endif
3838# if PGM_GST_TYPE == PGM_TYPE_32BIT
3839 GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(&pVM->pgm.s);
3840# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3841 PCX86PD pPDDst = pPGM->CTXMID(p,32BitPD);
3842# else
3843 PCX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
3844# endif
3845# endif
3846 /*
3847 * Iterate the shadow page directory.
3848 */
3849 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3850 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3851
3852 for (;
3853 iPDDst < cPDEs;
3854 iPDDst++, GCPtr += cIncrement)
3855 {
3856 const SHWPDE PdeDst = pPDDst->a[iPDDst];
3857 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3858 {
3859 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3860 if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3861 {
3862 AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3863 cErrors++;
3864 continue;
3865 }
3866 }
3867 else if ( (PdeDst.u & X86_PDE_P)
3868 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3869 )
3870 {
3871 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3872 PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3873 if (!pPoolPage)
3874 {
3875 AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3876 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3877 cErrors++;
3878 continue;
3879 }
3880 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3881
3882 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
3883 {
3884 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3885 GCPtr, (uint64_t)PdeDst.u));
3886 cErrors++;
3887 }
3888
3889 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
3890 {
3891 AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3892 GCPtr, (uint64_t)PdeDst.u));
3893 cErrors++;
3894 }
3895
3896 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3897 if (!PdeSrc.n.u1Present)
3898 {
3899 AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3900 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3901 cErrors++;
3902 continue;
3903 }
3904
3905 if ( !PdeSrc.b.u1Size
3906 || !fBigPagesSupported)
3907 {
3908 GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3909# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3910 GCPhysGst |= (iPDDst & 1) * (PAGE_SIZE / 2);
3911# endif
3912 }
3913 else
3914 {
3915# if PGM_GST_TYPE == PGM_TYPE_32BIT
3916 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3917 {
3918 AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3919 GCPtr, (uint64_t)PdeSrc.u));
3920 cErrors++;
3921 continue;
3922 }
3923# endif
3924 GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3925# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3926 GCPhysGst |= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3927# endif
3928 }
3929
3930 if ( pPoolPage->enmKind
3931 != (!PdeSrc.b.u1Size || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3932 {
3933 AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3934 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3935 cErrors++;
3936 }
3937
3938 PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3939 if (!pPhysPage)
3940 {
3941 AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3942 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3943 cErrors++;
3944 continue;
3945 }
3946
3947 if (GCPhysGst != pPoolPage->GCPhys)
3948 {
3949 AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3950 GCPhysGst, pPoolPage->GCPhys, GCPtr));
3951 cErrors++;
3952 continue;
3953 }
3954
3955 if ( !PdeSrc.b.u1Size
3956 || !fBigPagesSupported)
3957 {
3958 /*
3959 * Page Table.
3960 */
3961 const GSTPT *pPTSrc;
3962 rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3963 if (RT_FAILURE(rc))
3964 {
3965 AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3966 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3967 cErrors++;
3968 continue;
3969 }
3970 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
3971 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
3972 {
3973 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3974 // (This problem will go away when/if we shadow multiple CR3s.)
3975 AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3976 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3977 cErrors++;
3978 continue;
3979 }
3980 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3981 {
3982 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3983 GCPtr, (uint64_t)PdeDst.u));
3984 cErrors++;
3985 continue;
3986 }
3987
3988 /* iterate the page table. */
3989# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3990 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3991 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3992# else
3993 const unsigned offPTSrc = 0;
3994# endif
3995 for (unsigned iPT = 0, off = 0;
3996 iPT < RT_ELEMENTS(pPTDst->a);
3997 iPT++, off += PAGE_SIZE)
3998 {
3999 const SHWPTE PteDst = pPTDst->a[iPT];
4000
4001 /* skip not-present entries. */
4002 if (!(PteDst.u & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4003 continue;
4004 Assert(PteDst.n.u1Present);
4005
4006 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4007 if (!PteSrc.n.u1Present)
4008 {
4009# ifdef IN_RING3
4010 PGMAssertHandlerAndFlagsInSync(pVM);
4011 PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
4012# endif
4013 AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4014 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4015 (PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
4016 cErrors++;
4017 continue;
4018 }
4019
4020 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
4021# if 1 /** @todo sync accessed bit properly... */
4022 fIgnoreFlags |= X86_PTE_A;
4023# endif
4024
4025 /* match the physical addresses */
4026 HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
4027 GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
4028
4029# ifdef IN_RING3
4030 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4031 if (RT_FAILURE(rc))
4032 {
4033 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
4034 {
4035 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4036 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4037 cErrors++;
4038 continue;
4039 }
4040 }
4041 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4042 {
4043 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4044 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4045 cErrors++;
4046 continue;
4047 }
4048# endif
4049
4050 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
4051 if (!pPhysPage)
4052 {
4053# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4054 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
4055 {
4056 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4057 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4058 cErrors++;
4059 continue;
4060 }
4061# endif
4062 if (PteDst.n.u1Write)
4063 {
4064 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4065 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4066 cErrors++;
4067 }
4068 fIgnoreFlags |= X86_PTE_RW;
4069 }
4070 else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
4071 {
4072 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4073 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4074 cErrors++;
4075 continue;
4076 }
4077
4078 /* flags */
4079 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4080 {
4081 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4082 {
4083 if (PteDst.n.u1Write)
4084 {
4085 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! HCPhys=%RHp PteSrc=%#RX64 PteDst=%#RX64\n",
4086 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4087 cErrors++;
4088 continue;
4089 }
4090 fIgnoreFlags |= X86_PTE_RW;
4091 }
4092 else
4093 {
4094 if (PteDst.n.u1Present)
4095 {
4096 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! HCPhys=%RHp PteSrc=%#RX64 PteDst=%#RX64\n",
4097 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4098 cErrors++;
4099 continue;
4100 }
4101 fIgnoreFlags |= X86_PTE_P;
4102 }
4103 }
4104 else
4105 {
4106 if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4107 {
4108 if (PteDst.n.u1Write)
4109 {
4110 AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4111 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4112 cErrors++;
4113 continue;
4114 }
4115 if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
4116 {
4117 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4118 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4119 cErrors++;
4120 continue;
4121 }
4122 if (PteDst.n.u1Dirty)
4123 {
4124 AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4125 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4126 cErrors++;
4127 }
4128# if 0 /** @todo sync access bit properly... */
4129 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4130 {
4131 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4132 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4133 cErrors++;
4134 }
4135 fIgnoreFlags |= X86_PTE_RW;
4136# else
4137 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4138# endif
4139 }
4140 else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4141 {
4142 /* access bit emulation (not implemented). */
4143 if (PteSrc.n.u1Accessed || PteDst.n.u1Present)
4144 {
4145 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4146 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4147 cErrors++;
4148 continue;
4149 }
4150 if (!PteDst.n.u1Accessed)
4151 {
4152 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4153 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4154 cErrors++;
4155 }
4156 fIgnoreFlags |= X86_PTE_P;
4157 }
4158# ifdef DEBUG_sandervl
4159 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4160# endif
4161 }
4162
4163 if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4164 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4165 )
4166 {
4167 AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4168 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4169 fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4170 cErrors++;
4171 continue;
4172 }
4173 } /* foreach PTE */
4174 }
4175 else
4176 {
4177 /*
4178 * Big Page.
4179 */
4180 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4181 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4182 {
4183 if (PdeDst.n.u1Write)
4184 {
4185 AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4186 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4187 cErrors++;
4188 continue;
4189 }
4190 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4191 {
4192 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4193 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4194 cErrors++;
4195 continue;
4196 }
4197# if 0 /** @todo sync access bit properly... */
4198 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4199 {
4200 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4201 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4202 cErrors++;
4203 }
4204 fIgnoreFlags |= X86_PTE_RW;
4205# else
4206 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4207# endif
4208 }
4209 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4210 {
4211 /* access bit emulation (not implemented). */
4212 if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
4213 {
4214 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4215 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4216 cErrors++;
4217 continue;
4218 }
4219 if (!PdeDst.n.u1Accessed)
4220 {
4221 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4222 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4223 cErrors++;
4224 }
4225 fIgnoreFlags |= X86_PTE_P;
4226 }
4227
4228 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4229 {
4230 AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4231 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4232 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4233 cErrors++;
4234 }
4235
4236 /* iterate the page table. */
4237 for (unsigned iPT = 0, off = 0;
4238 iPT < RT_ELEMENTS(pPTDst->a);
4239 iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4240 {
4241 const SHWPTE PteDst = pPTDst->a[iPT];
4242
4243 if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4244 {
4245 AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4246 GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4247 cErrors++;
4248 }
4249
4250 /* skip not-present entries. */
4251 if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4252 continue;
4253
4254 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
4255
4256 /* match the physical addresses */
4257 HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4258
4259# ifdef IN_RING3
4260 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4261 if (RT_FAILURE(rc))
4262 {
4263 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
4264 {
4265 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4266 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4267 cErrors++;
4268 }
4269 }
4270 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4271 {
4272 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4273 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4274 cErrors++;
4275 continue;
4276 }
4277# endif
4278 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
4279 if (!pPhysPage)
4280 {
4281# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4282 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
4283 {
4284 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4285 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4286 cErrors++;
4287 continue;
4288 }
4289# endif
4290 if (PteDst.n.u1Write)
4291 {
4292 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4293 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4294 cErrors++;
4295 }
4296 fIgnoreFlags |= X86_PTE_RW;
4297 }
4298 else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
4299 {
4300 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4301 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4302 cErrors++;
4303 continue;
4304 }
4305
4306 /* flags */
4307 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4308 {
4309 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4310 {
4311 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4312 {
4313 if (PteDst.n.u1Write)
4314 {
4315 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! HCPhys=%RHp PdeSrc=%#RX64 PteDst=%#RX64\n",
4316 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4317 cErrors++;
4318 continue;
4319 }
4320 fIgnoreFlags |= X86_PTE_RW;
4321 }
4322 }
4323 else
4324 {
4325 if (PteDst.n.u1Present)
4326 {
4327 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! HCPhys=%RHp PdeSrc=%#RX64 PteDst=%#RX64\n",
4328 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4329 cErrors++;
4330 continue;
4331 }
4332 fIgnoreFlags |= X86_PTE_P;
4333 }
4334 }
4335
4336 if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4337 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4338 )
4339 {
4340 AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4341 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4342 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4343 cErrors++;
4344 continue;
4345 }
4346 } /* for each PTE */
4347 }
4348 }
4349 /* not present */
4350
4351 } /* for each PDE */
4352
4353 } /* for each PDPTE */
4354
4355 } /* for each PML4E */
4356
4357# ifdef DEBUG
4358 if (cErrors)
4359 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4360# endif
4361
4362#endif
4363 return cErrors;
4364
4365#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4366}
4367#endif /* VBOX_STRICT */
4368
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette