VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 9871

Last change on this file since 9871 was 9871, checked in by vboxsync, 16 years ago

Same goes for X86_PTE_G. (big page to pt mapping)
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 185.9 KB
Line 
1/* $Id: PGMAllBth.h 9871 2008-06-23 12:57:47Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * This file is a big challenge!
6 */
7
8/*
9 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
10 *
11 * This file is part of VirtualBox Open Source Edition (OSE), as
12 * available from http://www.virtualbox.org. This file is free software;
13 * you can redistribute it and/or modify it under the terms of the GNU
14 * General Public License (GPL) as published by the Free Software
15 * Foundation, in version 2 as it comes in the "COPYING" file of the
16 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20 * Clara, CA 95054 USA or visit http://www.sun.com if you need
21 * additional information or have any questions.
22 */
23
24/*******************************************************************************
25* Internal Functions *
26*******************************************************************************/
27__BEGIN_DECLS
28PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
33PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36#ifdef VBOX_STRICT
37PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38#endif
39#ifdef PGMPOOL_WITH_USER_TRACKING
40DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41#endif
42__END_DECLS
43
44
45/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
46#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED
47# error "Invalid combination; PAE guest implies PAE shadow"
48#endif
49
50#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
51 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED)
52# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
53#endif
54
55#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
56 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED)
57# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
58#endif
59
60#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED) \
61 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
62# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
63#endif
64
65#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
66# define PGM_WITHOUT_MAPPINGS
67#endif
68
69/**
70 * #PF Handler for raw-mode guest execution.
71 *
72 * @returns VBox status code (appropriate for trap handling and GC return).
73 * @param pVM VM Handle.
74 * @param uErr The trap error code.
75 * @param pRegFrame Trap register frame.
76 * @param pvFault The fault address.
77 */
78PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
79{
80#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
81 && PGM_SHW_TYPE != PGM_TYPE_NESTED
82
83# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
84 /*
85 * Hide the instruction fetch trap indicator for now.
86 */
87 /** @todo NXE will change this and we must fix NXE in the switcher too! */
88 if (uErr & X86_TRAP_PF_ID)
89 {
90 uErr &= ~X86_TRAP_PF_ID;
91 TRPMSetErrorCode(pVM, uErr);
92 }
93# endif
94
95 /*
96 * Get PDs.
97 */
98 int rc;
99# if PGM_WITH_PAGING(PGM_GST_TYPE)
100# if PGM_GST_TYPE == PGM_TYPE_32BIT
101 const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
102 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
103
104# elif PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
105
106# if PGM_GST_TYPE == PGM_TYPE_PAE
107 unsigned iPDSrc;
108 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc);
109
110# elif PGM_GST_TYPE == PGM_TYPE_AMD64
111 unsigned iPDSrc;
112 PX86PML4E pPml4eSrc;
113 X86PDPE PdpeSrc;
114 PGSTPD pPDSrc;
115
116 pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
117 Assert(pPml4eSrc);
118# endif
119 /* Quick check for a valid guest trap. */
120 if (!pPDSrc)
121 {
122 LogFlow(("Trap0eHandler: guest PDPTR not present CR3=%VGp\n", (CPUMGetGuestCR3(pVM) & X86_CR3_PAGE_MASK)));
123 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eGuestTrap; });
124 TRPMSetErrorCode(pVM, uErr);
125 return VINF_EM_RAW_GUEST_TRAP;
126 }
127# endif
128# else
129 PGSTPD pPDSrc = NULL;
130 const unsigned iPDSrc = 0;
131# endif
132
133# if PGM_SHW_TYPE == PGM_TYPE_32BIT
134 const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
135 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
136# elif PGM_SHW_TYPE == PGM_TYPE_PAE
137 const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
138 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
139
140# if PGM_GST_TYPE == PGM_TYPE_PAE
141 /* Did we mark the PDPT as not present in SyncCR3? */
142 unsigned iPDPTE = ((RTGCUINTPTR)pvFault >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
143 if (!pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
144 pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 1;
145
146# endif
147
148# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
149 const unsigned iPDDst = (((RTGCUINTPTR)pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
150 PX86PDPAE pPDDst;
151# if PGM_GST_TYPE == PGM_TYPE_PROT
152 /* AMD-V nested paging */
153 X86PML4E Pml4eSrc;
154 X86PDPE PdpeSrc;
155 PX86PML4E pPml4eSrc = &Pml4eSrc;
156
157 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
158 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_NX | X86_PML4E_A;
159 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_NX | X86_PDPE_A;
160# endif
161
162 rc = PGMShwSyncLongModePDPtr(pVM, (RTGCUINTPTR)pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
163 if (rc != VINF_SUCCESS)
164 {
165 AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
166 return rc;
167 }
168 Assert(pPDDst);
169# endif
170
171# if PGM_WITH_PAGING(PGM_GST_TYPE)
172 /*
173 * If we successfully correct the write protection fault due to dirty bit
174 * tracking, or this page fault is a genuine one, then return immediately.
175 */
176 STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
177 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
178 STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
179 if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
180 || rc == VINF_EM_RAW_GUEST_TRAP)
181 {
182 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
183 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
184 LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
185 return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
186 }
187
188 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
189# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
190
191 /*
192 * A common case is the not-present error caused by lazy page table syncing.
193 *
194 * It is IMPORTANT that we weed out any access to non-present shadow PDEs here
195 * so we can safely assume that the shadow PT is present when calling SyncPage later.
196 *
197 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
198 * of mapping conflict and defer to SyncCR3 in R3.
199 * (Again, we do NOT support access handlers for non-present guest pages.)
200 *
201 */
202# if PGM_WITH_PAGING(PGM_GST_TYPE)
203 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
204# else
205 GSTPDE PdeSrc;
206 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
207 PdeSrc.n.u1Present = 1;
208 PdeSrc.n.u1Write = 1;
209 PdeSrc.n.u1Accessed = 1;
210 PdeSrc.n.u1User = 1;
211# endif
212 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
213 && !pPDDst->a[iPDDst].n.u1Present
214 && PdeSrc.n.u1Present
215 )
216
217 {
218 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
219 STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
220 LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
221 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
222 if (VBOX_SUCCESS(rc))
223 {
224 STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
225 return rc;
226 }
227 Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
228 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
229 STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
230 return VINF_PGM_SYNC_CR3;
231 }
232
233# if PGM_WITH_PAGING(PGM_GST_TYPE)
234 /*
235 * Check if this address is within any of our mappings.
236 *
237 * This is *very* fast and it's gonna save us a bit of effort below and prevent
238 * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
239 * (BTW, it's impossible to have physical access handlers in a mapping.)
240 */
241 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
242 {
243 STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
244 PPGMMAPPING pMapping = CTXALLSUFF(pVM->pgm.s.pMappings);
245 for ( ; pMapping; pMapping = CTXALLSUFF(pMapping->pNext))
246 {
247 if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
248 break;
249 if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
250 {
251 /*
252 * The first thing we check is if we've got an undetected conflict.
253 */
254 if (!pVM->pgm.s.fMappingsFixed)
255 {
256 unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
257 while (iPT-- > 0)
258 if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
259 {
260 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
261 Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
262 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
263 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
264 return VINF_PGM_SYNC_CR3;
265 }
266 }
267
268 /*
269 * Check if the fault address is in a virtual page access handler range.
270 */
271 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->HyperVirtHandlers, pvFault);
272 if ( pCur
273 && (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
274 && uErr & X86_TRAP_PF_RW)
275 {
276# ifdef IN_GC
277 STAM_PROFILE_START(&pCur->Stat, h);
278 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
279 STAM_PROFILE_STOP(&pCur->Stat, h);
280# else
281 AssertFailed();
282 rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
283# endif
284 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
285 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
286 return rc;
287 }
288
289 /*
290 * Pretend we're not here and let the guest handle the trap.
291 */
292 TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
293 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
294 LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
295 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
296 return VINF_EM_RAW_GUEST_TRAP;
297 }
298 }
299 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
300 } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
301# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
302
303 /*
304 * Check if this fault address is flagged for special treatment,
305 * which means we'll have to figure out the physical address and
306 * check flags associated with it.
307 *
308 * ASSUME that we can limit any special access handling to pages
309 * in page tables which the guest believes to be present.
310 */
311 if (PdeSrc.n.u1Present)
312 {
313 RTGCPHYS GCPhys = NIL_RTGCPHYS;
314
315# if PGM_WITH_PAGING(PGM_GST_TYPE)
316# if PGM_GST_TYPE == PGM_TYPE_AMD64
317 bool fBigPagesSupported = true;
318# else
319 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
320# endif
321 if ( PdeSrc.b.u1Size
322 && fBigPagesSupported)
323 GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK)
324 | ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
325 else
326 {
327 PGSTPT pPTSrc;
328 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
329 if (VBOX_SUCCESS(rc))
330 {
331 unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
332 if (pPTSrc->a[iPTESrc].n.u1Present)
333 GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
334 }
335 }
336# else
337 /* No paging so the fault address is the physical address */
338 GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
339# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
340
341 /*
342 * If we have a GC address we'll check if it has any flags set.
343 */
344 if (GCPhys != NIL_RTGCPHYS)
345 {
346 STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
347
348 PPGMPAGE pPage;
349 rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
350 if (VBOX_SUCCESS(rc))
351 {
352 if (PGM_PAGE_HAS_ANY_HANDLERS(pPage))
353 {
354 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
355 {
356 /*
357 * Physical page access handler.
358 */
359 const RTGCPHYS GCPhysFault = GCPhys | ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
360 PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
361 if (pCur)
362 {
363# ifdef PGM_SYNC_N_PAGES
364 /*
365 * If the region is write protected and we got a page not present fault, then sync
366 * the pages. If the fault was caused by a read, then restart the instruction.
367 * In case of write access continue to the GC write handler.
368 *
369 * ASSUMES that there is only one handler per page or that they have similar write properties.
370 */
371 if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
372 && !(uErr & X86_TRAP_PF_P))
373 {
374 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
375 if ( VBOX_FAILURE(rc)
376 || !(uErr & X86_TRAP_PF_RW)
377 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
378 {
379 AssertRC(rc);
380 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
381 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
382 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
383 return rc;
384 }
385 }
386# endif
387
388 AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
389 || (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
390 ("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
391
392#if defined(IN_GC) || defined(IN_RING0)
393 if (CTXALLSUFF(pCur->pfnHandler))
394 {
395 STAM_PROFILE_START(&pCur->Stat, h);
396 rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
397 STAM_PROFILE_STOP(&pCur->Stat, h);
398 }
399 else
400#endif
401 rc = VINF_EM_RAW_EMULATE_INSTR;
402 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
403 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
404 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
405 return rc;
406 }
407 }
408# if PGM_WITH_PAGING(PGM_GST_TYPE)
409 else
410 {
411# ifdef PGM_SYNC_N_PAGES
412 /*
413 * If the region is write protected and we got a page not present fault, then sync
414 * the pages. If the fault was caused by a read, then restart the instruction.
415 * In case of write access continue to the GC write handler.
416 */
417 if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
418 && !(uErr & X86_TRAP_PF_P))
419 {
420 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
421 if ( VBOX_FAILURE(rc)
422 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
423 || !(uErr & X86_TRAP_PF_RW))
424 {
425 AssertRC(rc);
426 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
427 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
428 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
429 return rc;
430 }
431 }
432# endif
433 /*
434 * Ok, it's an virtual page access handler.
435 *
436 * Since it's faster to search by address, we'll do that first
437 * and then retry by GCPhys if that fails.
438 */
439 /** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
440 /** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
441 * page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
442 */
443 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
444 if (pCur)
445 {
446 AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
447 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
448 || !(uErr & X86_TRAP_PF_P)
449 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
450 ("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
451
452 if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
453 && ( uErr & X86_TRAP_PF_RW
454 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
455 {
456# ifdef IN_GC
457 STAM_PROFILE_START(&pCur->Stat, h);
458 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
459 STAM_PROFILE_STOP(&pCur->Stat, h);
460# else
461 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
462# endif
463 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
464 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
465 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
466 return rc;
467 }
468 /* Unhandled part of a monitored page */
469 }
470 else
471 {
472 /* Check by physical address. */
473 PPGMVIRTHANDLER pCur;
474 unsigned iPage;
475 rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
476 &pCur, &iPage);
477 Assert(VBOX_SUCCESS(rc) || !pCur);
478 if ( pCur
479 && ( uErr & X86_TRAP_PF_RW
480 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
481 {
482 Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
483# ifdef IN_GC
484 RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->GCPtr & PAGE_OFFSET_MASK);
485 Assert(off < pCur->cb);
486 STAM_PROFILE_START(&pCur->Stat, h);
487 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
488 STAM_PROFILE_STOP(&pCur->Stat, h);
489# else
490 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
491# endif
492 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
493 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
494 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
495 return rc;
496 }
497 }
498 }
499# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
500
501 /*
502 * There is a handled area of the page, but this fault doesn't belong to it.
503 * We must emulate the instruction.
504 *
505 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
506 * we first check if this was a page-not-present fault for a page with only
507 * write access handlers. Restart the instruction if it wasn't a write access.
508 */
509 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
510
511 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
512 && !(uErr & X86_TRAP_PF_P))
513 {
514 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
515 if ( VBOX_FAILURE(rc)
516 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
517 || !(uErr & X86_TRAP_PF_RW))
518 {
519 AssertRC(rc);
520 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
521 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
522 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
523 return rc;
524 }
525 }
526
527 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
528 * It's writing to an unhandled part of the LDT page several million times.
529 */
530 rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
531 LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%RHp%s%s\n",
532 rc, pPage->HCPhys,
533 PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ? " phys" : "",
534 PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ? " virt" : ""));
535 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
536 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
537 return rc;
538 } /* if any kind of handler */
539
540# if PGM_WITH_PAGING(PGM_GST_TYPE)
541 if (uErr & X86_TRAP_PF_P)
542 {
543 /*
544 * The page isn't marked, but it might still be monitored by a virtual page access handler.
545 * (ASSUMES no temporary disabling of virtual handlers.)
546 */
547 /** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
548 * we should correct both the shadow page table and physical memory flags, and not only check for
549 * accesses within the handler region but for access to pages with virtual handlers. */
550 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
551 if (pCur)
552 {
553 AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
554 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
555 || !(uErr & X86_TRAP_PF_P)
556 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
557 ("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
558
559 if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
560 && ( uErr & X86_TRAP_PF_RW
561 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
562 {
563# ifdef IN_GC
564 STAM_PROFILE_START(&pCur->Stat, h);
565 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
566 STAM_PROFILE_STOP(&pCur->Stat, h);
567# else
568 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
569# endif
570 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
571 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
572 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
573 return rc;
574 }
575 }
576 }
577# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
578 }
579 else
580 {
581 /* When the guest accesses invalid physical memory (e.g. probing of RAM or accessing a remapped MMIO range), then we'll fall
582 * back to the recompiler to emulate the instruction.
583 */
584 LogFlow(("pgmPhysGetPageEx %VGp failed with %Vrc\n", GCPhys, rc));
585 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersInvalid);
586 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
587 return VINF_EM_RAW_EMULATE_INSTR;
588 }
589
590 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
591
592# ifdef PGM_OUT_OF_SYNC_IN_GC
593 /*
594 * We are here only if page is present in Guest page tables and trap is not handled
595 * by our handlers.
596 * Check it for page out-of-sync situation.
597 */
598 STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
599
600 if (!(uErr & X86_TRAP_PF_P))
601 {
602 /*
603 * Page is not present in our page tables.
604 * Try to sync it!
605 * BTW, fPageShw is invalid in this branch!
606 */
607 if (uErr & X86_TRAP_PF_US)
608 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
609 else /* supervisor */
610 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
611
612# if defined(LOG_ENABLED) && !defined(IN_RING0)
613 RTGCPHYS GCPhys;
614 uint64_t fPageGst;
615 PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
616 Log(("Page out of sync: %VGv eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
617 pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
618# endif /* LOG_ENABLED */
619
620# if PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0)
621 if (CPUMGetGuestCPL(pVM, pRegFrame) == 0)
622 {
623 uint64_t fPageGst;
624 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
625 if ( VBOX_SUCCESS(rc)
626 && !(fPageGst & X86_PTE_US))
627 {
628 /* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
629 if ( pvFault == (RTGCPTR)pRegFrame->eip
630 || (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
631# ifdef CSAM_DETECT_NEW_CODE_PAGES
632 || ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
633 && CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
634# endif /* CSAM_DETECT_NEW_CODE_PAGES */
635 )
636 {
637 LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
638 rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
639 if (rc != VINF_SUCCESS)
640 {
641 /*
642 * CSAM needs to perform a job in ring 3.
643 *
644 * Sync the page before going to the host context; otherwise we'll end up in a loop if
645 * CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
646 */
647 LogFlow(("CSAM ring 3 job\n"));
648 int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
649 AssertRC(rc2);
650
651 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
652 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
653 return rc;
654 }
655 }
656# ifdef CSAM_DETECT_NEW_CODE_PAGES
657 else
658 if ( uErr == X86_TRAP_PF_RW
659 && pRegFrame->ecx >= 0x100 /* early check for movswd count */
660 && pRegFrame->ecx < 0x10000
661 )
662 {
663 /* In case of a write to a non-present supervisor shadow page, we'll take special precautions
664 * to detect loading of new code pages.
665 */
666
667 /*
668 * Decode the instruction.
669 */
670 RTGCPTR PC;
671 rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
672 if (rc == VINF_SUCCESS)
673 {
674 DISCPUSTATE Cpu;
675 uint32_t cbOp;
676 rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
677
678 /* For now we'll restrict this to rep movsw/d instructions */
679 if ( rc == VINF_SUCCESS
680 && Cpu.pCurInstr->opcode == OP_MOVSWD
681 && (Cpu.prefix & PREFIX_REP))
682 {
683 CSAMMarkPossibleCodePage(pVM, pvFault);
684 }
685 }
686 }
687# endif /* CSAM_DETECT_NEW_CODE_PAGES */
688
689 /*
690 * Mark this page as safe.
691 */
692 /** @todo not correct for pages that contain both code and data!! */
693 Log2(("CSAMMarkPage %VGv; scanned=%d\n", pvFault, true));
694 CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
695 }
696 }
697# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0) */
698 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
699 if (VBOX_SUCCESS(rc))
700 {
701 /* The page was successfully synced, return to the guest. */
702 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
703 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
704 return VINF_SUCCESS;
705 }
706 }
707 else
708 {
709 /*
710 * A side effect of not flushing global PDEs are out of sync pages due
711 * to physical monitored regions, that are no longer valid.
712 * Assume for now it only applies to the read/write flag
713 */
714 if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
715 {
716 if (uErr & X86_TRAP_PF_US)
717 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
718 else /* supervisor */
719 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
720
721
722 /*
723 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
724 */
725 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
726 if (VBOX_SUCCESS(rc))
727 {
728 /*
729 * Page was successfully synced, return to guest.
730 */
731# ifdef VBOX_STRICT
732 RTGCPHYS GCPhys;
733 uint64_t fPageGst;
734 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
735 Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
736 LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
737
738 uint64_t fPageShw;
739 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
740 AssertMsg(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW, ("rc=%Vrc fPageShw=%VX64\n", rc, fPageShw));
741# endif /* VBOX_STRICT */
742 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
743 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
744 return VINF_SUCCESS;
745 }
746
747 /* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
748 if ( CPUMGetGuestCPL(pVM, pRegFrame) == 0
749 && ((CPUMGetGuestCR0(pVM) & (X86_CR0_WP|X86_CR0_PG)) == X86_CR0_PG)
750 && (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P))
751 {
752 uint64_t fPageGst;
753 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
754 if ( VBOX_SUCCESS(rc)
755 && !(fPageGst & X86_PTE_RW))
756 {
757 rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
758 if (VBOX_SUCCESS(rc))
759 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulGC);
760 else
761 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulR3);
762 return rc;
763 }
764 else
765 AssertMsgFailed(("Unexpected r/w page %x flag=%x\n", pvFault, (uint32_t)fPageGst));
766 }
767
768 }
769
770# if PGM_WITH_PAGING(PGM_GST_TYPE)
771# ifdef VBOX_STRICT
772 /*
773 * Check for VMM page flags vs. Guest page flags consistency.
774 * Currently only for debug purposes.
775 */
776 if (VBOX_SUCCESS(rc))
777 {
778 /* Get guest page flags. */
779 uint64_t fPageGst;
780 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
781 if (VBOX_SUCCESS(rc))
782 {
783 uint64_t fPageShw;
784 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
785
786 /*
787 * Compare page flags.
788 * Note: we have AVL, A, D bits desynched.
789 */
790 AssertMsg((fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)),
791 ("Page flags mismatch! pvFault=%VGv GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
792 }
793 else
794 AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
795 }
796 else
797 AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
798# endif /* VBOX_STRICT */
799# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
800 }
801 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
802# endif /* PGM_OUT_OF_SYNC_IN_GC */
803 }
804 else
805 {
806 /*
807 * Page not present in Guest OS or invalid page table address.
808 * This is potential virtual page access handler food.
809 *
810 * For the present we'll say that our access handlers don't
811 * work for this case - we've already discarded the page table
812 * not present case which is identical to this.
813 *
814 * When we perchance find we need this, we will probably have AVL
815 * trees (offset based) to operate on and we can measure their speed
816 * agains mapping a page table and probably rearrange this handling
817 * a bit. (Like, searching virtual ranges before checking the
818 * physical address.)
819 */
820 }
821 }
822
823
824# if PGM_WITH_PAGING(PGM_GST_TYPE)
825 /*
826 * Conclusion, this is a guest trap.
827 */
828 LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
829 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
830 return VINF_EM_RAW_GUEST_TRAP;
831# else
832 /* present, but not a monitored page; perhaps the guest is probing physical memory */
833 return VINF_EM_RAW_EMULATE_INSTR;
834# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
835
836
837#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
838
839 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
840 return VERR_INTERNAL_ERROR;
841#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
842}
843
844
845/**
846 * Emulation of the invlpg instruction.
847 *
848 *
849 * @returns VBox status code.
850 *
851 * @param pVM VM handle.
852 * @param GCPtrPage Page to invalidate.
853 *
854 * @remark ASSUMES that the guest is updating before invalidating. This order
855 * isn't required by the CPU, so this is speculative and could cause
856 * trouble.
857 *
858 * @todo Flush page or page directory only if necessary!
859 * @todo Add a #define for simply invalidating the page.
860 */
861PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
862{
863#if PGM_WITH_PAGING(PGM_GST_TYPE) \
864 && PGM_SHW_TYPE != PGM_TYPE_NESTED
865 int rc;
866
867 LogFlow(("InvalidatePage %VGv\n", GCPtrPage));
868 /*
869 * Get the shadow PD entry and skip out if this PD isn't present.
870 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
871 */
872# if PGM_SHW_TYPE == PGM_TYPE_32BIT
873 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
874 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
875# elif PGM_SHW_TYPE == PGM_TYPE_PAE
876 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
877 const unsigned iPdPte = (GCPtrPage >> X86_PDPT_SHIFT); /* no mask; flat index into the 2048 entry array. */
878 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
879 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT);
880# else /* AMD64 */
881 /* PML4 */
882 const unsigned iPml4e = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
883 const unsigned iPdPte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
884 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
885 PX86PDPAE pPDDst;
886 PX86PDPT pPdptDst;
887 PX86PML4E pPml4eDst = &pVM->pgm.s.pHCPaePML4->a[iPml4e];
888 rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
889 if (rc != VINF_SUCCESS)
890 {
891 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Vrc\n", rc));
892 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
893 return VINF_SUCCESS;
894 }
895 Assert(pPDDst);
896
897 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
898 PX86PDPE pPdpeDst = &pPdptDst->a[iPdPte];
899# endif
900
901 const SHWPDE PdeDst = *pPdeDst;
902 if (!PdeDst.n.u1Present)
903 {
904 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
905 return VINF_SUCCESS;
906 }
907
908 /*
909 * Get the guest PD entry and calc big page.
910 */
911# if PGM_GST_TYPE == PGM_TYPE_32BIT
912 PX86PD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
913 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
914 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
915# else
916 unsigned iPDSrc;
917# if PGM_GST_TYPE == PGM_TYPE_PAE
918 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
919# else /* AMD64 */
920 PX86PML4E pPml4eSrc;
921 X86PDPE PdpeSrc;
922 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
923# endif
924 GSTPDE PdeSrc;
925
926 if (pPDSrc)
927 PdeSrc = pPDSrc->a[iPDSrc];
928 else
929 PdeSrc.u = 0;
930# endif
931
932 const uint32_t cr4 = CPUMGetGuestCR4(pVM);
933# if PGM_GST_TYPE == PGM_TYPE_AMD64
934 const bool fIsBigPage = PdeSrc.b.u1Size;
935# else
936 const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
937# endif
938
939# ifdef IN_RING3
940 /*
941 * If a CR3 Sync is pending we may ignore the invalidate page operation
942 * depending on the kind of sync and if it's a global page or not.
943 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
944 */
945# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
946 if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
947 || ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
948 && fIsBigPage
949 && PdeSrc.b.u1Global
950 && (cr4 & X86_CR4_PGE)
951 )
952 )
953# else
954 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
955# endif
956 {
957 STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
958 return VINF_SUCCESS;
959 }
960# endif /* IN_RING3 */
961
962# if PGM_GST_TYPE == PGM_TYPE_AMD64
963 /* Fetch the pgm pool shadow descriptor. */
964 PPGMPOOLPAGE pShwPdpt = pgmPoolGetPageByHCPhys(pVM, pPml4eDst->u & X86_PML4E_PG_MASK);
965 Assert(pShwPdpt);
966# endif
967
968# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
969 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
970
971 /* Fetch the pgm pool shadow descriptor. */
972 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdPte].u & SHW_PDPE_PG_MASK);
973 Assert(pShwPde);
974# endif
975
976# if PGM_GST_TYPE == PGM_TYPE_AMD64
977 Assert(pPml4eDst->n.u1Present && pPml4eDst->u & SHW_PDPT_MASK);
978 if (pPml4eSrc->n.u1Present)
979 {
980 if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
981 || (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
982 {
983 /*
984 * Mark not present so we can resync the PML4E when it's used.
985 */
986 LogFlow(("InvalidatePage: Out-of-sync PML4E at %VGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
987 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
988 pgmPoolFreeByPage(pPool, pShwPdpt, PGMPOOL_IDX_PML4, iPml4e);
989 pPml4eDst->u = 0;
990 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
991 PGM_INVL_GUEST_TLBS();
992 }
993 else if (!pPml4eSrc->n.u1Accessed)
994 {
995 /*
996 * Mark not present so we can set the accessed bit.
997 */
998 LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %VGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
999 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1000 pgmPoolFreeByPage(pPool, pShwPdpt, PGMPOOL_IDX_PML4, iPml4e);
1001 pPml4eDst->u = 0;
1002 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
1003 PGM_INVL_GUEST_TLBS();
1004 }
1005 }
1006 else
1007 {
1008 LogFlow(("InvalidatePage: Out-of-sync PML4E (P) at %VGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1009 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1010 pgmPoolFreeByPage(pPool, pShwPdpt, PGMPOOL_IDX_PML4, iPml4e);
1011 pPml4eDst->u = 0;
1012 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1013 PGM_INVL_PG(GCPtrPage);
1014 return VINF_SUCCESS;
1015 }
1016 RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1017
1018 /* Check if the PML4 entry has changed. */
1019 if (pShwPdpt->GCPhys != GCPhysPdpt)
1020 {
1021 LogFlow(("InvalidatePage: Out-of-sync PML4E (GCPhys) at %VGv %VGp vs %VGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1022 GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1023 pgmPoolFreeByPage(pPool, pShwPdpt, PGMPOOL_IDX_PML4, iPml4e);
1024 pPml4eDst->u = 0;
1025 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1026 PGM_INVL_PG(GCPtrPage);
1027 return VINF_SUCCESS;
1028 }
1029
1030
1031 Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1032 if (PdpeSrc.n.u1Present)
1033 {
1034 if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1035 || (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1036 {
1037 /*
1038 * Mark not present so we can resync the PDPTE when it's used.
1039 */
1040 LogFlow(("InvalidatePage: Out-of-sync PDPE at %VGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1041 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1042 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdPte);
1043 pPdpeDst->u = 0;
1044 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1045 PGM_INVL_GUEST_TLBS();
1046 }
1047 else if (!PdpeSrc.lm.u1Accessed)
1048 {
1049 /*
1050 * Mark not present so we can set the accessed bit.
1051 */
1052 LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %VGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1053 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1054 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdPte);
1055 pPdpeDst->u = 0;
1056 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
1057 PGM_INVL_GUEST_TLBS();
1058 }
1059 }
1060 else
1061 {
1062 LogFlow(("InvalidatePage: Out-of-sync PDPE (P) at %VGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1063 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1064 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdPte);
1065 pPdpeDst->u = 0;
1066 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1067 PGM_INVL_PG(GCPtrPage);
1068 return VINF_SUCCESS;
1069 }
1070 RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1071
1072 /* Check if the PDPT entry has changed. */
1073 if (pShwPde->GCPhys != GCPhysPd)
1074 {
1075 LogFlow(("InvalidatePage: Out-of-sync PDPE (GCPhys) at %VGv %VGp vs %VGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1076 GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1077 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdPte);
1078 pPdpeDst->u = 0;
1079 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1080 PGM_INVL_PG(GCPtrPage);
1081 return VINF_SUCCESS;
1082 }
1083# endif
1084
1085 /*
1086 * Deal with the Guest PDE.
1087 */
1088 rc = VINF_SUCCESS;
1089 if (PdeSrc.n.u1Present)
1090 {
1091 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1092 {
1093 /*
1094 * Conflict - Let SyncPT deal with it to avoid duplicate code.
1095 */
1096 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1097 Assert(PGMGetGuestMode(pVM) <= PGMMODE_PAE);
1098 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
1099 }
1100 else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1101 || (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1102 {
1103 /*
1104 * Mark not present so we can resync the PDE when it's used.
1105 */
1106 LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1107 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1108# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
1109 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1110# else
1111 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1112# endif
1113 pPdeDst->u = 0;
1114 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1115 PGM_INVL_GUEST_TLBS();
1116 }
1117 else if (!PdeSrc.n.u1Accessed)
1118 {
1119 /*
1120 * Mark not present so we can set the accessed bit.
1121 */
1122 LogFlow(("InvalidatePage: Out-of-sync (A) at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1123 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1124# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
1125 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1126# else
1127 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1128# endif
1129 pPdeDst->u = 0;
1130 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
1131 PGM_INVL_GUEST_TLBS();
1132 }
1133 else if (!fIsBigPage)
1134 {
1135 /*
1136 * 4KB - page.
1137 */
1138 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1139 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1140# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1141 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1142 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1143# endif
1144 if (pShwPage->GCPhys == GCPhys)
1145 {
1146# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1147 const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1148 PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1149 if (pPT->a[iPTEDst].n.u1Present)
1150 {
1151# ifdef PGMPOOL_WITH_USER_TRACKING
1152 /* This is very unlikely with caching/monitoring enabled. */
1153 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1154# endif
1155 pPT->a[iPTEDst].u = 0;
1156 }
1157# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1158 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
1159 if (VBOX_SUCCESS(rc))
1160 rc = VINF_SUCCESS;
1161# endif
1162 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
1163 PGM_INVL_PG(GCPtrPage);
1164 }
1165 else
1166 {
1167 /*
1168 * The page table address changed.
1169 */
1170 LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
1171 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1172# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
1173 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1174# else
1175 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1176# endif
1177 pPdeDst->u = 0;
1178 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1179 PGM_INVL_GUEST_TLBS();
1180 }
1181 }
1182 else
1183 {
1184 /*
1185 * 2/4MB - page.
1186 */
1187 /* Before freeing the page, check if anything really changed. */
1188 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1189 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1190# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1191 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1192 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1193# endif
1194 if ( pShwPage->GCPhys == GCPhys
1195 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1196 {
1197 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1198 /** @todo PAT */
1199 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1200 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1201 && ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1202 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1203 {
1204 LogFlow(("Skipping flush for big page containing %VGv (PD=%X .u=%VX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1205 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
1206 return VINF_SUCCESS;
1207 }
1208 }
1209
1210 /*
1211 * Ok, the page table is present and it's been changed in the guest.
1212 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1213 * We could do this for some flushes in GC too, but we need an algorithm for
1214 * deciding which 4MB pages containing code likely to be executed very soon.
1215 */
1216 LogFlow(("InvalidatePage: Out-of-sync PD at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1217 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1218# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
1219 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1220# else
1221 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1222# endif
1223 pPdeDst->u = 0;
1224 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
1225 PGM_INVL_BIG_PG(GCPtrPage);
1226 }
1227 }
1228 else
1229 {
1230 /*
1231 * Page directory is not present, mark shadow PDE not present.
1232 */
1233 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1234 {
1235# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
1236 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1237# else
1238 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1239# endif
1240 pPdeDst->u = 0;
1241 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1242 PGM_INVL_PG(GCPtrPage);
1243 }
1244 else
1245 {
1246 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1247 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
1248 }
1249 }
1250
1251 return rc;
1252
1253#else /* guest real and protected mode */
1254 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1255 return VINF_SUCCESS;
1256#endif
1257}
1258
1259
1260#ifdef PGMPOOL_WITH_USER_TRACKING
1261/**
1262 * Update the tracking of shadowed pages.
1263 *
1264 * @param pVM The VM handle.
1265 * @param pShwPage The shadow page.
1266 * @param HCPhys The physical page we is being dereferenced.
1267 */
1268DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1269{
1270# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1271 STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1272 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1273
1274 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1275 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1276 * 2. write protect all shadowed pages. I.e. implement caching.
1277 */
1278 /*
1279 * Find the guest address.
1280 */
1281 for (PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
1282 pRam;
1283 pRam = CTXALLSUFF(pRam->pNext))
1284 {
1285 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1286 while (iPage-- > 0)
1287 {
1288 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1289 {
1290 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1291 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1292 pShwPage->cPresent--;
1293 pPool->cPresent--;
1294 STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1295 return;
1296 }
1297 }
1298 }
1299
1300 for (;;)
1301 AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1302# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1303 pShwPage->cPresent--;
1304 pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1305# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1306}
1307
1308
1309/**
1310 * Update the tracking of shadowed pages.
1311 *
1312 * @param pVM The VM handle.
1313 * @param pShwPage The shadow page.
1314 * @param u16 The top 16-bit of the pPage->HCPhys.
1315 * @param pPage Pointer to the guest page. this will be modified.
1316 * @param iPTDst The index into the shadow table.
1317 */
1318DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1319{
1320# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1321 /*
1322 * We're making certain assumptions about the placement of cRef and idx.
1323 */
1324 Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1325 Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1326
1327 /*
1328 * Just deal with the simple first time here.
1329 */
1330 if (!u16)
1331 {
1332 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1333 u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) | pShwPage->idx;
1334 }
1335 else
1336 u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1337
1338 /* write back, trying to be clever... */
1339 Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%VHp->%VHp iPTDst=%#x\n",
1340 u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) | ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1341 *((uint16_t *)&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1342# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1343
1344 /* update statistics. */
1345 pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1346 pShwPage->cPresent++;
1347 if (pShwPage->iFirstPresent > iPTDst)
1348 pShwPage->iFirstPresent = iPTDst;
1349}
1350#endif /* PGMPOOL_WITH_USER_TRACKING */
1351
1352
1353/**
1354 * Creates a 4K shadow page for a guest page.
1355 *
1356 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1357 * physical address. The PdeSrc argument only the flags are used. No page structured
1358 * will be mapped in this function.
1359 *
1360 * @param pVM VM handle.
1361 * @param pPteDst Destination page table entry.
1362 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1363 * Can safely assume that only the flags are being used.
1364 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1365 * @param pShwPage Pointer to the shadow page.
1366 * @param iPTDst The index into the shadow table.
1367 *
1368 * @remark Not used for 2/4MB pages!
1369 */
1370DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1371{
1372 if (PteSrc.n.u1Present)
1373 {
1374 /*
1375 * Find the ram range.
1376 */
1377 PPGMPAGE pPage;
1378 int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1379 if (VBOX_SUCCESS(rc))
1380 {
1381 /** @todo investiage PWT, PCD and PAT. */
1382 /*
1383 * Make page table entry.
1384 */
1385 const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1386 SHWPTE PteDst;
1387 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1388 {
1389 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1390 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1391 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1392 | (HCPhys & X86_PTE_PAE_PG_MASK);
1393 else
1394 {
1395 LogFlow(("SyncPageWorker: monitored page (%VGp) -> mark not present\n", HCPhys));
1396 PteDst.u = 0;
1397 }
1398 /** @todo count these two kinds. */
1399 }
1400 else
1401 {
1402 /*
1403 * If the page or page directory entry is not marked accessed,
1404 * we mark the page not present.
1405 */
1406 if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
1407 {
1408 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1409 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1410 PteDst.u = 0;
1411 }
1412 else
1413 /*
1414 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1415 * when the page is modified.
1416 */
1417 if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1418 {
1419 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1420 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1421 | (HCPhys & X86_PTE_PAE_PG_MASK)
1422 | PGM_PTFLAGS_TRACK_DIRTY;
1423 }
1424 else
1425 {
1426 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1427 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1428 | (HCPhys & X86_PTE_PAE_PG_MASK);
1429 }
1430 }
1431
1432#ifdef PGMPOOL_WITH_USER_TRACKING
1433 /*
1434 * Keep user track up to date.
1435 */
1436 if (PteDst.n.u1Present)
1437 {
1438 if (!pPteDst->n.u1Present)
1439 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1440 else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1441 {
1442 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1443 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1444 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1445 }
1446 }
1447 else if (pPteDst->n.u1Present)
1448 {
1449 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1450 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1451 }
1452#endif /* PGMPOOL_WITH_USER_TRACKING */
1453
1454 /*
1455 * Update statistics and commit the entry.
1456 */
1457 if (!PteSrc.n.u1Global)
1458 pShwPage->fSeenNonGlobal = true;
1459 *pPteDst = PteDst;
1460 }
1461 /* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1462 /** @todo count these. */
1463 }
1464 else
1465 {
1466 /*
1467 * Page not-present.
1468 */
1469 LogFlow(("SyncPageWorker: page not present in Pte\n"));
1470#ifdef PGMPOOL_WITH_USER_TRACKING
1471 /* Keep user track up to date. */
1472 if (pPteDst->n.u1Present)
1473 {
1474 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1475 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1476 }
1477#endif /* PGMPOOL_WITH_USER_TRACKING */
1478 pPteDst->u = 0;
1479 /** @todo count these. */
1480 }
1481}
1482
1483
1484/**
1485 * Syncs a guest OS page.
1486 *
1487 * There are no conflicts at this point, neither is there any need for
1488 * page table allocations.
1489 *
1490 * @returns VBox status code.
1491 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1492 * @param pVM VM handle.
1493 * @param PdeSrc Page directory entry of the guest.
1494 * @param GCPtrPage Guest context page address.
1495 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1496 * @param uErr Fault error (X86_TRAP_PF_*).
1497 */
1498PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1499{
1500 LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1501
1502#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1503 || PGM_GST_TYPE == PGM_TYPE_PAE \
1504 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
1505 && PGM_SHW_TYPE != PGM_TYPE_NESTED
1506
1507# if PGM_WITH_NX(PGM_GST_TYPE)
1508 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1509# endif
1510
1511 /*
1512 * Assert preconditions.
1513 */
1514 STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1515 Assert(PdeSrc.n.u1Present);
1516 Assert(cPages);
1517
1518 /*
1519 * Get the shadow PDE, find the shadow page table in the pool.
1520 */
1521# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1522 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1523 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1524# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1525 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1526 const unsigned iPdPte = (GCPtrPage >> X86_PDPT_SHIFT); /* no mask; flat index into the 2048 entry array. */
1527 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT);
1528 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1529# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1530 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1531 const unsigned iPdPte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1532 PX86PDPAE pPDDst;
1533 X86PDEPAE PdeDst;
1534 PX86PDPT pPdptDst;
1535
1536 int rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
1537 AssertRCReturn(rc, rc);
1538 Assert(pPDDst && pPdptDst);
1539 PdeDst = pPDDst->a[iPDDst];
1540# endif
1541 Assert(PdeDst.n.u1Present);
1542 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1543
1544# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
1545 /* Fetch the pgm pool shadow descriptor. */
1546 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdPte].u & X86_PDPE_PG_MASK);
1547 Assert(pShwPde);
1548# endif
1549
1550 /*
1551 * Check that the page is present and that the shadow PDE isn't out of sync.
1552 */
1553# if PGM_GST_TYPE == PGM_TYPE_AMD64
1554 const bool fBigPage = PdeSrc.b.u1Size;
1555# else
1556 const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1557# endif
1558 RTGCPHYS GCPhys;
1559 if (!fBigPage)
1560 {
1561 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1562# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1563 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1564 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1565# endif
1566 }
1567 else
1568 {
1569 GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1570# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1571 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1572 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1573# endif
1574 }
1575 if ( pShwPage->GCPhys == GCPhys
1576 && PdeSrc.n.u1Present
1577 && (PdeSrc.n.u1User == PdeDst.n.u1User)
1578 && (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
1579# if PGM_WITH_NX(PGM_GST_TYPE)
1580 && (!fNoExecuteBitValid || PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1581# endif
1582 )
1583 {
1584 /*
1585 * Check that the PDE is marked accessed already.
1586 * Since we set the accessed bit *before* getting here on a #PF, this
1587 * check is only meant for dealing with non-#PF'ing paths.
1588 */
1589 if (PdeSrc.n.u1Accessed)
1590 {
1591 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1592 if (!fBigPage)
1593 {
1594 /*
1595 * 4KB Page - Map the guest page table.
1596 */
1597 PGSTPT pPTSrc;
1598 int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1599 if (VBOX_SUCCESS(rc))
1600 {
1601# ifdef PGM_SYNC_N_PAGES
1602 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1603 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1604 {
1605 /*
1606 * This code path is currently only taken when the caller is PGMTrap0eHandler
1607 * for non-present pages!
1608 *
1609 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1610 * deal with locality.
1611 */
1612 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1613# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1614 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1615 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1616# else
1617 const unsigned offPTSrc = 0;
1618# endif
1619 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1620 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1621 iPTDst = 0;
1622 else
1623 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1624 for (; iPTDst < iPTDstEnd; iPTDst++)
1625 {
1626 if (!pPTDst->a[iPTDst].n.u1Present)
1627 {
1628 GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1629 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1630 NOREF(GCPtrCurPage);
1631#ifndef IN_RING0
1632 /*
1633 * Assuming kernel code will be marked as supervisor - and not as user level
1634 * and executed using a conforming code selector - And marked as readonly.
1635 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
1636 */
1637 PPGMPAGE pPage;
1638 if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW | X86_PTE_US))
1639 || iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1640 || !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1641 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1642 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1643 )
1644#endif /* else: CSAM not active */
1645 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1646 Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1647 GCPtrCurPage, PteSrc.n.u1Present,
1648 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1649 PteSrc.n.u1User & PdeSrc.n.u1User,
1650 (uint64_t)PteSrc.u,
1651 (uint64_t)pPTDst->a[iPTDst].u,
1652 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1653 }
1654 }
1655 }
1656 else
1657# endif /* PGM_SYNC_N_PAGES */
1658 {
1659 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1660 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1661 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1662 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1663 Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1664 GCPtrPage, PteSrc.n.u1Present,
1665 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1666 PteSrc.n.u1User & PdeSrc.n.u1User,
1667 (uint64_t)PteSrc.u,
1668 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1669 }
1670 }
1671 else /* MMIO or invalid page: emulated in #PF handler. */
1672 {
1673 LogFlow(("PGM_GCPHYS_2_PTR %VGp failed with %Vrc\n", GCPhys, rc));
1674 Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1675 }
1676 }
1677 else
1678 {
1679 /*
1680 * 4/2MB page - lazy syncing shadow 4K pages.
1681 * (There are many causes of getting here, it's no longer only CSAM.)
1682 */
1683 /* Calculate the GC physical address of this 4KB shadow page. */
1684 RTGCPHYS GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK) | ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1685 /* Find ram range. */
1686 PPGMPAGE pPage;
1687 int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1688 if (VBOX_SUCCESS(rc))
1689 {
1690 /*
1691 * Make shadow PTE entry.
1692 */
1693 const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1694 SHWPTE PteDst;
1695 PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1696 | (HCPhys & X86_PTE_PAE_PG_MASK);
1697 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1698 {
1699 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1700 PteDst.n.u1Write = 0;
1701 else
1702 PteDst.u = 0;
1703 }
1704 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1705# ifdef PGMPOOL_WITH_USER_TRACKING
1706 if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1707 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1708# endif
1709 pPTDst->a[iPTDst] = PteDst;
1710
1711
1712 /*
1713 * If the page is not flagged as dirty and is writable, then make it read-only
1714 * at PD level, so we can set the dirty bit when the page is modified.
1715 *
1716 * ASSUMES that page access handlers are implemented on page table entry level.
1717 * Thus we will first catch the dirty access and set PDE.D and restart. If
1718 * there is an access handler, we'll trap again and let it work on the problem.
1719 */
1720 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1721 * As for invlpg, it simply frees the whole shadow PT.
1722 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1723 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1724 {
1725 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1726 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
1727 PdeDst.n.u1Write = 0;
1728 }
1729 else
1730 {
1731 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1732 PdeDst.n.u1Write = PdeSrc.n.u1Write;
1733 }
1734# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1735 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1736# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1737 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1738# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1739 pPDDst->a[iPDDst] = PdeDst;
1740# endif
1741 Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1742 GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1743 PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1744 }
1745 else
1746 LogFlow(("PGM_GCPHYS_2_PTR %VGp (big) failed with %Vrc\n", GCPhys, rc));
1747 }
1748 return VINF_SUCCESS;
1749 }
1750 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1751 }
1752 else
1753 {
1754 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1755 Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1756 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1757 }
1758
1759 /*
1760 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1761 * Yea, I'm lazy.
1762 */
1763 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1764# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
1765 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1766# else
1767 pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPDDst);
1768# endif
1769
1770# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1771 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1772# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1773 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1774# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1775 pPDDst->a[iPDDst].u = 0;
1776# endif
1777 PGM_INVL_GUEST_TLBS();
1778 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1779
1780#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
1781 && PGM_SHW_TYPE != PGM_TYPE_NESTED
1782
1783# ifdef PGM_SYNC_N_PAGES
1784 /*
1785 * Get the shadow PDE, find the shadow page table in the pool.
1786 */
1787 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1788# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1789 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1790# else /* PAE */
1791 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1792# endif
1793 Assert(PdeDst.n.u1Present);
1794 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1795 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1796
1797# if PGM_SHW_TYPE == PGM_TYPE_PAE
1798 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1799 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1800# else
1801 const unsigned offPTSrc = 0;
1802# endif
1803
1804 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1805 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1806 {
1807 /*
1808 * This code path is currently only taken when the caller is PGMTrap0eHandler
1809 * for non-present pages!
1810 *
1811 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1812 * deal with locality.
1813 */
1814 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1815 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1816 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1817 iPTDst = 0;
1818 else
1819 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1820 for (; iPTDst < iPTDstEnd; iPTDst++)
1821 {
1822 if (!pPTDst->a[iPTDst].n.u1Present)
1823 {
1824 GSTPTE PteSrc;
1825
1826 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1827
1828 /* Fake the page table entry */
1829 PteSrc.u = GCPtrCurPage;
1830 PteSrc.n.u1Present = 1;
1831 PteSrc.n.u1Dirty = 1;
1832 PteSrc.n.u1Accessed = 1;
1833 PteSrc.n.u1Write = 1;
1834 PteSrc.n.u1User = 1;
1835
1836 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1837
1838 Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1839 GCPtrCurPage, PteSrc.n.u1Present,
1840 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1841 PteSrc.n.u1User & PdeSrc.n.u1User,
1842 (uint64_t)PteSrc.u,
1843 (uint64_t)pPTDst->a[iPTDst].u,
1844 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1845 }
1846 }
1847 }
1848 else
1849# endif /* PGM_SYNC_N_PAGES */
1850 {
1851 GSTPTE PteSrc;
1852 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1853 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1854
1855 /* Fake the page table entry */
1856 PteSrc.u = GCPtrCurPage;
1857 PteSrc.n.u1Present = 1;
1858 PteSrc.n.u1Dirty = 1;
1859 PteSrc.n.u1Accessed = 1;
1860 PteSrc.n.u1Write = 1;
1861 PteSrc.n.u1User = 1;
1862 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1863
1864 Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
1865 GCPtrPage, PteSrc.n.u1Present,
1866 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1867 PteSrc.n.u1User & PdeSrc.n.u1User,
1868 (uint64_t)PteSrc.u,
1869 (uint64_t)pPTDst->a[iPTDst].u,
1870 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1871 }
1872 return VINF_SUCCESS;
1873
1874#else
1875 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1876 return VERR_INTERNAL_ERROR;
1877#endif
1878}
1879
1880
1881
1882#if PGM_WITH_PAGING(PGM_GST_TYPE)
1883
1884/**
1885 * Investigate page fault and handle write protection page faults caused by
1886 * dirty bit tracking.
1887 *
1888 * @returns VBox status code.
1889 * @param pVM VM handle.
1890 * @param uErr Page fault error code.
1891 * @param pPdeDst Shadow page directory entry.
1892 * @param pPdeSrc Guest page directory entry.
1893 * @param GCPtrPage Guest context page address.
1894 */
1895PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1896{
1897 bool fWriteProtect = !!(CPUMGetGuestCR0(pVM) & X86_CR0_WP);
1898 bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
1899 bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
1900# if PGM_GST_TYPE == PGM_TYPE_AMD64
1901 bool fBigPagesSupported = true;
1902# else
1903 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1904# endif
1905# if PGM_WITH_NX(PGM_GST_TYPE)
1906 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1907# endif
1908 unsigned uPageFaultLevel;
1909 int rc;
1910
1911 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1912 LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1913
1914# if PGM_GST_TYPE == PGM_TYPE_PAE \
1915 || PGM_GST_TYPE == PGM_TYPE_AMD64
1916
1917# if PGM_GST_TYPE == PGM_TYPE_AMD64
1918 PX86PML4E pPml4eSrc;
1919 PX86PDPE pPdpeSrc;
1920
1921 pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc);
1922 Assert(pPml4eSrc);
1923
1924 /*
1925 * Real page fault? (PML4E level)
1926 */
1927 if ( (uErr & X86_TRAP_PF_RSVD)
1928 || !pPml4eSrc->n.u1Present
1929 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
1930 || (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
1931 || (fUserLevelFault && !pPml4eSrc->n.u1User)
1932 )
1933 {
1934 uPageFaultLevel = 0;
1935 goto UpperLevelPageFault;
1936 }
1937 Assert(pPdpeSrc);
1938
1939# else /* PAE */
1940 PX86PDPE pPdpeSrc = &pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtrPage >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
1941# endif
1942
1943 /*
1944 * Real page fault? (PDPE level)
1945 */
1946 if ( (uErr & X86_TRAP_PF_RSVD)
1947 || !pPdpeSrc->n.u1Present
1948# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
1949 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
1950 || (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault || fWriteProtect))
1951 || (fUserLevelFault && !pPdpeSrc->lm.u1User)
1952# endif
1953 )
1954 {
1955 uPageFaultLevel = 1;
1956 goto UpperLevelPageFault;
1957 }
1958# endif
1959
1960 /*
1961 * Real page fault? (PDE level)
1962 */
1963 if ( (uErr & X86_TRAP_PF_RSVD)
1964 || !pPdeSrc->n.u1Present
1965# if PGM_WITH_NX(PGM_GST_TYPE)
1966 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
1967# endif
1968 || (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
1969 || (fUserLevelFault && !pPdeSrc->n.u1User) )
1970 {
1971 uPageFaultLevel = 2;
1972 goto UpperLevelPageFault;
1973 }
1974
1975 /*
1976 * First check the easy case where the page directory has been marked read-only to track
1977 * the dirty bit of an emulated BIG page
1978 */
1979 if (pPdeSrc->b.u1Size && fBigPagesSupported)
1980 {
1981 /* Mark guest page directory as accessed */
1982# if PGM_GST_TYPE == PGM_TYPE_AMD64
1983 pPml4eSrc->n.u1Accessed = 1;
1984 pPdpeSrc->lm.u1Accessed = 1;
1985# endif
1986 pPdeSrc->b.u1Accessed = 1;
1987
1988 /*
1989 * Only write protection page faults are relevant here.
1990 */
1991 if (fWriteFault)
1992 {
1993 /* Mark guest page directory as dirty (BIG page only). */
1994 pPdeSrc->b.u1Dirty = 1;
1995
1996 if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1997 {
1998 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1999
2000 Assert(pPdeSrc->b.u1Write);
2001
2002 pPdeDst->n.u1Write = 1;
2003 pPdeDst->n.u1Accessed = 1;
2004 pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2005 PGM_INVL_BIG_PG(GCPtrPage);
2006 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2007 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2008 }
2009 }
2010 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2011 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2012 }
2013 /* else: 4KB page table */
2014
2015 /*
2016 * Map the guest page table.
2017 */
2018 PGSTPT pPTSrc;
2019 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2020 if (VBOX_SUCCESS(rc))
2021 {
2022 /*
2023 * Real page fault?
2024 */
2025 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2026 const GSTPTE PteSrc = *pPteSrc;
2027 if ( !PteSrc.n.u1Present
2028# if PGM_WITH_NX(PGM_GST_TYPE)
2029 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2030# endif
2031 || (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault || fWriteProtect))
2032 || (fUserLevelFault && !PteSrc.n.u1User)
2033 )
2034 {
2035# ifdef IN_GC
2036 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
2037# endif
2038 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2039 LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2040
2041 /* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2042 * See the 2nd case above as well.
2043 */
2044 if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2045 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2046
2047 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2048 return VINF_EM_RAW_GUEST_TRAP;
2049 }
2050 LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2051
2052 /*
2053 * Set the accessed bits in the page directory and the page table.
2054 */
2055# if PGM_GST_TYPE == PGM_TYPE_AMD64
2056 pPml4eSrc->n.u1Accessed = 1;
2057 pPdpeSrc->lm.u1Accessed = 1;
2058# endif
2059 pPdeSrc->n.u1Accessed = 1;
2060 pPteSrc->n.u1Accessed = 1;
2061
2062 /*
2063 * Only write protection page faults are relevant here.
2064 */
2065 if (fWriteFault)
2066 {
2067 /* Write access, so mark guest entry as dirty. */
2068# if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
2069 if (!pPteSrc->n.u1Dirty)
2070 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
2071 else
2072 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
2073# endif
2074
2075 pPteSrc->n.u1Dirty = 1;
2076
2077 if (pPdeDst->n.u1Present)
2078 {
2079 /* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2080 * Our individual shadow handlers will provide more information and force a fatal exit.
2081 */
2082 if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2083 {
2084 LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
2085 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2086 return VINF_SUCCESS;
2087 }
2088
2089 /*
2090 * Map shadow page table.
2091 */
2092 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2093 if (pShwPage)
2094 {
2095 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2096 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2097 if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
2098 && (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
2099 {
2100 LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
2101# ifdef VBOX_STRICT
2102 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2103 if (pPage)
2104 AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
2105 ("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
2106# endif
2107 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
2108
2109 Assert(pPteSrc->n.u1Write);
2110
2111 pPteDst->n.u1Write = 1;
2112 pPteDst->n.u1Dirty = 1;
2113 pPteDst->n.u1Accessed = 1;
2114 pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2115 PGM_INVL_PG(GCPtrPage);
2116
2117 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2118 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2119 }
2120 }
2121 else
2122 AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2123 }
2124 }
2125/** @todo Optimize accessed bit emulation? */
2126# ifdef VBOX_STRICT
2127 /*
2128 * Sanity check.
2129 */
2130 else if ( !pPteSrc->n.u1Dirty
2131 && (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2132 && pPdeDst->n.u1Present)
2133 {
2134 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2135 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2136 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2137 if ( pPteDst->n.u1Present
2138 && pPteDst->n.u1Write)
2139 LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2140 }
2141# endif /* VBOX_STRICT */
2142 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2143 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2144 }
2145 AssertRC(rc);
2146 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2147 return rc;
2148
2149
2150UpperLevelPageFault:
2151 /* Pagefault detected while checking the PML4E, PDPE or PDE.
2152 * Single exit handler to get rid of duplicate code paths.
2153 */
2154# ifdef IN_GC
2155 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
2156# endif
2157 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
2158 LogFlow(("CheckPageFault: real page fault at %VGv (%d)\n", GCPtrPage, uPageFaultLevel));
2159
2160 if (
2161# if PGM_GST_TYPE == PGM_TYPE_AMD64
2162 pPml4eSrc->n.u1Present &&
2163# endif
2164# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
2165 pPdpeSrc->n.u1Present &&
2166# endif
2167 pPdeSrc->n.u1Present)
2168 {
2169 /* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2170 if (pPdeSrc->b.u1Size && fBigPagesSupported)
2171 {
2172 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2173 }
2174 else
2175 {
2176 /*
2177 * Map the guest page table.
2178 */
2179 PGSTPT pPTSrc;
2180 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2181 if (VBOX_SUCCESS(rc))
2182 {
2183 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2184 const GSTPTE PteSrc = *pPteSrc;
2185 if (pPteSrc->n.u1Present)
2186 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2187 }
2188 AssertRC(rc);
2189 }
2190 }
2191 return VINF_EM_RAW_GUEST_TRAP;
2192}
2193
2194#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2195
2196
2197/**
2198 * Sync a shadow page table.
2199 *
2200 * The shadow page table is not present. This includes the case where
2201 * there is a conflict with a mapping.
2202 *
2203 * @returns VBox status code.
2204 * @param pVM VM handle.
2205 * @param iPD Page directory index.
2206 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
2207 * Assume this is a temporary mapping.
2208 * @param GCPtrPage GC Pointer of the page that caused the fault
2209 */
2210PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
2211{
2212 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2213 STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
2214 LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
2215
2216#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2217 || PGM_GST_TYPE == PGM_TYPE_PAE \
2218 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2219 && PGM_SHW_TYPE != PGM_TYPE_NESTED
2220
2221 int rc = VINF_SUCCESS;
2222
2223 /*
2224 * Validate input a little bit.
2225 */
2226 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%VGv\n", iPDSrc, GCPtrPage));
2227# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2228 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2229 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2230# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2231 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2232 const unsigned iPdPte = (GCPtrPage >> X86_PDPT_SHIFT);
2233 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT);
2234 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2235# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2236 const unsigned iPml4e = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
2237 const unsigned iPdPte = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2238 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2239 PX86PDPAE pPDDst;
2240 PX86PDPT pPdptDst;
2241 rc = PGMShwGetLongModePDPtr(pVM, GCPtrPage, &pPdptDst, &pPDDst);
2242 if (rc != VINF_SUCCESS)
2243 {
2244 AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
2245 return rc;
2246 }
2247 Assert(pPDDst);
2248# endif
2249
2250 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2251 SHWPDE PdeDst = *pPdeDst;
2252
2253# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
2254 /* Fetch the pgm pool shadow descriptor. */
2255 PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(pVM, pPdptDst->a[iPdPte].u & X86_PDPE_PG_MASK);
2256 Assert(pShwPde);
2257# endif
2258
2259# ifndef PGM_WITHOUT_MAPPINGS
2260 /*
2261 * Check for conflicts.
2262 * GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2263 * HC: Simply resolve the conflict.
2264 */
2265 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2266 {
2267 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2268# ifndef IN_RING3
2269 Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
2270 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2271 return VERR_ADDRESS_CONFLICT;
2272# else
2273 PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2274 Assert(pMapping);
2275# if PGM_GST_TYPE == PGM_TYPE_32BIT
2276 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2277# elif PGM_GST_TYPE == PGM_TYPE_PAE
2278 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2279# else
2280 AssertFailed(); /* can't happen for amd64 */
2281# endif
2282 if (VBOX_FAILURE(rc))
2283 {
2284 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2285 return rc;
2286 }
2287 PdeDst = *pPdeDst;
2288# endif
2289 }
2290# else /* PGM_WITHOUT_MAPPINGS */
2291 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2292# endif /* PGM_WITHOUT_MAPPINGS */
2293 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2294
2295 /*
2296 * Sync page directory entry.
2297 */
2298 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2299 if (PdeSrc.n.u1Present)
2300 {
2301 /*
2302 * Allocate & map the page table.
2303 */
2304 PSHWPT pPTDst;
2305# if PGM_GST_TYPE == PGM_TYPE_AMD64
2306 const bool fPageTable = !PdeSrc.b.u1Size;
2307# else
2308 const bool fPageTable = !PdeSrc.b.u1Size || !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
2309# endif
2310 PPGMPOOLPAGE pShwPage;
2311 RTGCPHYS GCPhys;
2312 if (fPageTable)
2313 {
2314 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2315# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2316 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2317 GCPhys |= (iPDDst & 1) * (PAGE_SIZE / 2);
2318# endif
2319# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
2320 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2321# else
2322 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2323# endif
2324 }
2325 else
2326 {
2327 GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2328# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2329 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2330 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2331# endif
2332# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
2333 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, pShwPde->idx, iPDDst, &pShwPage);
2334# else
2335 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2336# endif
2337 }
2338 if (rc == VINF_SUCCESS)
2339 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2340 else if (rc == VINF_PGM_CACHED_PAGE)
2341 {
2342 /*
2343 * The PT was cached, just hook it up.
2344 */
2345 if (fPageTable)
2346 PdeDst.u = pShwPage->Core.Key
2347 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2348 else
2349 {
2350 PdeDst.u = pShwPage->Core.Key
2351 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2352 /* (see explanation and assumptions further down.) */
2353 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2354 {
2355 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2356 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2357 PdeDst.b.u1Write = 0;
2358 }
2359 }
2360 *pPdeDst = PdeDst;
2361 return VINF_SUCCESS;
2362 }
2363 else if (rc == VERR_PGM_POOL_FLUSHED)
2364 return VINF_PGM_SYNC_CR3;
2365 else
2366 AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2367 PdeDst.u &= X86_PDE_AVL_MASK;
2368 PdeDst.u |= pShwPage->Core.Key;
2369
2370 /*
2371 * Page directory has been accessed (this is a fault situation, remember).
2372 */
2373 pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2374 if (fPageTable)
2375 {
2376 /*
2377 * Page table - 4KB.
2378 *
2379 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2380 */
2381 Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2382 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2383 PGSTPT pPTSrc;
2384 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2385 if (VBOX_SUCCESS(rc))
2386 {
2387 /*
2388 * Start by syncing the page directory entry so CSAM's TLB trick works.
2389 */
2390 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
2391 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2392 *pPdeDst = PdeDst;
2393
2394 /*
2395 * Directory/page user or supervisor privilege: (same goes for read/write)
2396 *
2397 * Directory Page Combined
2398 * U/S U/S U/S
2399 * 0 0 0
2400 * 0 1 0
2401 * 1 0 0
2402 * 1 1 1
2403 *
2404 * Simple AND operation. Table listed for completeness.
2405 *
2406 */
2407 STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
2408# ifdef PGM_SYNC_N_PAGES
2409 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2410 unsigned iPTDst = iPTBase;
2411 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
2412 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2413 iPTDst = 0;
2414 else
2415 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2416# else /* !PGM_SYNC_N_PAGES */
2417 unsigned iPTDst = 0;
2418 const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
2419# endif /* !PGM_SYNC_N_PAGES */
2420# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2421 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2422 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2423# else
2424 const unsigned offPTSrc = 0;
2425# endif
2426 for (; iPTDst < iPTDstEnd; iPTDst++)
2427 {
2428 const unsigned iPTSrc = iPTDst + offPTSrc;
2429 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2430
2431 if (PteSrc.n.u1Present) /* we've already cleared it above */
2432 {
2433# ifndef IN_RING0
2434 /*
2435 * Assuming kernel code will be marked as supervisor - and not as user level
2436 * and executed using a conforming code selector - And marked as readonly.
2437 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
2438 */
2439 PPGMPAGE pPage;
2440 if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW | X86_PTE_US))
2441 || !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)))
2442 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2443 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2444 )
2445# endif
2446 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2447 Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
2448 (RTGCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)),
2449 PteSrc.n.u1Present,
2450 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2451 PteSrc.n.u1User & PdeSrc.n.u1User,
2452 (uint64_t)PteSrc.u,
2453 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2454 (PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
2455 }
2456 } /* for PTEs */
2457 }
2458 }
2459 else
2460 {
2461 /*
2462 * Big page - 2/4MB.
2463 *
2464 * We'll walk the ram range list in parallel and optimize lookups.
2465 * We will only sync on shadow page table at a time.
2466 */
2467 STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
2468
2469 /**
2470 * @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2471 */
2472
2473 /*
2474 * Start by syncing the page directory entry.
2475 */
2476 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2477 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2478
2479 /*
2480 * If the page is not flagged as dirty and is writable, then make it read-only
2481 * at PD level, so we can set the dirty bit when the page is modified.
2482 *
2483 * ASSUMES that page access handlers are implemented on page table entry level.
2484 * Thus we will first catch the dirty access and set PDE.D and restart. If
2485 * there is an access handler, we'll trap again and let it work on the problem.
2486 */
2487 /** @todo move the above stuff to a section in the PGM documentation. */
2488 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2489 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2490 {
2491 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2492 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2493 PdeDst.b.u1Write = 0;
2494 }
2495 *pPdeDst = PdeDst;
2496
2497 /*
2498 * Fill the shadow page table.
2499 */
2500 /* Get address and flags from the source PDE. */
2501 SHWPTE PteDstBase;
2502 PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT);
2503
2504 /* Loop thru the entries in the shadow PT. */
2505 const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2506 Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2507 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2508 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2509 PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
2510 unsigned iPTDst = 0;
2511 while (iPTDst < ELEMENTS(pPTDst->a))
2512 {
2513 /* Advance ram range list. */
2514 while (pRam && GCPhys > pRam->GCPhysLast)
2515 pRam = CTXALLSUFF(pRam->pNext);
2516 if (pRam && GCPhys >= pRam->GCPhys)
2517 {
2518 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2519 do
2520 {
2521 /* Make shadow PTE. */
2522 PPGMPAGE pPage = &pRam->aPages[iHCPage];
2523 SHWPTE PteDst;
2524
2525 /* Make sure the RAM has already been allocated. */
2526 if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2527 {
2528 if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2529 {
2530# ifdef IN_RING3
2531 int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2532# else
2533 int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2534# endif
2535 if (rc != VINF_SUCCESS)
2536 return rc;
2537 }
2538 }
2539
2540 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2541 {
2542 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2543 {
2544 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2545 PteDst.n.u1Write = 0;
2546 }
2547 else
2548 PteDst.u = 0;
2549 }
2550# ifndef IN_RING0
2551 /*
2552 * Assuming kernel code will be marked as supervisor and not as user level and executed
2553 * using a conforming code selector. Don't check for readonly, as that implies the whole
2554 * 4MB can be code or readonly data. Linux enables write access for its large pages.
2555 */
2556 else if ( !PdeSrc.n.u1User
2557 && CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))))
2558 PteDst.u = 0;
2559# endif
2560 else
2561 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2562# ifdef PGMPOOL_WITH_USER_TRACKING
2563 if (PteDst.n.u1Present)
2564 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2565# endif
2566 /* commit it */
2567 pPTDst->a[iPTDst] = PteDst;
2568 Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2569 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2570 PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2571
2572 /* advance */
2573 GCPhys += PAGE_SIZE;
2574 iHCPage++;
2575 iPTDst++;
2576 } while ( iPTDst < ELEMENTS(pPTDst->a)
2577 && GCPhys <= pRam->GCPhysLast);
2578 }
2579 else if (pRam)
2580 {
2581 Log(("Invalid pages at %VGp\n", GCPhys));
2582 do
2583 {
2584 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2585 GCPhys += PAGE_SIZE;
2586 iPTDst++;
2587 } while ( iPTDst < ELEMENTS(pPTDst->a)
2588 && GCPhys < pRam->GCPhys);
2589 }
2590 else
2591 {
2592 Log(("Invalid pages at %VGp (2)\n", GCPhys));
2593 for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2594 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2595 }
2596 } /* while more PTEs */
2597 } /* 4KB / 4MB */
2598 }
2599 else
2600 AssertRelease(!PdeDst.n.u1Present);
2601
2602 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2603# ifdef IN_GC
2604 if (VBOX_FAILURE(rc))
2605 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2606# endif
2607 return rc;
2608
2609#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2610 && PGM_SHW_TYPE != PGM_TYPE_NESTED
2611
2612 int rc = VINF_SUCCESS;
2613
2614 /*
2615 * Validate input a little bit.
2616 */
2617# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2618 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2619# else
2620 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2621# endif
2622 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2623 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2624 SHWPDE PdeDst = *pPdeDst;
2625
2626 Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2627 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2628
2629 GSTPDE PdeSrc;
2630 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2631 PdeSrc.n.u1Present = 1;
2632 PdeSrc.n.u1Write = 1;
2633 PdeSrc.n.u1Accessed = 1;
2634 PdeSrc.n.u1User = 1;
2635
2636 /*
2637 * Allocate & map the page table.
2638 */
2639 PSHWPT pPTDst;
2640 PPGMPOOLPAGE pShwPage;
2641 RTGCPHYS GCPhys;
2642
2643 /* Virtual address = physical address */
2644 GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK_32;
2645 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2646
2647 if ( rc == VINF_SUCCESS
2648 || rc == VINF_PGM_CACHED_PAGE)
2649 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2650 else
2651 AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2652
2653 PdeDst.u &= X86_PDE_AVL_MASK;
2654 PdeDst.u |= pShwPage->Core.Key;
2655 PdeDst.n.u1Present = 1;
2656 PdeDst.n.u1Write = 1;
2657 PdeDst.n.u1User = 1;
2658 *pPdeDst = PdeDst;
2659
2660 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2661 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2662 return rc;
2663
2664#else
2665 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2666 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2667 return VERR_INTERNAL_ERROR;
2668#endif
2669}
2670
2671
2672
2673/**
2674 * Prefetch a page/set of pages.
2675 *
2676 * Typically used to sync commonly used pages before entering raw mode
2677 * after a CR3 reload.
2678 *
2679 * @returns VBox status code.
2680 * @param pVM VM handle.
2681 * @param GCPtrPage Page to invalidate.
2682 */
2683PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2684{
2685#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2686 && PGM_SHW_TYPE != PGM_TYPE_NESTED
2687 /*
2688 * Check that all Guest levels thru the PDE are present, getting the
2689 * PD and PDE in the processes.
2690 */
2691 int rc = VINF_SUCCESS;
2692# if PGM_WITH_PAGING(PGM_GST_TYPE)
2693# if PGM_GST_TYPE == PGM_TYPE_32BIT
2694 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2695 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2696# elif PGM_GST_TYPE == PGM_TYPE_PAE
2697 unsigned iPDSrc;
2698 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2699 if (!pPDSrc)
2700 return VINF_SUCCESS; /* not present */
2701# elif PGM_GST_TYPE == PGM_TYPE_AMD64
2702 unsigned iPDSrc;
2703 PX86PML4E pPml4eSrc;
2704 X86PDPE PdpeSrc;
2705 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
2706 if (!pPDSrc)
2707 return VINF_SUCCESS; /* not present */
2708# endif
2709 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2710# else
2711 PGSTPD pPDSrc = NULL;
2712 const unsigned iPDSrc = 0;
2713 GSTPDE PdeSrc;
2714
2715 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2716 PdeSrc.n.u1Present = 1;
2717 PdeSrc.n.u1Write = 1;
2718 PdeSrc.n.u1Accessed = 1;
2719 PdeSrc.n.u1User = 1;
2720# endif
2721
2722 if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2723 {
2724# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2725 const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2726# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2727 const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2728# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2729 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2730 PX86PDPAE pPDDst;
2731 X86PDEPAE PdeDst;
2732
2733# if PGM_GST_TYPE == PGM_TYPE_PROT
2734 /* AMD-V nested paging */
2735 X86PML4E Pml4eSrc;
2736 X86PDPE PdpeSrc;
2737 PX86PML4E pPml4eSrc = &Pml4eSrc;
2738
2739 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
2740 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_NX | X86_PML4E_A;
2741 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_NX | X86_PDPE_A;
2742# endif
2743
2744 int rc = PGMShwSyncLongModePDPtr(pVM, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
2745 if (rc != VINF_SUCCESS)
2746 {
2747 AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
2748 return rc;
2749 }
2750 Assert(pPDDst);
2751 PdeDst = pPDDst->a[iPDDst];
2752# endif
2753 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2754 {
2755 if (!PdeDst.n.u1Present)
2756 /** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2757 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2758 else
2759 {
2760 /** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2761 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2762 * makes no sense to prefetch more than one page.
2763 */
2764 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2765 if (VBOX_SUCCESS(rc))
2766 rc = VINF_SUCCESS;
2767 }
2768 }
2769 }
2770 return rc;
2771#elif PGM_SHW_TYPE == PGM_TYPE_NESTED
2772 return VINF_SUCCESS; /* ignore */
2773#endif
2774}
2775
2776
2777
2778
2779/**
2780 * Syncs a page during a PGMVerifyAccess() call.
2781 *
2782 * @returns VBox status code (informational included).
2783 * @param GCPtrPage The address of the page to sync.
2784 * @param fPage The effective guest page flags.
2785 * @param uErr The trap error code.
2786 */
2787PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2788{
2789 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2790
2791 Assert(!HWACCMIsNestedPagingActive(pVM));
2792#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_TYPE_AMD64) \
2793 && PGM_SHW_TYPE != PGM_TYPE_NESTED
2794
2795# ifndef IN_RING0
2796 if (!(fPage & X86_PTE_US))
2797 {
2798 /*
2799 * Mark this page as safe.
2800 */
2801 /** @todo not correct for pages that contain both code and data!! */
2802 Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2803 CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
2804 }
2805# endif
2806 /*
2807 * Get guest PD and index.
2808 */
2809
2810# if PGM_WITH_PAGING(PGM_GST_TYPE)
2811# if PGM_GST_TYPE == PGM_TYPE_32BIT
2812 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2813 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2814# elif PGM_GST_TYPE == PGM_TYPE_PAE
2815 unsigned iPDSrc;
2816 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2817
2818 if (pPDSrc)
2819 {
2820 Log(("PGMVerifyAccess: access violation for %VGv due to non-present PDPTR\n", GCPtrPage));
2821 return VINF_EM_RAW_GUEST_TRAP;
2822 }
2823# elif PGM_GST_TYPE == PGM_TYPE_AMD64
2824 unsigned iPDSrc;
2825 PX86PML4E pPml4eSrc;
2826 X86PDPE PdpeSrc;
2827 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
2828 if (!pPDSrc)
2829 {
2830 Log(("PGMVerifyAccess: access violation for %VGv due to non-present PDPTR\n", GCPtrPage));
2831 return VINF_EM_RAW_GUEST_TRAP;
2832 }
2833# endif
2834# else
2835 PGSTPD pPDSrc = NULL;
2836 const unsigned iPDSrc = 0;
2837# endif
2838 int rc = VINF_SUCCESS;
2839
2840 /*
2841 * First check if the shadow pd is present.
2842 */
2843# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2844 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2845# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2846 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2847# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2848 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2849 PX86PDPAE pPDDst;
2850 PX86PDEPAE pPdeDst;
2851
2852# if PGM_GST_TYPE == PGM_TYPE_PROT
2853 /* AMD-V nested paging */
2854 X86PML4E Pml4eSrc;
2855 X86PDPE PdpeSrc;
2856 PX86PML4E pPml4eSrc = &Pml4eSrc;
2857
2858 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
2859 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_NX | X86_PML4E_A;
2860 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_NX | X86_PDPE_A;
2861# endif
2862
2863 rc = PGMShwSyncLongModePDPtr(pVM, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
2864 if (rc != VINF_SUCCESS)
2865 {
2866 AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
2867 return rc;
2868 }
2869 Assert(pPDDst);
2870 pPdeDst = &pPDDst->a[iPDDst];
2871# endif
2872 if (!pPdeDst->n.u1Present)
2873 {
2874 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2875 AssertRC(rc);
2876 if (rc != VINF_SUCCESS)
2877 return rc;
2878 }
2879
2880# if PGM_WITH_PAGING(PGM_GST_TYPE)
2881 /* Check for dirty bit fault */
2882 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2883 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2884 Log(("PGMVerifyAccess: success (dirty)\n"));
2885 else
2886 {
2887 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2888#else
2889 {
2890 GSTPDE PdeSrc;
2891 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2892 PdeSrc.n.u1Present = 1;
2893 PdeSrc.n.u1Write = 1;
2894 PdeSrc.n.u1Accessed = 1;
2895 PdeSrc.n.u1User = 1;
2896
2897#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2898 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2899 if (uErr & X86_TRAP_PF_US)
2900 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2901 else /* supervisor */
2902 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2903
2904 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2905 if (VBOX_SUCCESS(rc))
2906 {
2907 /* Page was successfully synced */
2908 Log2(("PGMVerifyAccess: success (sync)\n"));
2909 rc = VINF_SUCCESS;
2910 }
2911 else
2912 {
2913 Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2914 return VINF_EM_RAW_GUEST_TRAP;
2915 }
2916 }
2917 return rc;
2918
2919#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2920
2921 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2922 return VERR_INTERNAL_ERROR;
2923#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2924}
2925
2926
2927#if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
2928# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
2929/**
2930 * Figures out which kind of shadow page this guest PDE warrants.
2931 *
2932 * @returns Shadow page kind.
2933 * @param pPdeSrc The guest PDE in question.
2934 * @param cr4 The current guest cr4 value.
2935 */
2936DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
2937{
2938# if PMG_GST_TYPE == PGM_TYPE_AMD64
2939 if (!pPdeSrc->n.u1Size)
2940# else
2941 if (!pPdeSrc->n.u1Size || !(cr4 & X86_CR4_PSE))
2942# endif
2943 return BTH_PGMPOOLKIND_PT_FOR_PT;
2944 //switch (pPdeSrc->u & (X86_PDE4M_RW | X86_PDE4M_US /*| X86_PDE4M_PAE_NX*/))
2945 //{
2946 // case 0:
2947 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2948 // case X86_PDE4M_RW:
2949 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2950 // case X86_PDE4M_US:
2951 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2952 // case X86_PDE4M_RW | X86_PDE4M_US:
2953 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2954# if 0
2955 // case X86_PDE4M_PAE_NX:
2956 // return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2957 // case X86_PDE4M_RW | X86_PDE4M_PAE_NX:
2958 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2959 // case X86_PDE4M_US | X86_PDE4M_PAE_NX:
2960 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2961 // case X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PAE_NX:
2962 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2963# endif
2964 return BTH_PGMPOOLKIND_PT_FOR_BIG;
2965 //}
2966}
2967# endif
2968#endif
2969
2970#undef MY_STAM_COUNTER_INC
2971#define MY_STAM_COUNTER_INC(a) do { } while (0)
2972
2973
2974/**
2975 * Syncs the paging hierarchy starting at CR3.
2976 *
2977 * @returns VBox status code, no specials.
2978 * @param pVM The virtual machine.
2979 * @param cr0 Guest context CR0 register
2980 * @param cr3 Guest context CR3 register
2981 * @param cr4 Guest context CR4 register
2982 * @param fGlobal Including global page directories or not
2983 */
2984PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2985{
2986#if PGM_SHW_TYPE == PGM_TYPE_NESTED
2987 /** @todo check if this is really necessary */
2988 HWACCMFlushTLB(pVM);
2989 return VINF_SUCCESS;
2990
2991#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED */
2992 if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2993 fGlobal = true; /* Change this CR3 reload to be a global one. */
2994
2995 /*
2996 * Update page access handlers.
2997 * The virtual are always flushed, while the physical are only on demand.
2998 * WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2999 * have to look into that later because it will have a bad influence on the performance.
3000 * @note SvL: There's no need for that. Just invalidate the virtual range(s).
3001 * bird: Yes, but that won't work for aliases.
3002 */
3003 /** @todo this MUST go away. See #1557. */
3004 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
3005 PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3006 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
3007
3008# ifdef PGMPOOL_WITH_MONITORING
3009 /*
3010 * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
3011 * Occationally we will have to clear all the shadow page tables because we wanted
3012 * to monitor a page which was mapped by too many shadowed page tables. This operation
3013 * sometimes refered to as a 'lightweight flush'.
3014 */
3015 if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
3016 pgmPoolMonitorModifiedClearAll(pVM);
3017 else
3018 {
3019# ifdef IN_RING3
3020 pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
3021 pgmPoolClearAll(pVM);
3022# else
3023 LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
3024 return VINF_PGM_SYNC_CR3;
3025# endif
3026 }
3027# endif
3028
3029 Assert(fGlobal || (cr4 & X86_CR4_PGE));
3030 MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
3031
3032# if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3033# if PGM_GST_TYPE == PGM_TYPE_AMD64
3034 bool fBigPagesSupported = true;
3035# else
3036 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
3037# endif
3038
3039 /*
3040 * Get page directory addresses.
3041 */
3042# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3043 PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
3044# else /* PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64*/
3045# if PGM_GST_TYPE == PGM_TYPE_32BIT
3046 PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
3047# endif
3048# endif
3049
3050# if PGM_GST_TYPE == PGM_TYPE_32BIT
3051 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
3052 Assert(pPDSrc);
3053# ifndef IN_GC
3054 Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & GST_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
3055# endif
3056# endif
3057
3058 /*
3059 * Iterate the page directory.
3060 */
3061 PPGMMAPPING pMapping;
3062 unsigned iPdNoMapping;
3063 const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
3064 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3065
3066 /* Only check mappings if they are supposed to be put into the shadow page table. */
3067 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
3068 {
3069 pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
3070 iPdNoMapping = (pMapping) ? (pMapping->GCPtr >> GST_PD_SHIFT) : ~0U;
3071 }
3072 else
3073 {
3074 pMapping = 0;
3075 iPdNoMapping = ~0U;
3076 }
3077# if PGM_GST_TYPE == PGM_TYPE_AMD64
3078 for (uint64_t iPml4e = 0; iPml4e < X86_PG_PAE_ENTRIES; iPml4e++)
3079 {
3080 PPGMPOOLPAGE pShwPdpt = NULL;
3081 PX86PML4E pPml4eSrc, pPml4eDst;
3082 RTGCPHYS GCPhysPdptSrc;
3083
3084 pPml4eSrc = &pVM->pgm.s.CTXSUFF(pGstPaePML4)->a[iPml4e];
3085 pPml4eDst = &pVM->pgm.s.CTXMID(p,PaePML4)->a[iPml4e];
3086
3087 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3088 if (!pVM->pgm.s.CTXMID(p,PaePML4)->a[iPml4e].n.u1Present)
3089 continue;
3090 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3091
3092 GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK;
3093
3094 /* Anything significant changed? */
3095 if ( pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present
3096 || GCPhysPdptSrc != pShwPdpt->GCPhys)
3097 {
3098 /* Free it. */
3099 LogFlow(("SyncCR3: Out-of-sync PML4E (GCPhys) GCPtr=%VGv %VGp vs %VGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
3100 (uint64_t)iPml4e << X86_PML4_SHIFT, pShwPdpt->GCPhys, GCPhysPdptSrc, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
3101 pgmPoolFreeByPage(pPool, pShwPdpt, PGMPOOL_IDX_PML4, iPml4e);
3102 pPml4eDst->u = 0;
3103 continue;
3104 }
3105 /* Force an attribute sync. */
3106 pPml4eDst->n.u1User = pPml4eSrc->n.u1User;
3107 pPml4eDst->n.u1Write = pPml4eSrc->n.u1Write;
3108 pPml4eDst->n.u1NoExecute = pPml4eSrc->n.u1NoExecute;
3109
3110# else
3111 {
3112# endif
3113# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3114 for (uint64_t iPDPTE = 0; iPDPTE < GST_PDPE_ENTRIES; iPDPTE++)
3115 {
3116 unsigned iPDSrc;
3117 PPGMPOOLPAGE pShwPde = NULL;
3118 PX86PDPE pPdpeDst;
3119 RTGCPHYS GCPhysPdeSrc;
3120# if PGM_GST_TYPE == PGM_TYPE_PAE
3121 PX86PDPAE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
3122 PX86PDEPAE pPDEDst = &pPDPAE->a[iPDPTE * X86_PG_PAE_ENTRIES];
3123 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, iPDPTE << X86_PDPT_SHIFT, &iPDSrc);
3124 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT);
3125 X86PDPE PdpeSrc = CTXSUFF(pVM->pgm.s.pGstPaePDPT)->a[iPDPTE];
3126# else
3127 PX86PML4E pPml4eSrc;
3128 X86PDPE PdpeSrc;
3129 PX86PDPT pPdptDst;
3130 PX86PDPAE pPDDst;
3131 PX86PDEPAE pPDEDst;
3132 RTGCUINTPTR GCPtr = (iPml4e << X86_PML4_SHIFT) || (iPDPTE << X86_PDPT_SHIFT);
3133 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3134
3135 int rc = PGMShwGetLongModePDPtr(pVM, GCPtr, &pPdptDst, &pPDDst);
3136 if (rc != VINF_SUCCESS)
3137 {
3138 if (rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
3139 break; /* next PML4E */
3140
3141 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Vrc\n", rc));
3142 continue; /* next PDPTE */
3143 }
3144 Assert(pPDDst);
3145 pPDEDst = &pPDDst->a[0];
3146# endif
3147 Assert(iPDSrc == 0);
3148
3149 pPdpeDst = &pPdptDst->a[iPDPTE];
3150
3151 /* Fetch the pgm pool shadow descriptor if the shadow pdpte is present. */
3152 if (!pPdpeDst->n.u1Present)
3153 continue; /* next PDPTE */
3154
3155 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3156 GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3157
3158 /* Anything significant changed? */
3159 if ( PdpeSrc.n.u1Present != pPdpeDst->n.u1Present
3160 || GCPhysPdeSrc != pShwPde->GCPhys)
3161 {
3162 /* Free it. */
3163# if PGM_GST_TYPE == PGM_TYPE_AMD64
3164 LogFlow(("SyncCR3: Out-of-sync PDPE (GCPhys) GCPtr=%VGv %VGp vs %VGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
3165 ((uint64_t)iPml4e << X86_PML4_SHIFT) + ((uint64_t)iPDPTE << X86_PDPT_SHIFT), pShwPde->GCPhys, GCPhysPdeSrc, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
3166# else
3167 LogFlow(("SyncCR3: Out-of-sync PDPE (GCPhys) GCPtr=%VGv %VGp vs %VGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
3168 (uint64_t)iPDPTE << X86_PDPT_SHIFT, pShwPde->GCPhys, GCPhysPdeSrc, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
3169# endif
3170
3171 /* Mark it as not present if there's no hypervisor mapping present. (bit flipped at the top of Trap0eHandler) */
3172 if (!(pPdpeDst->u & PGM_PLXFLAGS_MAPPING))
3173 {
3174 pgmPoolFreeByPage(pPool, pShwPde, pShwPde->idx, iPDPTE);
3175 pPdpeDst->u = 0;
3176 continue; /* next guest PDPTE */
3177 }
3178 }
3179# if PGM_GST_TYPE == PGM_TYPE_AMD64
3180 /* Force an attribute sync. */
3181 pPdpeDst->lm.u1User = PdpeSrc.lm.u1User;
3182 pPdpeDst->lm.u1Write = PdpeSrc.lm.u1Write;
3183 pPdpeDst->lm.u1NoExecute = PdpeSrc.lm.u1NoExecute;
3184# endif
3185
3186# else /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
3187 {
3188# endif /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
3189 for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
3190 {
3191# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3192 Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
3193# elif PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3194 AssertMsg(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst, ("%p vs %p\n", &pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512], pPDEDst));
3195# endif
3196 register GSTPDE PdeSrc = pPDSrc->a[iPD];
3197 if ( PdeSrc.n.u1Present
3198 && (PdeSrc.n.u1User || fRawR0Enabled))
3199 {
3200# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3201 || PGM_GST_TYPE == PGM_TYPE_PAE) \
3202 && !defined(PGM_WITHOUT_MAPPINGS)
3203
3204 /*
3205 * Check for conflicts with GC mappings.
3206 */
3207# if PGM_GST_TYPE == PGM_TYPE_PAE
3208 if (iPD + iPDPTE * X86_PG_PAE_ENTRIES == iPdNoMapping)
3209# else
3210 if (iPD == iPdNoMapping)
3211# endif
3212 {
3213 if (pVM->pgm.s.fMappingsFixed)
3214 {
3215 /* It's fixed, just skip the mapping. */
3216 const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
3217 iPD += cPTs - 1;
3218 pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3219 pMapping = pMapping->CTXALLSUFF(pNext);
3220 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3221 continue;
3222 }
3223# ifdef IN_RING3
3224# if PGM_GST_TYPE == PGM_TYPE_32BIT
3225 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
3226# elif PGM_GST_TYPE == PGM_TYPE_PAE
3227 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
3228# endif
3229 if (VBOX_FAILURE(rc))
3230 return rc;
3231
3232 /*
3233 * Update iPdNoMapping and pMapping.
3234 */
3235 pMapping = pVM->pgm.s.pMappingsR3;
3236 while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
3237 pMapping = pMapping->pNextR3;
3238 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3239# else
3240 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3241 return VINF_PGM_SYNC_CR3;
3242# endif
3243 }
3244# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3245 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3246# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3247 /*
3248 * Sync page directory entry.
3249 *
3250 * The current approach is to allocated the page table but to set
3251 * the entry to not-present and postpone the page table synching till
3252 * it's actually used.
3253 */
3254# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3255 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
3256# else
3257 const unsigned iPdShw = iPD; NOREF(iPdShw);
3258# endif
3259 {
3260 SHWPDE PdeDst = *pPDEDst;
3261 if (PdeDst.n.u1Present)
3262 {
3263 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
3264 RTGCPHYS GCPhys;
3265 if ( !PdeSrc.b.u1Size
3266 || !fBigPagesSupported)
3267 {
3268 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
3269# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3270 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3271 GCPhys |= i * (PAGE_SIZE / 2);
3272# endif
3273 }
3274 else
3275 {
3276 GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
3277# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3278 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
3279 GCPhys |= i * X86_PAGE_2M_SIZE;
3280# endif
3281 }
3282
3283 if ( pShwPage->GCPhys == GCPhys
3284 && pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
3285 && ( pShwPage->fCached
3286 || ( !fGlobal
3287 && ( false
3288# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
3289 || ( (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
3290# if PGM_GST_TYPE == PGM_TYPE_AMD64
3291 && (cr4 & X86_CR4_PGE)) /* global 2/4MB page. */
3292# else
3293 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE)) /* global 2/4MB page. */
3294# endif
3295 || ( !pShwPage->fSeenNonGlobal
3296 && (cr4 & X86_CR4_PGE))
3297# endif
3298 )
3299 )
3300 )
3301 && ( (PdeSrc.u & (X86_PDE_US | X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US | X86_PDE_RW))
3302 || ( fBigPagesSupported
3303 && ((PdeSrc.u & (X86_PDE_US | X86_PDE4M_PS | X86_PDE4M_D)) | PGM_PDFLAGS_TRACK_DIRTY)
3304 == ((PdeDst.u & (X86_PDE_US | X86_PDE_RW | PGM_PDFLAGS_TRACK_DIRTY)) | X86_PDE4M_PS))
3305 )
3306 )
3307 {
3308# ifdef VBOX_WITH_STATISTICS
3309 if ( !fGlobal
3310 && (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
3311# if PGM_GST_TYPE == PGM_TYPE_AMD64
3312 && (cr4 & X86_CR4_PGE)) /* global 2/4MB page. */
3313# else
3314 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE))
3315# endif
3316 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
3317 else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
3318 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
3319 else
3320 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
3321# endif /* VBOX_WITH_STATISTICS */
3322 /** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
3323 * The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
3324 //# ifdef PGMPOOL_WITH_CACHE
3325 // pgmPoolCacheUsed(pPool, pShwPage);
3326 //# endif
3327 }
3328 else
3329 {
3330# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3331 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPdShw);
3332# else
3333 pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
3334# endif
3335 pPDEDst->u = 0;
3336 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
3337 }
3338 }
3339 else
3340 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
3341 pPDEDst++;
3342 }
3343 }
3344# if PGM_GST_TYPE == PGM_TYPE_PAE
3345 else if (iPD + iPDPTE * X86_PG_PAE_ENTRIES != iPdNoMapping)
3346# else
3347 else if (iPD != iPdNoMapping)
3348# endif
3349 {
3350 /*
3351 * Check if there is any page directory to mark not present here.
3352 */
3353# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3354 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
3355# else
3356 const unsigned iPdShw = iPD; NOREF(iPdShw);
3357# endif
3358 {
3359 if (pPDEDst->n.u1Present)
3360 {
3361# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3362 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), pShwPde->idx, iPdShw);
3363# else
3364 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
3365# endif
3366 pPDEDst->u = 0;
3367 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
3368 }
3369 pPDEDst++;
3370 }
3371 }
3372 else
3373 {
3374# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3375 || PGM_GST_TYPE == PGM_TYPE_PAE) \
3376 && !defined(PGM_WITHOUT_MAPPINGS)
3377
3378 const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
3379
3380 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3381 if (pVM->pgm.s.fMappingsFixed)
3382 {
3383 /* It's fixed, just skip the mapping. */
3384 pMapping = pMapping->CTXALLSUFF(pNext);
3385 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3386 }
3387 else
3388 {
3389 /*
3390 * Check for conflicts for subsequent pagetables
3391 * and advance to the next mapping.
3392 */
3393 iPdNoMapping = ~0U;
3394 unsigned iPT = cPTs;
3395 while (iPT-- > 1)
3396 {
3397 if ( pPDSrc->a[iPD + iPT].n.u1Present
3398 && (pPDSrc->a[iPD + iPT].n.u1User || fRawR0Enabled))
3399 {
3400# ifdef IN_RING3
3401# if PGM_GST_TYPE == PGM_TYPE_32BIT
3402 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
3403# elif PGM_GST_TYPE == PGM_TYPE_PAE
3404 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
3405# endif
3406 if (VBOX_FAILURE(rc))
3407 return rc;
3408
3409 /*
3410 * Update iPdNoMapping and pMapping.
3411 */
3412 pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
3413 while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
3414 pMapping = pMapping->CTXALLSUFF(pNext);
3415 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3416 break;
3417# else
3418 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3419 return VINF_PGM_SYNC_CR3;
3420# endif
3421 }
3422 }
3423 if (iPdNoMapping == ~0U && pMapping)
3424 {
3425 pMapping = pMapping->CTXALLSUFF(pNext);
3426 if (pMapping)
3427 iPdNoMapping = pMapping->GCPtr >> GST_PD_SHIFT;
3428 }
3429 }
3430
3431 /* advance. */
3432 iPD += cPTs - 1;
3433 pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3434# if PGM_GST_TYPE != PGM_SHW_TYPE
3435 AssertCompile(PGM_GST_TYPE == PGM_TYPE_32BIT && PGM_SHW_TYPE == PGM_TYPE_PAE);
3436# endif
3437# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3438 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3439# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3440 }
3441
3442 } /* for iPD */
3443 } /* for each PDPTE (PAE) */
3444 } /* for each page map level 4 entry (amd64) */
3445 return VINF_SUCCESS;
3446
3447# else /* guest real and protected mode */
3448 return VINF_SUCCESS;
3449# endif
3450#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED */
3451}
3452
3453
3454
3455
3456#ifdef VBOX_STRICT
3457#ifdef IN_GC
3458# undef AssertMsgFailed
3459# define AssertMsgFailed Log
3460#endif
3461#ifdef IN_RING3
3462# include <VBox/dbgf.h>
3463
3464/**
3465 * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3466 *
3467 * @returns VBox status code (VINF_SUCCESS).
3468 * @param pVM The VM handle.
3469 * @param cr3 The root of the hierarchy.
3470 * @param crr The cr4, only PAE and PSE is currently used.
3471 * @param fLongMode Set if long mode, false if not long mode.
3472 * @param cMaxDepth Number of levels to dump.
3473 * @param pHlp Pointer to the output functions.
3474 */
3475__BEGIN_DECLS
3476PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3477__END_DECLS
3478
3479#endif
3480
3481/**
3482 * Checks that the shadow page table is in sync with the guest one.
3483 *
3484 * @returns The number of errors.
3485 * @param pVM The virtual machine.
3486 * @param cr3 Guest context CR3 register
3487 * @param cr4 Guest context CR4 register
3488 * @param GCPtr Where to start. Defaults to 0.
3489 * @param cb How much to check. Defaults to everything.
3490 */
3491PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
3492{
3493#if PGM_SHW_TYPE == PGM_TYPE_NESTED
3494 return 0;
3495#else
3496 unsigned cErrors = 0;
3497
3498#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3499 || PGM_GST_TYPE == PGM_TYPE_PAE \
3500 || PGM_GST_TYPE == PGM_TYPE_AMD64
3501
3502# if PGM_GST_TYPE == PGM_TYPE_AMD64
3503 bool fBigPagesSupported = true;
3504# else
3505 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
3506# endif
3507 PPGM pPGM = &pVM->pgm.s;
3508 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3509 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3510# ifndef IN_RING0
3511 RTHCPHYS HCPhys; /* general usage. */
3512# endif
3513 int rc;
3514
3515 /*
3516 * Check that the Guest CR3 and all its mappings are correct.
3517 */
3518 AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3519 ("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3520 false);
3521# ifndef IN_RING0
3522# if PGM_GST_TYPE == PGM_TYPE_32BIT
3523 rc = PGMShwGetPage(pVM, (RTGCPTR)pPGM->pGuestPDGC, NULL, &HCPhysShw);
3524# else
3525 rc = PGMShwGetPage(pVM, (RTGCPTR)pPGM->pGstPaePDPTGC, NULL, &HCPhysShw);
3526# endif
3527 AssertRCReturn(rc, 1);
3528 HCPhys = NIL_RTHCPHYS;
3529 rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3530 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
3531# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3532 RTGCPHYS GCPhys;
3533 rc = PGMR3DbgHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
3534 AssertRCReturn(rc, 1);
3535 AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
3536# endif
3537#endif /* !IN_RING0 */
3538
3539 /*
3540 * Get and check the Shadow CR3.
3541 */
3542# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3543 unsigned cPDEs = X86_PG_ENTRIES;
3544 unsigned ulIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3545# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3546# if PGM_GST_TYPE == PGM_TYPE_32BIT
3547 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3548# else
3549 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3550# endif
3551 unsigned ulIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3552# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3553 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3554 unsigned ulIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3555# endif
3556 if (cb != ~(RTGCUINTPTR)0)
3557 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3558
3559/** @todo call the other two PGMAssert*() functions. */
3560
3561# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
3562 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3563# endif
3564
3565# if PGM_GST_TYPE == PGM_TYPE_AMD64
3566 unsigned iPml4e = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3567
3568 for (; iPml4e < X86_PG_PAE_ENTRIES; iPml4e++)
3569 {
3570 PPGMPOOLPAGE pShwPdpt = NULL;
3571 PX86PML4E pPml4eSrc, pPml4eDst;
3572 RTGCPHYS GCPhysPdptSrc;
3573
3574 pPml4eSrc = &pVM->pgm.s.CTXSUFF(pGstPaePML4)->a[iPml4e];
3575 pPml4eDst = &pVM->pgm.s.CTXMID(p,PaePML4)->a[iPml4e];
3576
3577 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3578 if (!pVM->pgm.s.CTXMID(p,PaePML4)->a[iPml4e].n.u1Present)
3579 {
3580 GCPtr += UINT64_C(_2M * 512 * 512);
3581 continue;
3582 }
3583
3584 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3585 GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK;
3586
3587 if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3588 {
3589 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3590 GCPtr += UINT64_C(_2M * 512 * 512);
3591 cErrors++;
3592 continue;
3593 }
3594
3595 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3596 {
3597 AssertMsgFailed(("Physical address doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs RX64\n", pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3598 GCPtr += UINT64_C(_2M * 512 * 512);
3599 cErrors++;
3600 continue;
3601 }
3602
3603 if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3604 || pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3605 || pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3606 {
3607 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3608 GCPtr += UINT64_C(_2M * 512 * 512);
3609 cErrors++;
3610 continue;
3611 }
3612# else
3613 {
3614# endif
3615
3616# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
3617 /*
3618 * Check the PDPTEs too.
3619 */
3620 unsigned iPdpte = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3621
3622 for (;iPdpte <= SHW_PDPT_MASK; iPdpte++)
3623 {
3624 unsigned iPDSrc;
3625 PPGMPOOLPAGE pShwPde = NULL;
3626 PX86PDPE pPdpeDst;
3627 RTGCPHYS GCPhysPdeSrc;
3628# if PGM_GST_TYPE == PGM_TYPE_PAE
3629 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
3630 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtr, &iPDSrc);
3631 PX86PDPT pPdptDst = pVM->pgm.s.CTXMID(p,PaePDPT);
3632 X86PDPE PdpeSrc = CTXSUFF(pVM->pgm.s.pGstPaePDPT)->a[iPdpte];
3633# else
3634 PX86PML4E pPml4eSrc;
3635 X86PDPE PdpeSrc;
3636 PX86PDPT pPdptDst;
3637 PX86PDPAE pPDDst;
3638 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3639
3640 int rc = PGMShwGetLongModePDPtr(pVM, GCPtr, &pPdptDst, &pPDDst);
3641 if (rc != VINF_SUCCESS)
3642 {
3643 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Vrc\n", rc));
3644 GCPtr += 512 * _2M;
3645 continue; /* next PDPTE */
3646 }
3647 Assert(pPDDst);
3648# endif
3649 Assert(iPDSrc == 0);
3650
3651 pPdpeDst = &pPdptDst->a[iPdpte];
3652
3653 if (!pPdpeDst->n.u1Present)
3654 {
3655 GCPtr += 512 * _2M;
3656 continue; /* next PDPTE */
3657 }
3658
3659 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3660 GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3661
3662 if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3663 {
3664 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3665 GCPtr += 512 * _2M;
3666 cErrors++;
3667 continue;
3668 }
3669
3670 if (GCPhysPdeSrc != pShwPde->GCPhys)
3671 {
3672 AssertMsgFailed(("Physical address doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs RX64\n", pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3673 GCPtr += 512 * _2M;
3674 cErrors++;
3675 continue;
3676 }
3677
3678# if PGM_GST_TYPE == PGM_TYPE_AMD64
3679 if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3680 || pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3681 || pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3682 {
3683 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3684 GCPtr += 512 * _2M;
3685 cErrors++;
3686 continue;
3687 }
3688# endif
3689
3690# else
3691 {
3692# endif
3693# if PGM_GST_TYPE == PGM_TYPE_32BIT
3694 const GSTPD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
3695# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3696 const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
3697# else
3698 const PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
3699# endif
3700# endif
3701 /*
3702 * Iterate the shadow page directory.
3703 */
3704 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3705 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3706
3707 for (;
3708 iPDDst < cPDEs;
3709 iPDDst++, GCPtr += ulIncrement)
3710 {
3711 const SHWPDE PdeDst = pPDDst->a[iPDDst];
3712 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3713 {
3714 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3715 if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3716 {
3717 AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3718 cErrors++;
3719 continue;
3720 }
3721 }
3722 else if ( (PdeDst.u & X86_PDE_P)
3723 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3724 )
3725 {
3726 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3727 PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3728 if (!pPoolPage)
3729 {
3730 AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
3731 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3732 cErrors++;
3733 continue;
3734 }
3735 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3736
3737 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
3738 {
3739 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
3740 GCPtr, (uint64_t)PdeDst.u));
3741 cErrors++;
3742 }
3743
3744 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
3745 {
3746 AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
3747 GCPtr, (uint64_t)PdeDst.u));
3748 cErrors++;
3749 }
3750
3751 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3752 if (!PdeSrc.n.u1Present)
3753 {
3754 AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3755 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3756 cErrors++;
3757 continue;
3758 }
3759
3760 if ( !PdeSrc.b.u1Size
3761 || !fBigPagesSupported)
3762 {
3763 GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3764# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3765 GCPhysGst |= (iPDDst & 1) * (PAGE_SIZE / 2);
3766# endif
3767 }
3768 else
3769 {
3770# if PGM_GST_TYPE == PGM_TYPE_32BIT
3771 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3772 {
3773 AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3774 GCPtr, (uint64_t)PdeSrc.u));
3775 cErrors++;
3776 continue;
3777 }
3778# endif
3779 GCPhysGst = PdeSrc.u & GST_PDE_BIG_PG_MASK;
3780# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3781 GCPhysGst |= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3782# endif
3783 }
3784
3785 if ( pPoolPage->enmKind
3786 != (!PdeSrc.b.u1Size || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3787 {
3788 AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
3789 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3790 cErrors++;
3791 }
3792
3793 PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3794 if (!pPhysPage)
3795 {
3796 AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3797 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3798 cErrors++;
3799 continue;
3800 }
3801
3802 if (GCPhysGst != pPoolPage->GCPhys)
3803 {
3804 AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
3805 GCPhysGst, pPoolPage->GCPhys, GCPtr));
3806 cErrors++;
3807 continue;
3808 }
3809
3810 if ( !PdeSrc.b.u1Size
3811 || !fBigPagesSupported)
3812 {
3813 /*
3814 * Page Table.
3815 */
3816 const GSTPT *pPTSrc;
3817 rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3818 if (VBOX_FAILURE(rc))
3819 {
3820 AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3821 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3822 cErrors++;
3823 continue;
3824 }
3825 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
3826 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
3827 {
3828 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3829 // (This problem will go away when/if we shadow multiple CR3s.)
3830 AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3831 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3832 cErrors++;
3833 continue;
3834 }
3835 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3836 {
3837 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
3838 GCPtr, (uint64_t)PdeDst.u));
3839 cErrors++;
3840 continue;
3841 }
3842
3843 /* iterate the page table. */
3844# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3845 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3846 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3847# else
3848 const unsigned offPTSrc = 0;
3849# endif
3850 for (unsigned iPT = 0, off = 0;
3851 iPT < ELEMENTS(pPTDst->a);
3852 iPT++, off += PAGE_SIZE)
3853 {
3854 const SHWPTE PteDst = pPTDst->a[iPT];
3855
3856 /* skip not-present entries. */
3857 if (!(PteDst.u & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3858 continue;
3859 Assert(PteDst.n.u1Present);
3860
3861 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3862 if (!PteSrc.n.u1Present)
3863 {
3864# ifdef IN_RING3
3865 PGMAssertHandlerAndFlagsInSync(pVM);
3866 PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3867# endif
3868 AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
3869 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3870 (PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3871 cErrors++;
3872 continue;
3873 }
3874
3875 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
3876# if 1 /** @todo sync accessed bit properly... */
3877 fIgnoreFlags |= X86_PTE_A;
3878# endif
3879
3880 /* match the physical addresses */
3881 HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3882 GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3883
3884# ifdef IN_RING3
3885 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3886 if (VBOX_FAILURE(rc))
3887 {
3888 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3889 {
3890 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3891 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3892 cErrors++;
3893 continue;
3894 }
3895 }
3896 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3897 {
3898 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3899 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3900 cErrors++;
3901 continue;
3902 }
3903# endif
3904
3905 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3906 if (!pPhysPage)
3907 {
3908# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3909 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3910 {
3911 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3912 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3913 cErrors++;
3914 continue;
3915 }
3916# endif
3917 if (PteDst.n.u1Write)
3918 {
3919 AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3920 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3921 cErrors++;
3922 }
3923 fIgnoreFlags |= X86_PTE_RW;
3924 }
3925 else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
3926 {
3927 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3928 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3929 cErrors++;
3930 continue;
3931 }
3932
3933 /* flags */
3934 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3935 {
3936 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3937 {
3938 if (PteDst.n.u1Write)
3939 {
3940 AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
3941 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3942 cErrors++;
3943 continue;
3944 }
3945 fIgnoreFlags |= X86_PTE_RW;
3946 }
3947 else
3948 {
3949 if (PteDst.n.u1Present)
3950 {
3951 AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
3952 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3953 cErrors++;
3954 continue;
3955 }
3956 fIgnoreFlags |= X86_PTE_P;
3957 }
3958 }
3959 else
3960 {
3961 if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3962 {
3963 if (PteDst.n.u1Write)
3964 {
3965 AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3966 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3967 cErrors++;
3968 continue;
3969 }
3970 if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3971 {
3972 AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3973 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3974 cErrors++;
3975 continue;
3976 }
3977 if (PteDst.n.u1Dirty)
3978 {
3979 AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3980 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3981 cErrors++;
3982 }
3983# if 0 /** @todo sync access bit properly... */
3984 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3985 {
3986 AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3987 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3988 cErrors++;
3989 }
3990 fIgnoreFlags |= X86_PTE_RW;
3991# else
3992 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
3993# endif
3994 }
3995 else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3996 {
3997 /* access bit emulation (not implemented). */
3998 if (PteSrc.n.u1Accessed || PteDst.n.u1Present)
3999 {
4000 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4001 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4002 cErrors++;
4003 continue;
4004 }
4005 if (!PteDst.n.u1Accessed)
4006 {
4007 AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4008 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4009 cErrors++;
4010 }
4011 fIgnoreFlags |= X86_PTE_P;
4012 }
4013# ifdef DEBUG_sandervl
4014 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4015# endif
4016 }
4017
4018 if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4019 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4020 )
4021 {
4022 AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4023 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4024 fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4025 cErrors++;
4026 continue;
4027 }
4028 } /* foreach PTE */
4029 }
4030 else
4031 {
4032 /*
4033 * Big Page.
4034 */
4035 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4036 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4037 {
4038 if (PdeDst.n.u1Write)
4039 {
4040 AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4041 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4042 cErrors++;
4043 continue;
4044 }
4045 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4046 {
4047 AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4048 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4049 cErrors++;
4050 continue;
4051 }
4052# if 0 /** @todo sync access bit properly... */
4053 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4054 {
4055 AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4056 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4057 cErrors++;
4058 }
4059 fIgnoreFlags |= X86_PTE_RW;
4060# else
4061 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4062# endif
4063 }
4064 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4065 {
4066 /* access bit emulation (not implemented). */
4067 if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
4068 {
4069 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4070 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4071 cErrors++;
4072 continue;
4073 }
4074 if (!PdeDst.n.u1Accessed)
4075 {
4076 AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4077 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4078 cErrors++;
4079 }
4080 fIgnoreFlags |= X86_PTE_P;
4081 }
4082
4083 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4084 {
4085 AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4086 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4087 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4088 cErrors++;
4089 }
4090
4091 /* iterate the page table. */
4092 for (unsigned iPT = 0, off = 0;
4093 iPT < ELEMENTS(pPTDst->a);
4094 iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4095 {
4096 const SHWPTE PteDst = pPTDst->a[iPT];
4097
4098 if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4099 {
4100 AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4101 GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4102 cErrors++;
4103 }
4104
4105 /* skip not-present entries. */
4106 if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4107 continue;
4108
4109 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G;
4110
4111 /* match the physical addresses */
4112 HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4113
4114# ifdef IN_RING3
4115 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4116 if (VBOX_FAILURE(rc))
4117 {
4118 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
4119 {
4120 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4121 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4122 cErrors++;
4123 }
4124 }
4125 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4126 {
4127 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4128 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4129 cErrors++;
4130 continue;
4131 }
4132# endif
4133 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
4134 if (!pPhysPage)
4135 {
4136# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4137 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
4138 {
4139 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4140 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4141 cErrors++;
4142 continue;
4143 }
4144# endif
4145 if (PteDst.n.u1Write)
4146 {
4147 AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4148 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4149 cErrors++;
4150 }
4151 fIgnoreFlags |= X86_PTE_RW;
4152 }
4153 else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
4154 {
4155 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4156 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4157 cErrors++;
4158 continue;
4159 }
4160
4161 /* flags */
4162 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4163 {
4164 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4165 {
4166 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4167 {
4168 if (PteDst.n.u1Write)
4169 {
4170 AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
4171 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4172 cErrors++;
4173 continue;
4174 }
4175 fIgnoreFlags |= X86_PTE_RW;
4176 }
4177 }
4178 else
4179 {
4180 if (PteDst.n.u1Present)
4181 {
4182 AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
4183 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4184 cErrors++;
4185 continue;
4186 }
4187 fIgnoreFlags |= X86_PTE_P;
4188 }
4189 }
4190
4191 if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4192 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4193 )
4194 {
4195 AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4196 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4197 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4198 cErrors++;
4199 continue;
4200 }
4201 } /* for each PTE */
4202 }
4203 }
4204 /* not present */
4205
4206 } /* for each PDE */
4207
4208 } /* for each PDPTE */
4209
4210 } /* for each PML4E */
4211
4212# ifdef DEBUG
4213 if (cErrors)
4214 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4215# endif
4216
4217#endif
4218 return cErrors;
4219
4220#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED */
4221}
4222#endif /* VBOX_STRICT */
4223
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette