VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 8911

Last change on this file since 8911 was 8739, checked in by vboxsync, 17 years ago

Set the shadow PDE for real and protected mode without paging to read/write & user.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 162.8 KB
Line 
1/* $Id: PGMAllBth.h 8739 2008-05-09 11:21:20Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * This file is a big challenge!
6 */
7
8/*
9 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
10 *
11 * This file is part of VirtualBox Open Source Edition (OSE), as
12 * available from http://www.virtualbox.org. This file is free software;
13 * you can redistribute it and/or modify it under the terms of the GNU
14 * General Public License (GPL) as published by the Free Software
15 * Foundation, in version 2 as it comes in the "COPYING" file of the
16 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20 * Clara, CA 95054 USA or visit http://www.sun.com if you need
21 * additional information or have any questions.
22 */
23
24/*******************************************************************************
25* Internal Functions *
26*******************************************************************************/
27__BEGIN_DECLS
28PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
33PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36#ifdef VBOX_STRICT
37PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38#endif
39#ifdef PGMPOOL_WITH_USER_TRACKING
40DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41#endif
42__END_DECLS
43
44
45/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
46#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE
47# error "Invalid combination; PAE guest implies PAE shadow"
48#endif
49
50#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
51 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE)
52# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
53#endif
54
55#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
56 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE)
57# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
58#endif
59
60#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64)
61 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64)
62# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
63#endif
64
65#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
66# define PGM_WITHOUT_MAPPINGS
67#endif
68
69/**
70 * #PF Handler for raw-mode guest execution.
71 *
72 * @returns VBox status code (appropriate for trap handling and GC return).
73 * @param pVM VM Handle.
74 * @param uErr The trap error code.
75 * @param pRegFrame Trap register frame.
76 * @param pvFault The fault address.
77 */
78PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
79{
80#if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
81
82# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
83 /*
84 * Hide the instruction fetch trap indicator for now.
85 */
86 /** @todo NXE will change this and we must fix NXE in the switcher too! */
87 if (uErr & X86_TRAP_PF_ID)
88 {
89 uErr &= ~X86_TRAP_PF_ID;
90 TRPMSetErrorCode(pVM, uErr);
91 }
92# endif
93
94 /*
95 * Get PDs.
96 */
97 int rc;
98# if PGM_WITH_PAGING(PGM_GST_TYPE)
99# if PGM_GST_TYPE == PGM_TYPE_32BIT
100 const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
101 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
102
103# elif PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
104
105# if PGM_GST_TYPE == PGM_TYPE_PAE
106 unsigned iPDSrc;
107 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc);
108
109# elif PGM_GST_TYPE == PGM_TYPE_AMD64
110 unsigned iPDSrc;
111 PX86PML4E pPml4e;
112 X86PDPE Pdpe;
113 PGSTPD pPDSrc;
114
115 pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, pvFault, &pPml4e, &Pdpe, &iPDSrc);
116 Assert(pPml4e);
117# endif
118 /* Quick check for a valid guest trap. */
119 if (!pPDSrc)
120 {
121 LogFlow(("Trap0eHandler: guest PDPTR not present CR3=%VGp\n", (CPUMGetGuestCR3(pVM) & X86_CR3_PAGE_MASK)));
122 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eGuestTrap; });
123 TRPMSetErrorCode(pVM, uErr);
124 return VINF_EM_RAW_GUEST_TRAP;
125 }
126# endif
127# else
128 PGSTPD pPDSrc = NULL;
129 const unsigned iPDSrc = 0;
130# endif
131
132# if PGM_SHW_TYPE == PGM_TYPE_32BIT
133 const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
134 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
135# elif PGM_SHW_TYPE == PGM_TYPE_PAE
136 const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
137 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
138
139# if PGM_GST_TYPE == PGM_TYPE_PAE
140 /* Did we mark the PDPT as not present in SyncCR3? */
141 unsigned iPDPTE = ((RTGCUINTPTR)pvFault >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
142 if (!pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
143 pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 1;
144
145# endif
146
147# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
148 const unsigned iPDDst = (((RTGCUINTPTR)pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
149 PX86PDPAE pPDDst;
150
151 rc = PGMShwGetLongModePDPtr(pVM, (RTGCUINTPTR)pvFault, &pPDDst);
152 if (rc != VINF_SUCCESS)
153 {
154 AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
155 return rc;
156 }
157 Assert(pPDDst);
158# endif
159
160# if PGM_WITH_PAGING(PGM_GST_TYPE)
161 /*
162 * If we successfully correct the write protection fault due to dirty bit
163 * tracking, or this page fault is a genuine one, then return immediately.
164 */
165 STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
166 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
167 STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
168 if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
169 || rc == VINF_EM_RAW_GUEST_TRAP)
170 {
171 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
172 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
173 LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
174 return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
175 }
176
177 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
178# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
179
180 /*
181 * A common case is the not-present error caused by lazy page table syncing.
182 *
183 * It is IMPORTANT that we weed out any access to non-present shadow PDEs here
184 * so we can safely assume that the shadow PT is present when calling SyncPage later.
185 *
186 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
187 * of mapping conflict and defer to SyncCR3 in R3.
188 * (Again, we do NOT support access handlers for non-present guest pages.)
189 *
190 */
191# if PGM_WITH_PAGING(PGM_GST_TYPE)
192 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
193# else
194 GSTPDE PdeSrc;
195 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
196 PdeSrc.n.u1Present = 1;
197 PdeSrc.n.u1Write = 1;
198 PdeSrc.n.u1Accessed = 1;
199 PdeSrc.n.u1User = 1;
200# endif
201 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
202 && !pPDDst->a[iPDDst].n.u1Present
203 && PdeSrc.n.u1Present
204 )
205
206 {
207 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
208 STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
209 LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
210 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
211 if (VBOX_SUCCESS(rc))
212 {
213 STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
214 return rc;
215 }
216 Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
217 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
218 STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
219 return VINF_PGM_SYNC_CR3;
220 }
221
222# if PGM_WITH_PAGING(PGM_GST_TYPE)
223 /*
224 * Check if this address is within any of our mappings.
225 *
226 * This is *very* fast and it's gonna save us a bit of effort below and prevent
227 * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
228 * (BTW, it's impossible to have physical access handlers in a mapping.)
229 */
230 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
231 {
232 STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
233 PPGMMAPPING pMapping = CTXALLSUFF(pVM->pgm.s.pMappings);
234 for ( ; pMapping; pMapping = CTXALLSUFF(pMapping->pNext))
235 {
236 if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
237 break;
238 if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
239 {
240 /*
241 * The first thing we check is if we've got an undetected conflict.
242 */
243 if (!pVM->pgm.s.fMappingsFixed)
244 {
245 unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
246 while (iPT-- > 0)
247 if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
248 {
249 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
250 Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
251 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
252 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
253 return VINF_PGM_SYNC_CR3;
254 }
255 }
256
257 /*
258 * Check if the fault address is in a virtual page access handler range.
259 */
260 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->HyperVirtHandlers, pvFault);
261 if ( pCur
262 && (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
263 && uErr & X86_TRAP_PF_RW)
264 {
265# ifdef IN_GC
266 STAM_PROFILE_START(&pCur->Stat, h);
267 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
268 STAM_PROFILE_STOP(&pCur->Stat, h);
269# else
270 AssertFailed();
271 rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
272# endif
273 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
274 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
275 return rc;
276 }
277
278 /*
279 * Pretend we're not here and let the guest handle the trap.
280 */
281 TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
282 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
283 LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
284 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
285 return VINF_EM_RAW_GUEST_TRAP;
286 }
287 }
288 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
289 } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
290# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
291
292 /*
293 * Check if this fault address is flagged for special treatment,
294 * which means we'll have to figure out the physical address and
295 * check flags associated with it.
296 *
297 * ASSUME that we can limit any special access handling to pages
298 * in page tables which the guest believes to be present.
299 */
300 if (PdeSrc.n.u1Present)
301 {
302 RTGCPHYS GCPhys = NIL_RTGCPHYS;
303
304# if PGM_WITH_PAGING(PGM_GST_TYPE)
305 uint32_t cr4 = CPUMGetGuestCR4(pVM);
306 if ( PdeSrc.b.u1Size
307 && (cr4 & X86_CR4_PSE))
308 GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK)
309 | ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
310 else
311 {
312 PGSTPT pPTSrc;
313 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
314 if (VBOX_SUCCESS(rc))
315 {
316 unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
317 if (pPTSrc->a[iPTESrc].n.u1Present)
318 GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
319 }
320 }
321# else
322 /* No paging so the fault address is the physical address */
323 GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
324# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
325
326 /*
327 * If we have a GC address we'll check if it has any flags set.
328 */
329 if (GCPhys != NIL_RTGCPHYS)
330 {
331 STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
332
333 PPGMPAGE pPage;
334 rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
335 if (VBOX_SUCCESS(rc))
336 {
337 if (PGM_PAGE_HAS_ANY_HANDLERS(pPage))
338 {
339 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
340 {
341 /*
342 * Physical page access handler.
343 */
344 const RTGCPHYS GCPhysFault = GCPhys | ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
345 PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
346 if (pCur)
347 {
348# ifdef PGM_SYNC_N_PAGES
349 /*
350 * If the region is write protected and we got a page not present fault, then sync
351 * the pages. If the fault was caused by a read, then restart the instruction.
352 * In case of write access continue to the GC write handler.
353 *
354 * ASSUMES that there is only one handler per page or that they have similar write properties.
355 */
356 if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
357 && !(uErr & X86_TRAP_PF_P))
358 {
359 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
360 if ( VBOX_FAILURE(rc)
361 || !(uErr & X86_TRAP_PF_RW)
362 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
363 {
364 AssertRC(rc);
365 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
366 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
367 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
368 return rc;
369 }
370 }
371# endif
372
373 AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
374 || (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
375 ("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
376
377#if defined(IN_GC) || defined(IN_RING0)
378 if (CTXALLSUFF(pCur->pfnHandler))
379 {
380 STAM_PROFILE_START(&pCur->Stat, h);
381 rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
382 STAM_PROFILE_STOP(&pCur->Stat, h);
383 }
384 else
385#endif
386 rc = VINF_EM_RAW_EMULATE_INSTR;
387 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
388 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
389 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
390 return rc;
391 }
392 }
393# if PGM_WITH_PAGING(PGM_GST_TYPE)
394 else
395 {
396# ifdef PGM_SYNC_N_PAGES
397 /*
398 * If the region is write protected and we got a page not present fault, then sync
399 * the pages. If the fault was caused by a read, then restart the instruction.
400 * In case of write access continue to the GC write handler.
401 */
402 if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
403 && !(uErr & X86_TRAP_PF_P))
404 {
405 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
406 if ( VBOX_FAILURE(rc)
407 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
408 || !(uErr & X86_TRAP_PF_RW))
409 {
410 AssertRC(rc);
411 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
412 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
413 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
414 return rc;
415 }
416 }
417# endif
418 /*
419 * Ok, it's an virtual page access handler.
420 *
421 * Since it's faster to search by address, we'll do that first
422 * and then retry by GCPhys if that fails.
423 */
424 /** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
425 /** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
426 * page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
427 */
428 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
429 if (pCur)
430 {
431 AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
432 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
433 || !(uErr & X86_TRAP_PF_P)
434 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
435 ("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
436
437 if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
438 && ( uErr & X86_TRAP_PF_RW
439 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
440 {
441# ifdef IN_GC
442 STAM_PROFILE_START(&pCur->Stat, h);
443 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
444 STAM_PROFILE_STOP(&pCur->Stat, h);
445# else
446 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
447# endif
448 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
449 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
450 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
451 return rc;
452 }
453 /* Unhandled part of a monitored page */
454 }
455 else
456 {
457 /* Check by physical address. */
458 PPGMVIRTHANDLER pCur;
459 unsigned iPage;
460 rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
461 &pCur, &iPage);
462 Assert(VBOX_SUCCESS(rc) || !pCur);
463 if ( pCur
464 && ( uErr & X86_TRAP_PF_RW
465 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
466 {
467 Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
468# ifdef IN_GC
469 RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->GCPtr & PAGE_OFFSET_MASK);
470 Assert(off < pCur->cb);
471 STAM_PROFILE_START(&pCur->Stat, h);
472 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
473 STAM_PROFILE_STOP(&pCur->Stat, h);
474# else
475 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
476# endif
477 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
478 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
479 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
480 return rc;
481 }
482 }
483 }
484# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
485
486 /*
487 * There is a handled area of the page, but this fault doesn't belong to it.
488 * We must emulate the instruction.
489 *
490 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
491 * we first check if this was a page-not-present fault for a page with only
492 * write access handlers. Restart the instruction if it wasn't a write access.
493 */
494 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
495
496 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
497 && !(uErr & X86_TRAP_PF_P))
498 {
499 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
500 if ( VBOX_FAILURE(rc)
501 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
502 || !(uErr & X86_TRAP_PF_RW))
503 {
504 AssertRC(rc);
505 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
506 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
507 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
508 return rc;
509 }
510 }
511
512 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
513 * It's writing to an unhandled part of the LDT page several million times.
514 */
515 rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
516 LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%RHp%s%s\n",
517 rc, pPage->HCPhys,
518 PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ? " phys" : "",
519 PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ? " virt" : ""));
520 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
521 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
522 return rc;
523 } /* if any kind of handler */
524
525# if PGM_WITH_PAGING(PGM_GST_TYPE)
526 if (uErr & X86_TRAP_PF_P)
527 {
528 /*
529 * The page isn't marked, but it might still be monitored by a virtual page access handler.
530 * (ASSUMES no temporary disabling of virtual handlers.)
531 */
532 /** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
533 * we should correct both the shadow page table and physical memory flags, and not only check for
534 * accesses within the handler region but for access to pages with virtual handlers. */
535 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
536 if (pCur)
537 {
538 AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
539 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
540 || !(uErr & X86_TRAP_PF_P)
541 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
542 ("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
543
544 if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
545 && ( uErr & X86_TRAP_PF_RW
546 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
547 {
548# ifdef IN_GC
549 STAM_PROFILE_START(&pCur->Stat, h);
550 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
551 STAM_PROFILE_STOP(&pCur->Stat, h);
552# else
553 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
554# endif
555 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
556 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
557 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
558 return rc;
559 }
560 }
561 }
562# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
563 }
564 else
565 {
566 /* When the guest accesses invalid physical memory (e.g. probing of RAM or accessing a remapped MMIO range), then we'll fall
567 * back to the recompiler to emulate the instruction.
568 */
569 LogFlow(("pgmPhysGetPageEx %VGp failed with %Vrc\n", GCPhys, rc));
570 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersInvalid);
571 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
572 return VINF_EM_RAW_EMULATE_INSTR;
573 }
574
575 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
576
577# ifdef PGM_OUT_OF_SYNC_IN_GC
578 /*
579 * We are here only if page is present in Guest page tables and trap is not handled
580 * by our handlers.
581 * Check it for page out-of-sync situation.
582 */
583 STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
584
585 if (!(uErr & X86_TRAP_PF_P))
586 {
587 /*
588 * Page is not present in our page tables.
589 * Try to sync it!
590 * BTW, fPageShw is invalid in this branch!
591 */
592 if (uErr & X86_TRAP_PF_US)
593 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
594 else /* supervisor */
595 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
596
597# if defined(LOG_ENABLED) && !defined(IN_RING0)
598 RTGCPHYS GCPhys;
599 uint64_t fPageGst;
600 PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
601 Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
602 pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
603# endif /* LOG_ENABLED */
604
605# if PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0)
606 if (CPUMGetGuestCPL(pVM, pRegFrame) == 0)
607 {
608 uint64_t fPageGst;
609 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
610 if ( VBOX_SUCCESS(rc)
611 && !(fPageGst & X86_PTE_US))
612 {
613 /* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
614 if ( pvFault == (RTGCPTR)pRegFrame->eip
615 || (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
616# ifdef CSAM_DETECT_NEW_CODE_PAGES
617 || ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
618 && CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
619# endif /* CSAM_DETECT_NEW_CODE_PAGES */
620 )
621 {
622 LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
623 rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
624 if (rc != VINF_SUCCESS)
625 {
626 /*
627 * CSAM needs to perform a job in ring 3.
628 *
629 * Sync the page before going to the host context; otherwise we'll end up in a loop if
630 * CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
631 */
632 LogFlow(("CSAM ring 3 job\n"));
633 int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
634 AssertRC(rc2);
635
636 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
637 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
638 return rc;
639 }
640 }
641# ifdef CSAM_DETECT_NEW_CODE_PAGES
642 else
643 if ( uErr == X86_TRAP_PF_RW
644 && pRegFrame->ecx >= 0x100 /* early check for movswd count */
645 && pRegFrame->ecx < 0x10000
646 )
647 {
648 /* In case of a write to a non-present supervisor shadow page, we'll take special precautions
649 * to detect loading of new code pages.
650 */
651
652 /*
653 * Decode the instruction.
654 */
655 RTGCPTR PC;
656 rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
657 if (rc == VINF_SUCCESS)
658 {
659 DISCPUSTATE Cpu;
660 uint32_t cbOp;
661 rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
662
663 /* For now we'll restrict this to rep movsw/d instructions */
664 if ( rc == VINF_SUCCESS
665 && Cpu.pCurInstr->opcode == OP_MOVSWD
666 && (Cpu.prefix & PREFIX_REP))
667 {
668 CSAMMarkPossibleCodePage(pVM, pvFault);
669 }
670 }
671 }
672# endif /* CSAM_DETECT_NEW_CODE_PAGES */
673
674 /*
675 * Mark this page as safe.
676 */
677 /** @todo not correct for pages that contain both code and data!! */
678 Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
679 CSAMMarkPage(pVM, pvFault, true);
680 }
681 }
682# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0) */
683 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
684 if (VBOX_SUCCESS(rc))
685 {
686 /* The page was successfully synced, return to the guest. */
687 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
688 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
689 return VINF_SUCCESS;
690 }
691 }
692 else
693 {
694 /*
695 * A side effect of not flushing global PDEs are out of sync pages due
696 * to physical monitored regions, that are no longer valid.
697 * Assume for now it only applies to the read/write flag
698 */
699 if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
700 {
701 if (uErr & X86_TRAP_PF_US)
702 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
703 else /* supervisor */
704 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
705
706
707 /*
708 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
709 */
710 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
711 if (VBOX_SUCCESS(rc))
712 {
713 /*
714 * Page was successfully synced, return to guest.
715 */
716# ifdef VBOX_STRICT
717 RTGCPHYS GCPhys;
718 uint64_t fPageGst;
719 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
720 Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
721 LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
722
723 uint64_t fPageShw;
724 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
725 AssertMsg(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW, ("rc=%Vrc fPageShw=%VX64\n", rc, fPageShw));
726# endif /* VBOX_STRICT */
727 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
728 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
729 return VINF_SUCCESS;
730 }
731
732 /* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
733 if ( CPUMGetGuestCPL(pVM, pRegFrame) == 0
734 && ((CPUMGetGuestCR0(pVM) & (X86_CR0_WP|X86_CR0_PG)) == X86_CR0_PG)
735 && (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P))
736 {
737 uint64_t fPageGst;
738 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
739 if ( VBOX_SUCCESS(rc)
740 && !(fPageGst & X86_PTE_RW))
741 {
742 rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
743 if (VBOX_SUCCESS(rc))
744 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulGC);
745 else
746 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulR3);
747 return rc;
748 }
749 else
750 AssertMsgFailed(("Unexpected r/w page %x flag=%x\n", pvFault, (uint32_t)fPageGst));
751 }
752
753 }
754
755# if PGM_WITH_PAGING(PGM_GST_TYPE)
756# ifdef VBOX_STRICT
757 /*
758 * Check for VMM page flags vs. Guest page flags consistency.
759 * Currently only for debug purposes.
760 */
761 if (VBOX_SUCCESS(rc))
762 {
763 /* Get guest page flags. */
764 uint64_t fPageGst;
765 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
766 if (VBOX_SUCCESS(rc))
767 {
768 uint64_t fPageShw;
769 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
770
771 /*
772 * Compare page flags.
773 * Note: we have AVL, A, D bits desynched.
774 */
775 AssertMsg((fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)),
776 ("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
777 }
778 else
779 AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
780 }
781 else
782 AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
783# endif /* VBOX_STRICT */
784# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
785 }
786 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
787# endif /* PGM_OUT_OF_SYNC_IN_GC */
788 }
789 else
790 {
791 /*
792 * Page not present in Guest OS or invalid page table address.
793 * This is potential virtual page access handler food.
794 *
795 * For the present we'll say that our access handlers don't
796 * work for this case - we've already discarded the page table
797 * not present case which is identical to this.
798 *
799 * When we perchance find we need this, we will probably have AVL
800 * trees (offset based) to operate on and we can measure their speed
801 * agains mapping a page table and probably rearrange this handling
802 * a bit. (Like, searching virtual ranges before checking the
803 * physical address.)
804 */
805 }
806 }
807
808
809# if PGM_WITH_PAGING(PGM_GST_TYPE)
810 /*
811 * Conclusion, this is a guest trap.
812 */
813 LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
814 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
815 return VINF_EM_RAW_GUEST_TRAP;
816# else
817 /* present, but not a monitored page; perhaps the guest is probing physical memory */
818 return VINF_EM_RAW_EMULATE_INSTR;
819# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
820
821
822#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
823
824 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
825 return VERR_INTERNAL_ERROR;
826#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
827}
828
829
830/**
831 * Emulation of the invlpg instruction.
832 *
833 *
834 * @returns VBox status code.
835 *
836 * @param pVM VM handle.
837 * @param GCPtrPage Page to invalidate.
838 *
839 * @remark ASSUMES that the guest is updating before invalidating. This order
840 * isn't required by the CPU, so this is speculative and could cause
841 * trouble.
842 *
843 * @todo Flush page or page directory only if necessary!
844 * @todo Add a #define for simply invalidating the page.
845 */
846PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
847{
848#if PGM_GST_TYPE == PGM_TYPE_32BIT \
849 || PGM_GST_TYPE == PGM_TYPE_PAE \
850 || PGM_GST_TYPE == PGM_TYPE_AMD64
851 int rc;
852
853 LogFlow(("InvalidatePage %x\n", GCPtrPage));
854 /*
855 * Get the shadow PD entry and skip out if this PD isn't present.
856 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
857 */
858 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
859# if PGM_SHW_TYPE == PGM_TYPE_32BIT
860 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
861# elif PGM_SHW_TYPE == PGM_TYPE_PAE
862 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
863# else /* AMD64 */
864 /* PML4 */
865 const unsigned iPml4 = ((RTGCUINTPTR64)GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
866 PX86PML4E pPml4eDst = &CTXMID(pVM->pgm.s.p,PaePML4)->a[iPml4];
867 if (!pPml4eDst->n.u1Present)
868 {
869 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
870 return VINF_SUCCESS;
871 }
872
873 /* PDPT */
874 PX86PDPT pPDPT;
875 rc = PGM_HCPHYS_2_PTR(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, &pPDPT);
876 if (VBOX_FAILURE(rc))
877 return rc;
878 const unsigned iPDPT = ((RTGCUINTPTR64)GCPtrPage >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
879 PX86PDPE pPdpeDst = &pPDPT->a[iPDPT];
880 if (!pPdpeDst->n.u1Present)
881 {
882 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
883 return VINF_SUCCESS;
884 }
885
886 /* PD */
887 PX86PDPAE pPd;
888 rc = PGM_HCPHYS_2_PTR(pVM, pPdpeDst->u & X86_PDPE_PG_MASK, &pPd);
889 if (VBOX_FAILURE(rc))
890 return rc;
891 const unsigned iPd = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
892 PX86PDEPAE pPdeDst = &pPd->a[iPd];
893# endif
894
895 const SHWPDE PdeDst = *pPdeDst;
896 if (!PdeDst.n.u1Present)
897 {
898 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
899 return VINF_SUCCESS;
900 }
901
902 /*
903 * Get the guest PD entry and calc big page.
904 */
905# if PGM_GST_TYPE == PGM_TYPE_32BIT
906 PX86PD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
907 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
908 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
909# else
910 unsigned iPDSrc;
911# if PGM_GST_TYPE == PGM_TYPE_PAE
912 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
913# else /* AMD64 */
914 PX86PML4E pPml4eSrc;
915 X86PDPE PdpeSrc;
916 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
917# endif
918 GSTPDE PdeSrc;
919
920 if (pPDSrc)
921 PdeSrc = pPDSrc->a[iPDSrc];
922 else
923 PdeSrc.u = 0;
924# endif
925
926 const uint32_t cr4 = CPUMGetGuestCR4(pVM);
927 const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
928
929# ifdef IN_RING3
930 /*
931 * If a CR3 Sync is pending we may ignore the invalidate page operation
932 * depending on the kind of sync and if it's a global page or not.
933 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
934 */
935# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
936 if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
937 || ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
938 && fIsBigPage
939 && PdeSrc.b.u1Global
940 && (cr4 & X86_CR4_PGE)
941 )
942 )
943# else
944 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
945# endif
946 {
947 STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
948 return VINF_SUCCESS;
949 }
950# endif /* IN_RING3 */
951
952
953# if PGM_GST_TYPE == PGM_TYPE_AMD64
954 Assert(pPml4eDst->n.u1Present && pPml4eDst->u & SHW_PDPT_MASK);
955 if (pPml4eSrc->n.u1Present)
956 {
957 if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
958 || (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
959 {
960 /*
961 * Mark not present so we can resync the PML4E when it's used.
962 */
963 LogFlow(("InvalidatePage: Out-of-sync PML4E at %VGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
964 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
965 pgmPoolFree(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, PGMPOOL_IDX_PML4, iPml4);
966 pPml4eDst->u = 0;
967 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
968 PGM_INVL_GUEST_TLBS();
969 }
970 else if (!pPml4eSrc->n.u1Accessed)
971 {
972 /*
973 * Mark not present so we can set the accessed bit.
974 */
975 pgmPoolFree(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, PGMPOOL_IDX_PML4, iPml4);
976 pPml4eDst->u = 0;
977 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
978 PGM_INVL_GUEST_TLBS();
979 }
980 }
981 else
982 {
983 pgmPoolFree(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, PGMPOOL_IDX_PML4, iPml4);
984 pPml4eDst->u = 0;
985 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
986 PGM_INVL_PG(GCPtrPage);
987 return VINF_SUCCESS;
988 }
989
990 Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
991 if (PdpeSrc.n.u1Present)
992 {
993 if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
994 || (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
995 {
996 /*
997 * Mark not present so we can resync the PML4E when it's used.
998 */
999 LogFlow(("InvalidatePage: Out-of-sync PDPE at %VGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1000 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1001 pgmPoolFree(pVM, pPdpeDst->u & SHW_PDPT_MASK, PGMPOOL_IDX_PML4, iPml4);
1002 pPdpeDst->u = 0;
1003 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1004 PGM_INVL_GUEST_TLBS();
1005 }
1006 else if (!PdpeSrc.lm.u1Accessed)
1007 {
1008 /*
1009 * Mark not present so we can set the accessed bit.
1010 */
1011 pgmPoolFree(pVM, pPdpeDst->u & SHW_PDPT_MASK, PGMPOOL_IDX_PML4, iPml4);
1012 pPdpeDst->u = 0;
1013 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
1014 PGM_INVL_GUEST_TLBS();
1015 }
1016 }
1017 else
1018 {
1019 pgmPoolFree(pVM, pPdpeDst->u & SHW_PDPT_MASK, PGMPOOL_IDX_PDPT, iPDDst);
1020 pPdpeDst->u = 0;
1021 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1022 PGM_INVL_PG(GCPtrPage);
1023 return VINF_SUCCESS;
1024 }
1025# endif
1026
1027 /*
1028 * Deal with the Guest PDE.
1029 */
1030 rc = VINF_SUCCESS;
1031 if (PdeSrc.n.u1Present)
1032 {
1033 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1034 {
1035 /*
1036 * Conflict - Let SyncPT deal with it to avoid duplicate code.
1037 */
1038 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1039 Assert(PGMGetGuestMode(pVM) <= PGMMODE_32_BIT);
1040 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
1041 }
1042 else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1043 || (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1044 {
1045 /*
1046 * Mark not present so we can resync the PDE when it's used.
1047 */
1048 LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1049 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1050 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1051 pPdeDst->u = 0;
1052 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1053 PGM_INVL_GUEST_TLBS();
1054 }
1055 else if (!PdeSrc.n.u1Accessed)
1056 {
1057 /*
1058 * Mark not present so we can set the accessed bit.
1059 */
1060 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1061 pPdeDst->u = 0;
1062 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
1063 PGM_INVL_GUEST_TLBS();
1064 }
1065 else if (!fIsBigPage)
1066 {
1067 /*
1068 * 4KB - page.
1069 */
1070 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1071 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1072# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1073 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1074 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1075# endif
1076 if (pShwPage->GCPhys == GCPhys)
1077 {
1078# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1079 const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1080 PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1081 if (pPT->a[iPTEDst].n.u1Present)
1082 {
1083# ifdef PGMPOOL_WITH_USER_TRACKING
1084 /* This is very unlikely with caching/monitoring enabled. */
1085 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1086# endif
1087 pPT->a[iPTEDst].u = 0;
1088 }
1089# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1090 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
1091 if (VBOX_SUCCESS(rc))
1092 rc = VINF_SUCCESS;
1093# endif
1094 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
1095 PGM_INVL_PG(GCPtrPage);
1096 }
1097 else
1098 {
1099 /*
1100 * The page table address changed.
1101 */
1102 LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
1103 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1104 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1105 pPdeDst->u = 0;
1106 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1107 PGM_INVL_GUEST_TLBS();
1108 }
1109 }
1110 else
1111 {
1112 /*
1113 * 2/4MB - page.
1114 */
1115 /* Before freeing the page, check if anything really changed. */
1116 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1117 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1118# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1119 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1120 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1121# endif
1122 if ( pShwPage->GCPhys == GCPhys
1123 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1124 {
1125 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1126 /** @todo PAT */
1127 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1128 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1129 && ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1130 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1131 {
1132 LogFlow(("Skipping flush for big page containing %VGv (PD=%X .u=%VX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1133 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
1134 return VINF_SUCCESS;
1135 }
1136 }
1137
1138 /*
1139 * Ok, the page table is present and it's been changed in the guest.
1140 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1141 * We could do this for some flushes in GC too, but we need an algorithm for
1142 * deciding which 4MB pages containing code likely to be executed very soon.
1143 */
1144 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1145 pPdeDst->u = 0;
1146 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
1147 PGM_INVL_BIG_PG(GCPtrPage);
1148 }
1149 }
1150 else
1151 {
1152 /*
1153 * Page directory is not present, mark shadow PDE not present.
1154 */
1155 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1156 {
1157 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1158 pPdeDst->u = 0;
1159 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1160 PGM_INVL_PG(GCPtrPage);
1161 }
1162 else
1163 {
1164 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1165 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
1166 }
1167 }
1168
1169 return rc;
1170
1171#else /* guest real and protected mode */
1172 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1173 return VINF_SUCCESS;
1174#endif
1175}
1176
1177
1178#ifdef PGMPOOL_WITH_USER_TRACKING
1179/**
1180 * Update the tracking of shadowed pages.
1181 *
1182 * @param pVM The VM handle.
1183 * @param pShwPage The shadow page.
1184 * @param HCPhys The physical page we is being dereferenced.
1185 */
1186DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1187{
1188# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1189 STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1190 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1191
1192 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1193 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1194 * 2. write protect all shadowed pages. I.e. implement caching.
1195 */
1196 /*
1197 * Find the guest address.
1198 */
1199 for (PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
1200 pRam;
1201 pRam = CTXALLSUFF(pRam->pNext))
1202 {
1203 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1204 while (iPage-- > 0)
1205 {
1206 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1207 {
1208 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1209 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1210 pShwPage->cPresent--;
1211 pPool->cPresent--;
1212 STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1213 return;
1214 }
1215 }
1216 }
1217
1218 for (;;)
1219 AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1220# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1221 pShwPage->cPresent--;
1222 pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1223# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1224}
1225
1226
1227/**
1228 * Update the tracking of shadowed pages.
1229 *
1230 * @param pVM The VM handle.
1231 * @param pShwPage The shadow page.
1232 * @param u16 The top 16-bit of the pPage->HCPhys.
1233 * @param pPage Pointer to the guest page. this will be modified.
1234 * @param iPTDst The index into the shadow table.
1235 */
1236DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1237{
1238# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1239 /*
1240 * We're making certain assumptions about the placement of cRef and idx.
1241 */
1242 Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1243 Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1244
1245 /*
1246 * Just deal with the simple first time here.
1247 */
1248 if (!u16)
1249 {
1250 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1251 u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) | pShwPage->idx;
1252 }
1253 else
1254 u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1255
1256 /* write back, trying to be clever... */
1257 Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%VHp->%VHp iPTDst=%#x\n",
1258 u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) | ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1259 *((uint16_t *)&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1260# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1261
1262 /* update statistics. */
1263 pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1264 pShwPage->cPresent++;
1265 if (pShwPage->iFirstPresent > iPTDst)
1266 pShwPage->iFirstPresent = iPTDst;
1267}
1268#endif /* PGMPOOL_WITH_USER_TRACKING */
1269
1270
1271/**
1272 * Creates a 4K shadow page for a guest page.
1273 *
1274 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1275 * physical address. The PdeSrc argument only the flags are used. No page structured
1276 * will be mapped in this function.
1277 *
1278 * @param pVM VM handle.
1279 * @param pPteDst Destination page table entry.
1280 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1281 * Can safely assume that only the flags are being used.
1282 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1283 * @param pShwPage Pointer to the shadow page.
1284 * @param iPTDst The index into the shadow table.
1285 *
1286 * @remark Not used for 2/4MB pages!
1287 */
1288DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1289{
1290 if (PteSrc.n.u1Present)
1291 {
1292 /*
1293 * Find the ram range.
1294 */
1295 PPGMPAGE pPage;
1296 int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1297 if (VBOX_SUCCESS(rc))
1298 {
1299 /** @todo investiage PWT, PCD and PAT. */
1300 /*
1301 * Make page table entry.
1302 */
1303 const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1304 SHWPTE PteDst;
1305 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1306 {
1307 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1308 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1309 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1310 | (HCPhys & X86_PTE_PAE_PG_MASK);
1311 else
1312 {
1313 LogFlow(("SyncPageWorker: monitored page (%VGp) -> mark not present\n", HCPhys));
1314 PteDst.u = 0;
1315 }
1316 /** @todo count these two kinds. */
1317 }
1318 else
1319 {
1320 /*
1321 * If the page or page directory entry is not marked accessed,
1322 * we mark the page not present.
1323 */
1324 if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
1325 {
1326 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1327 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1328 PteDst.u = 0;
1329 }
1330 else
1331 /*
1332 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1333 * when the page is modified.
1334 */
1335 if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1336 {
1337 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1338 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1339 | (HCPhys & X86_PTE_PAE_PG_MASK)
1340 | PGM_PTFLAGS_TRACK_DIRTY;
1341 }
1342 else
1343 {
1344 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1345 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1346 | (HCPhys & X86_PTE_PAE_PG_MASK);
1347 }
1348 }
1349
1350#ifdef PGMPOOL_WITH_USER_TRACKING
1351 /*
1352 * Keep user track up to date.
1353 */
1354 if (PteDst.n.u1Present)
1355 {
1356 if (!pPteDst->n.u1Present)
1357 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1358 else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1359 {
1360 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1361 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1362 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1363 }
1364 }
1365 else if (pPteDst->n.u1Present)
1366 {
1367 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1368 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1369 }
1370#endif /* PGMPOOL_WITH_USER_TRACKING */
1371
1372 /*
1373 * Update statistics and commit the entry.
1374 */
1375 if (!PteSrc.n.u1Global)
1376 pShwPage->fSeenNonGlobal = true;
1377 *pPteDst = PteDst;
1378 }
1379 /* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1380 /** @todo count these. */
1381 }
1382 else
1383 {
1384 /*
1385 * Page not-present.
1386 */
1387 LogFlow(("SyncPageWorker: page not present in Pte\n"));
1388#ifdef PGMPOOL_WITH_USER_TRACKING
1389 /* Keep user track up to date. */
1390 if (pPteDst->n.u1Present)
1391 {
1392 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1393 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1394 }
1395#endif /* PGMPOOL_WITH_USER_TRACKING */
1396 pPteDst->u = 0;
1397 /** @todo count these. */
1398 }
1399}
1400
1401
1402/**
1403 * Syncs a guest OS page.
1404 *
1405 * There are no conflicts at this point, neither is there any need for
1406 * page table allocations.
1407 *
1408 * @returns VBox status code.
1409 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1410 * @param pVM VM handle.
1411 * @param PdeSrc Page directory entry of the guest.
1412 * @param GCPtrPage Guest context page address.
1413 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1414 * @param uErr Fault error (X86_TRAP_PF_*).
1415 */
1416PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1417{
1418 LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1419
1420#if PGM_GST_TYPE == PGM_TYPE_32BIT \
1421 || PGM_GST_TYPE == PGM_TYPE_PAE
1422
1423# if PGM_WITH_NX(PGM_GST_TYPE)
1424 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1425# endif
1426
1427 /*
1428 * Assert preconditions.
1429 */
1430 STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1431 Assert(PdeSrc.n.u1Present);
1432 Assert(cPages);
1433
1434 /*
1435 * Get the shadow PDE, find the shadow page table in the pool.
1436 */
1437 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1438# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1439 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1440# else /* PAE */
1441 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1442# endif
1443 Assert(PdeDst.n.u1Present);
1444 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1445
1446 /*
1447 * Check that the page is present and that the shadow PDE isn't out of sync.
1448 */
1449 const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1450 RTGCPHYS GCPhys;
1451 if (!fBigPage)
1452 {
1453 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1454# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1455 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1456 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1457# endif
1458 }
1459 else
1460 {
1461 GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1462# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1463 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1464 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1465# endif
1466 }
1467 if ( pShwPage->GCPhys == GCPhys
1468 && PdeSrc.n.u1Present
1469 && (PdeSrc.n.u1User == PdeDst.n.u1User)
1470 && (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
1471# if PGM_WITH_NX(PGM_GST_TYPE)
1472 && (!fNoExecuteBitValid || PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1473# endif
1474 )
1475 {
1476 /*
1477 * Check that the PDE is marked accessed already.
1478 * Since we set the accessed bit *before* getting here on a #PF, this
1479 * check is only meant for dealing with non-#PF'ing paths.
1480 */
1481 if (PdeSrc.n.u1Accessed)
1482 {
1483 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1484 if (!fBigPage)
1485 {
1486 /*
1487 * 4KB Page - Map the guest page table.
1488 */
1489 PGSTPT pPTSrc;
1490 int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1491 if (VBOX_SUCCESS(rc))
1492 {
1493# ifdef PGM_SYNC_N_PAGES
1494 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1495 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1496 {
1497 /*
1498 * This code path is currently only taken when the caller is PGMTrap0eHandler
1499 * for non-present pages!
1500 *
1501 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1502 * deal with locality.
1503 */
1504 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1505# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1506 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1507 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1508# else
1509 const unsigned offPTSrc = 0;
1510# endif
1511 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1512 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1513 iPTDst = 0;
1514 else
1515 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1516 for (; iPTDst < iPTDstEnd; iPTDst++)
1517 {
1518 if (!pPTDst->a[iPTDst].n.u1Present)
1519 {
1520 GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1521 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1522 NOREF(GCPtrCurPage);
1523#ifndef IN_RING0
1524 /*
1525 * Assuming kernel code will be marked as supervisor - and not as user level
1526 * and executed using a conforming code selector - And marked as readonly.
1527 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
1528 */
1529 PPGMPAGE pPage;
1530 if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW | X86_PTE_US))
1531 || iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1532 || !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1533 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1534 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1535 )
1536#endif /* else: CSAM not active */
1537 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1538 Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1539 GCPtrCurPage, PteSrc.n.u1Present,
1540 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1541 PteSrc.n.u1User & PdeSrc.n.u1User,
1542 (uint64_t)PteSrc.u,
1543 (uint64_t)pPTDst->a[iPTDst].u,
1544 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1545 }
1546 }
1547 }
1548 else
1549# endif /* PGM_SYNC_N_PAGES */
1550 {
1551 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1552 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1553 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1554 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1555 Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1556 GCPtrPage, PteSrc.n.u1Present,
1557 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1558 PteSrc.n.u1User & PdeSrc.n.u1User,
1559 (uint64_t)PteSrc.u,
1560 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1561 }
1562 }
1563 else /* MMIO or invalid page: emulated in #PF handler. */
1564 {
1565 LogFlow(("PGM_GCPHYS_2_PTR %VGp failed with %Vrc\n", GCPhys, rc));
1566 Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1567 }
1568 }
1569 else
1570 {
1571 /*
1572 * 4/2MB page - lazy syncing shadow 4K pages.
1573 * (There are many causes of getting here, it's no longer only CSAM.)
1574 */
1575 /* Calculate the GC physical address of this 4KB shadow page. */
1576 RTGCPHYS GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK) | ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1577 /* Find ram range. */
1578 PPGMPAGE pPage;
1579 int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1580 if (VBOX_SUCCESS(rc))
1581 {
1582 /*
1583 * Make shadow PTE entry.
1584 */
1585 const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1586 SHWPTE PteDst;
1587 PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1588 | (HCPhys & X86_PTE_PAE_PG_MASK);
1589 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1590 {
1591 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1592 PteDst.n.u1Write = 0;
1593 else
1594 PteDst.u = 0;
1595 }
1596 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1597# ifdef PGMPOOL_WITH_USER_TRACKING
1598 if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1599 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1600# endif
1601 pPTDst->a[iPTDst] = PteDst;
1602
1603
1604 /*
1605 * If the page is not flagged as dirty and is writable, then make it read-only
1606 * at PD level, so we can set the dirty bit when the page is modified.
1607 *
1608 * ASSUMES that page access handlers are implemented on page table entry level.
1609 * Thus we will first catch the dirty access and set PDE.D and restart. If
1610 * there is an access handler, we'll trap again and let it work on the problem.
1611 */
1612 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1613 * As for invlpg, it simply frees the whole shadow PT.
1614 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1615 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1616 {
1617 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1618 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
1619 PdeDst.n.u1Write = 0;
1620 }
1621 else
1622 {
1623 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1624 PdeDst.n.u1Write = PdeSrc.n.u1Write;
1625 }
1626# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1627 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1628# else /* PAE */
1629 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1630# endif
1631 Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1632 GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1633 PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1634 }
1635 else
1636 LogFlow(("PGM_GCPHYS_2_PTR %VGp (big) failed with %Vrc\n", GCPhys, rc));
1637 }
1638 return VINF_SUCCESS;
1639 }
1640 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1641 }
1642 else
1643 {
1644 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1645 Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1646 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1647 }
1648
1649 /*
1650 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1651 * Yea, I'm lazy.
1652 */
1653 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1654# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1655 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1656# else /* PAE */
1657 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1658# endif
1659 PGM_INVL_GUEST_TLBS();
1660 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1661
1662#elif PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT
1663
1664# ifdef PGM_SYNC_N_PAGES
1665 /*
1666 * Get the shadow PDE, find the shadow page table in the pool.
1667 */
1668 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1669# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1670 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1671# else /* PAE */
1672 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1673# endif
1674 Assert(PdeDst.n.u1Present);
1675 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1676 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1677
1678# if PGM_SHW_TYPE == PGM_TYPE_PAE
1679 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1680 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1681# else
1682 const unsigned offPTSrc = 0;
1683# endif
1684
1685 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1686 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1687 {
1688 /*
1689 * This code path is currently only taken when the caller is PGMTrap0eHandler
1690 * for non-present pages!
1691 *
1692 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1693 * deal with locality.
1694 */
1695 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1696 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1697 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1698 iPTDst = 0;
1699 else
1700 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1701 for (; iPTDst < iPTDstEnd; iPTDst++)
1702 {
1703 if (!pPTDst->a[iPTDst].n.u1Present)
1704 {
1705 GSTPTE PteSrc;
1706
1707 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1708
1709 /* Fake the page table entry */
1710 PteSrc.u = GCPtrCurPage;
1711 PteSrc.n.u1Present = 1;
1712 PteSrc.n.u1Dirty = 1;
1713 PteSrc.n.u1Accessed = 1;
1714 PteSrc.n.u1Write = 1;
1715 PteSrc.n.u1User = 1;
1716
1717 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1718
1719 Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1720 GCPtrCurPage, PteSrc.n.u1Present,
1721 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1722 PteSrc.n.u1User & PdeSrc.n.u1User,
1723 (uint64_t)PteSrc.u,
1724 (uint64_t)pPTDst->a[iPTDst].u,
1725 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1726 }
1727 }
1728 }
1729 else
1730# endif /* PGM_SYNC_N_PAGES */
1731 {
1732 GSTPTE PteSrc;
1733 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1734 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1735
1736 /* Fake the page table entry */
1737 PteSrc.u = GCPtrCurPage;
1738 PteSrc.n.u1Present = 1;
1739 PteSrc.n.u1Dirty = 1;
1740 PteSrc.n.u1Accessed = 1;
1741 PteSrc.n.u1Write = 1;
1742 PteSrc.n.u1User = 1;
1743 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1744
1745 Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
1746 GCPtrPage, PteSrc.n.u1Present,
1747 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1748 PteSrc.n.u1User & PdeSrc.n.u1User,
1749 (uint64_t)PteSrc.u,
1750 (uint64_t)pPTDst->a[iPTDst].u,
1751 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1752 }
1753 return VINF_SUCCESS;
1754
1755#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1756 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1757 return VERR_INTERNAL_ERROR;
1758#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1759}
1760
1761
1762
1763#if PGM_WITH_PAGING(PGM_GST_TYPE)
1764
1765/**
1766 * Investigate page fault and handle write protection page faults caused by
1767 * dirty bit tracking.
1768 *
1769 * @returns VBox status code.
1770 * @param pVM VM handle.
1771 * @param uErr Page fault error code.
1772 * @param pPdeDst Shadow page directory entry.
1773 * @param pPdeSrc Guest page directory entry.
1774 * @param GCPtrPage Guest context page address.
1775 */
1776PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1777{
1778 bool fWriteProtect = !!(CPUMGetGuestCR0(pVM) & X86_CR0_WP);
1779 bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
1780 bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
1781 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1782# if PGM_WITH_NX(PGM_GST_TYPE)
1783 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1784# endif
1785 unsigned uPageFaultLevel;
1786 int rc;
1787
1788 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1789 LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1790
1791# if PGM_GST_TYPE == PGM_TYPE_PAE \
1792 || PGM_GST_TYPE == PGM_TYPE_AMD64
1793
1794# if PGM_GST_TYPE == PGM_TYPE_AMD64
1795 PX86PML4E pPml4eSrc;
1796 PX86PDPE pPdpeSrc;
1797
1798 pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc);
1799 Assert(pPml4eSrc);
1800
1801 /*
1802 * Real page fault? (PML4E level)
1803 */
1804 if ( (uErr & X86_TRAP_PF_RSVD)
1805 || !pPml4eSrc->n.u1Present
1806 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
1807 || (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
1808 || (fUserLevelFault && !pPml4eSrc->n.u1User)
1809 )
1810 {
1811 uPageFaultLevel = 0;
1812 goto UpperLevelPageFault;
1813 }
1814 Assert(pPdpeSrc);
1815
1816# else /* PAE */
1817 PX86PDPE pPdpeSrc = &pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtrPage >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
1818# endif
1819
1820 /*
1821 * Real page fault? (PDPE level)
1822 */
1823 if ( (uErr & X86_TRAP_PF_RSVD)
1824 || !pPdpeSrc->n.u1Present
1825# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
1826 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
1827 || (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault || fWriteProtect))
1828 || (fUserLevelFault && !pPdpeSrc->lm.u1User)
1829# endif
1830 )
1831 {
1832 uPageFaultLevel = 1;
1833 goto UpperLevelPageFault;
1834 }
1835# endif
1836
1837 /*
1838 * Real page fault? (PDE level)
1839 */
1840 if ( (uErr & X86_TRAP_PF_RSVD)
1841 || !pPdeSrc->n.u1Present
1842# if PGM_WITH_NX(PGM_GST_TYPE)
1843 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
1844# endif
1845 || (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
1846 || (fUserLevelFault && !pPdeSrc->n.u1User) )
1847 {
1848 uPageFaultLevel = 2;
1849 goto UpperLevelPageFault;
1850 }
1851
1852 /*
1853 * First check the easy case where the page directory has been marked read-only to track
1854 * the dirty bit of an emulated BIG page
1855 */
1856 if (pPdeSrc->b.u1Size && fBigPagesSupported)
1857 {
1858 /* Mark guest page directory as accessed */
1859# if PGM_GST_TYPE == PGM_TYPE_AMD64
1860 pPml4eSrc->n.u1Accessed = 1;
1861 pPdpeSrc->lm.u1Accessed = 1;
1862# endif
1863 pPdeSrc->b.u1Accessed = 1;
1864
1865 /*
1866 * Only write protection page faults are relevant here.
1867 */
1868 if (fWriteFault)
1869 {
1870 /* Mark guest page directory as dirty (BIG page only). */
1871 pPdeSrc->b.u1Dirty = 1;
1872
1873 if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1874 {
1875 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1876
1877 Assert(pPdeSrc->b.u1Write);
1878
1879 pPdeDst->n.u1Write = 1;
1880 pPdeDst->n.u1Accessed = 1;
1881 pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1882 PGM_INVL_BIG_PG(GCPtrPage);
1883 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1884 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1885 }
1886 }
1887 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1888 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1889 }
1890 /* else: 4KB page table */
1891
1892 /*
1893 * Map the guest page table.
1894 */
1895 PGSTPT pPTSrc;
1896 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1897 if (VBOX_SUCCESS(rc))
1898 {
1899 /*
1900 * Real page fault?
1901 */
1902 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
1903 const GSTPTE PteSrc = *pPteSrc;
1904 if ( !PteSrc.n.u1Present
1905# if PGM_WITH_NX(PGM_GST_TYPE)
1906 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
1907# endif
1908 || (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault || fWriteProtect))
1909 || (fUserLevelFault && !PteSrc.n.u1User)
1910 )
1911 {
1912# ifdef IN_GC
1913 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1914# endif
1915 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1916 LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1917
1918 /* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1919 * See the 2nd case above as well.
1920 */
1921 if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1922 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
1923
1924 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1925 return VINF_EM_RAW_GUEST_TRAP;
1926 }
1927 LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1928
1929 /*
1930 * Set the accessed bits in the page directory and the page table.
1931 */
1932# if PGM_GST_TYPE == PGM_TYPE_AMD64
1933 pPml4eSrc->n.u1Accessed = 1;
1934 pPdpeSrc->lm.u1Accessed = 1;
1935# endif
1936 pPdeSrc->n.u1Accessed = 1;
1937 pPteSrc->n.u1Accessed = 1;
1938
1939 /*
1940 * Only write protection page faults are relevant here.
1941 */
1942 if (fWriteFault)
1943 {
1944 /* Write access, so mark guest entry as dirty. */
1945# if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1946 if (!pPteSrc->n.u1Dirty)
1947 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1948 else
1949 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1950# endif
1951
1952 pPteSrc->n.u1Dirty = 1;
1953
1954 if (pPdeDst->n.u1Present)
1955 {
1956 /* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1957 * Our individual shadow handlers will provide more information and force a fatal exit.
1958 */
1959 if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1960 {
1961 LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1962 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1963 return VINF_SUCCESS;
1964 }
1965
1966 /*
1967 * Map shadow page table.
1968 */
1969 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1970 if (pShwPage)
1971 {
1972 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1973 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1974 if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1975 && (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1976 {
1977 LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1978# ifdef VBOX_STRICT
1979 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
1980 if (pPage)
1981 AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
1982 ("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1983# endif
1984 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1985
1986 Assert(pPteSrc->n.u1Write);
1987
1988 pPteDst->n.u1Write = 1;
1989 pPteDst->n.u1Dirty = 1;
1990 pPteDst->n.u1Accessed = 1;
1991 pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1992 PGM_INVL_PG(GCPtrPage);
1993
1994 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1995 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1996 }
1997 }
1998 else
1999 AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2000 }
2001 }
2002/** @todo Optimize accessed bit emulation? */
2003# ifdef VBOX_STRICT
2004 /*
2005 * Sanity check.
2006 */
2007 else if ( !pPteSrc->n.u1Dirty
2008 && (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2009 && pPdeDst->n.u1Present)
2010 {
2011 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2012 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2013 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2014 if ( pPteDst->n.u1Present
2015 && pPteDst->n.u1Write)
2016 LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2017 }
2018# endif /* VBOX_STRICT */
2019 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2020 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2021 }
2022 AssertRC(rc);
2023 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2024 return rc;
2025
2026
2027UpperLevelPageFault:
2028 /* Pagefault detected while checking the PML4E, PDPE or PDE.
2029 * Single exit handler to get rid of duplicate code paths.
2030 */
2031# ifdef IN_GC
2032 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
2033# endif
2034 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
2035 LogFlow(("CheckPageFault: real page fault at %VGv (%d)\n", GCPtrPage, uPageFaultLevel));
2036
2037 if (
2038# if PGM_GST_TYPE == PGM_TYPE_AMD64
2039 pPml4eSrc->n.u1Present &&
2040# endif
2041# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
2042 pPdpeSrc->n.u1Present &&
2043# endif
2044 pPdeSrc->n.u1Present)
2045 {
2046 /* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2047 if (pPdeSrc->b.u1Size && fBigPagesSupported)
2048 {
2049 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2050 }
2051 else
2052 {
2053 /*
2054 * Map the guest page table.
2055 */
2056 PGSTPT pPTSrc;
2057 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2058 if (VBOX_SUCCESS(rc))
2059 {
2060 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2061 const GSTPTE PteSrc = *pPteSrc;
2062 if (pPteSrc->n.u1Present)
2063 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2064 }
2065 AssertRC(rc);
2066 }
2067 }
2068 return VINF_EM_RAW_GUEST_TRAP;
2069}
2070
2071#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2072
2073
2074/**
2075 * Sync a shadow page table.
2076 *
2077 * The shadow page table is not present. This includes the case where
2078 * there is a conflict with a mapping.
2079 *
2080 * @returns VBox status code.
2081 * @param pVM VM handle.
2082 * @param iPD Page directory index.
2083 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
2084 * Assume this is a temporary mapping.
2085 * @param GCPtrPage GC Pointer of the page that caused the fault
2086 */
2087PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
2088{
2089 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2090 STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
2091 LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
2092
2093#if PGM_GST_TYPE == PGM_TYPE_32BIT \
2094 || PGM_GST_TYPE == PGM_TYPE_PAE
2095
2096 /*
2097 * Validate input a little bit.
2098 */
2099 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%VGv\n", iPDSrc, GCPtrPage));
2100# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2101 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2102# else
2103 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2104# endif
2105 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2106 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2107 SHWPDE PdeDst = *pPdeDst;
2108
2109# ifndef PGM_WITHOUT_MAPPINGS
2110 /*
2111 * Check for conflicts.
2112 * GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2113 * HC: Simply resolve the conflict.
2114 */
2115 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2116 {
2117 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2118# ifndef IN_RING3
2119 Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
2120 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2121 return VERR_ADDRESS_CONFLICT;
2122# else
2123 PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2124 Assert(pMapping);
2125# if PGM_GST_TYPE == PGM_TYPE_32BIT
2126 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2127# elif PGM_GST_TYPE == PGM_TYPE_PAE
2128 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2129# endif
2130 if (VBOX_FAILURE(rc))
2131 {
2132 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2133 return rc;
2134 }
2135 PdeDst = *pPdeDst;
2136# endif
2137 }
2138# else /* PGM_WITHOUT_MAPPINGS */
2139 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2140# endif /* PGM_WITHOUT_MAPPINGS */
2141 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2142
2143 /*
2144 * Sync page directory entry.
2145 */
2146 int rc = VINF_SUCCESS;
2147 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2148 if (PdeSrc.n.u1Present)
2149 {
2150 /*
2151 * Allocate & map the page table.
2152 */
2153 PSHWPT pPTDst;
2154 const bool fPageTable = !PdeSrc.b.u1Size || !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
2155 PPGMPOOLPAGE pShwPage;
2156 RTGCPHYS GCPhys;
2157 if (fPageTable)
2158 {
2159 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2160# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2161 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2162 GCPhys |= (iPDDst & 1) * (PAGE_SIZE / 2);
2163# endif
2164 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2165 }
2166 else
2167 {
2168 GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2169# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2170 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2171 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2172# endif
2173 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2174 }
2175 if (rc == VINF_SUCCESS)
2176 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2177 else if (rc == VINF_PGM_CACHED_PAGE)
2178 {
2179 /*
2180 * The PT was cached, just hook it up.
2181 */
2182 if (fPageTable)
2183 PdeDst.u = pShwPage->Core.Key
2184 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2185 else
2186 {
2187 PdeDst.u = pShwPage->Core.Key
2188 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2189 /* (see explanation and assumptions further down.) */
2190 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2191 {
2192 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2193 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2194 PdeDst.b.u1Write = 0;
2195 }
2196 }
2197 *pPdeDst = PdeDst;
2198 return VINF_SUCCESS;
2199 }
2200 else if (rc == VERR_PGM_POOL_FLUSHED)
2201 return VINF_PGM_SYNC_CR3;
2202 else
2203 AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2204 PdeDst.u &= X86_PDE_AVL_MASK;
2205 PdeDst.u |= pShwPage->Core.Key;
2206
2207 /*
2208 * Page directory has been accessed (this is a fault situation, remember).
2209 */
2210 pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2211 if (fPageTable)
2212 {
2213 /*
2214 * Page table - 4KB.
2215 *
2216 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2217 */
2218 Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2219 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2220 PGSTPT pPTSrc;
2221 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2222 if (VBOX_SUCCESS(rc))
2223 {
2224 /*
2225 * Start by syncing the page directory entry so CSAM's TLB trick works.
2226 */
2227 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
2228 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2229 *pPdeDst = PdeDst;
2230
2231 /*
2232 * Directory/page user or supervisor privilege: (same goes for read/write)
2233 *
2234 * Directory Page Combined
2235 * U/S U/S U/S
2236 * 0 0 0
2237 * 0 1 0
2238 * 1 0 0
2239 * 1 1 1
2240 *
2241 * Simple AND operation. Table listed for completeness.
2242 *
2243 */
2244 STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
2245# ifdef PGM_SYNC_N_PAGES
2246 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2247 unsigned iPTDst = iPTBase;
2248 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
2249 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2250 iPTDst = 0;
2251 else
2252 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2253# else /* !PGM_SYNC_N_PAGES */
2254 unsigned iPTDst = 0;
2255 const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
2256# endif /* !PGM_SYNC_N_PAGES */
2257# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2258 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2259 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2260# else
2261 const unsigned offPTSrc = 0;
2262# endif
2263 for (; iPTDst < iPTDstEnd; iPTDst++)
2264 {
2265 const unsigned iPTSrc = iPTDst + offPTSrc;
2266 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2267
2268 if (PteSrc.n.u1Present) /* we've already cleared it above */
2269 {
2270# ifndef IN_RING0
2271 /*
2272 * Assuming kernel code will be marked as supervisor - and not as user level
2273 * and executed using a conforming code selector - And marked as readonly.
2274 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
2275 */
2276 PPGMPAGE pPage;
2277 if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW | X86_PTE_US))
2278 || !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)))
2279 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2280 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2281 )
2282# endif
2283 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2284 Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
2285 (RTGCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)),
2286 PteSrc.n.u1Present,
2287 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2288 PteSrc.n.u1User & PdeSrc.n.u1User,
2289 (uint64_t)PteSrc.u,
2290 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2291 (PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
2292 }
2293 } /* for PTEs */
2294 }
2295 }
2296 else
2297 {
2298 /*
2299 * Big page - 2/4MB.
2300 *
2301 * We'll walk the ram range list in parallel and optimize lookups.
2302 * We will only sync on shadow page table at a time.
2303 */
2304 STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
2305
2306 /**
2307 * @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2308 */
2309
2310 /*
2311 * Start by syncing the page directory entry.
2312 */
2313 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2314 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2315
2316 /*
2317 * If the page is not flagged as dirty and is writable, then make it read-only
2318 * at PD level, so we can set the dirty bit when the page is modified.
2319 *
2320 * ASSUMES that page access handlers are implemented on page table entry level.
2321 * Thus we will first catch the dirty access and set PDE.D and restart. If
2322 * there is an access handler, we'll trap again and let it work on the problem.
2323 */
2324 /** @todo move the above stuff to a section in the PGM documentation. */
2325 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2326 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2327 {
2328 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2329 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2330 PdeDst.b.u1Write = 0;
2331 }
2332 *pPdeDst = PdeDst;
2333
2334 /*
2335 * Fill the shadow page table.
2336 */
2337 /* Get address and flags from the source PDE. */
2338 SHWPTE PteDstBase;
2339 PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT);
2340
2341 /* Loop thru the entries in the shadow PT. */
2342 const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2343 Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2344 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2345 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2346 PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
2347 unsigned iPTDst = 0;
2348 while (iPTDst < ELEMENTS(pPTDst->a))
2349 {
2350 /* Advance ram range list. */
2351 while (pRam && GCPhys > pRam->GCPhysLast)
2352 pRam = CTXALLSUFF(pRam->pNext);
2353 if (pRam && GCPhys >= pRam->GCPhys)
2354 {
2355 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2356 do
2357 {
2358 /* Make shadow PTE. */
2359 PPGMPAGE pPage = &pRam->aPages[iHCPage];
2360 SHWPTE PteDst;
2361
2362 /* Make sure the RAM has already been allocated. */
2363 if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2364 {
2365 if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2366 {
2367# ifdef IN_RING3
2368 int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2369# else
2370 int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2371# endif
2372 if (rc != VINF_SUCCESS)
2373 return rc;
2374 }
2375 }
2376
2377 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2378 {
2379 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2380 {
2381 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2382 PteDst.n.u1Write = 0;
2383 }
2384 else
2385 PteDst.u = 0;
2386 }
2387# ifndef IN_RING0
2388 /*
2389 * Assuming kernel code will be marked as supervisor and not as user level and executed
2390 * using a conforming code selector. Don't check for readonly, as that implies the whole
2391 * 4MB can be code or readonly data. Linux enables write access for its large pages.
2392 */
2393 else if ( !PdeSrc.n.u1User
2394 && CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))))
2395 PteDst.u = 0;
2396# endif
2397 else
2398 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2399# ifdef PGMPOOL_WITH_USER_TRACKING
2400 if (PteDst.n.u1Present)
2401 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2402# endif
2403 /* commit it */
2404 pPTDst->a[iPTDst] = PteDst;
2405 Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2406 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2407 PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2408
2409 /* advance */
2410 GCPhys += PAGE_SIZE;
2411 iHCPage++;
2412 iPTDst++;
2413 } while ( iPTDst < ELEMENTS(pPTDst->a)
2414 && GCPhys <= pRam->GCPhysLast);
2415 }
2416 else if (pRam)
2417 {
2418 Log(("Invalid pages at %VGp\n", GCPhys));
2419 do
2420 {
2421 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2422 GCPhys += PAGE_SIZE;
2423 iPTDst++;
2424 } while ( iPTDst < ELEMENTS(pPTDst->a)
2425 && GCPhys < pRam->GCPhys);
2426 }
2427 else
2428 {
2429 Log(("Invalid pages at %VGp (2)\n", GCPhys));
2430 for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2431 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2432 }
2433 } /* while more PTEs */
2434 } /* 4KB / 4MB */
2435 }
2436 else
2437 AssertRelease(!PdeDst.n.u1Present);
2438
2439 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2440# ifdef IN_GC
2441 if (VBOX_FAILURE(rc))
2442 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2443# endif
2444 return rc;
2445
2446#elif PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT
2447
2448 int rc = VINF_SUCCESS;
2449
2450 /*
2451 * Validate input a little bit.
2452 */
2453# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2454 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2455# else
2456 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2457# endif
2458 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2459 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2460 SHWPDE PdeDst = *pPdeDst;
2461
2462 Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2463 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2464
2465 GSTPDE PdeSrc;
2466 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2467 PdeSrc.n.u1Present = 1;
2468 PdeSrc.n.u1Write = 1;
2469 PdeSrc.n.u1Accessed = 1;
2470 PdeSrc.n.u1User = 1;
2471
2472 /*
2473 * Allocate & map the page table.
2474 */
2475 PSHWPT pPTDst;
2476 PPGMPOOLPAGE pShwPage;
2477 RTGCPHYS GCPhys;
2478
2479 /* Virtual address = physical address */
2480 GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK_32;
2481 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2482
2483 if ( rc == VINF_SUCCESS
2484 || rc == VINF_PGM_CACHED_PAGE)
2485 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2486 else
2487 AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2488
2489 PdeDst.u &= X86_PDE_AVL_MASK;
2490 PdeDst.u |= pShwPage->Core.Key;
2491 PdeDst.n.u1Present = 1;
2492 PdeDst.n.u1Write = 1;
2493 PdeDst.n.u1User = 1;
2494 *pPdeDst = PdeDst;
2495
2496 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2497 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2498 return rc;
2499
2500#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2501 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2502 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2503 return VERR_INTERNAL_ERROR;
2504#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2505}
2506
2507
2508
2509/**
2510 * Prefetch a page/set of pages.
2511 *
2512 * Typically used to sync commonly used pages before entering raw mode
2513 * after a CR3 reload.
2514 *
2515 * @returns VBox status code.
2516 * @param pVM VM handle.
2517 * @param GCPtrPage Page to invalidate.
2518 */
2519PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2520{
2521#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2522 /*
2523 * Check that all Guest levels thru the PDE are present, getting the
2524 * PD and PDE in the processes.
2525 */
2526 int rc = VINF_SUCCESS;
2527# if PGM_WITH_PAGING(PGM_GST_TYPE)
2528# if PGM_GST_TYPE == PGM_TYPE_32BIT
2529 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2530 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2531# else /* PAE */
2532 unsigned iPDSrc;
2533 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2534 if (!pPDSrc)
2535 return VINF_SUCCESS; /* not present */
2536# endif
2537 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2538# else
2539 PGSTPD pPDSrc = NULL;
2540 const unsigned iPDSrc = 0;
2541 GSTPDE PdeSrc;
2542
2543 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2544 PdeSrc.n.u1Present = 1;
2545 PdeSrc.n.u1Write = 1;
2546 PdeSrc.n.u1Accessed = 1;
2547 PdeSrc.n.u1User = 1;
2548# endif
2549
2550 if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2551 {
2552# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2553 const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2554# else
2555 const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2556# endif
2557 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2558 {
2559 if (!PdeDst.n.u1Present)
2560 /** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2561 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2562 else
2563 {
2564 /** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2565 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2566 * makes no sense to prefetch more than one page.
2567 */
2568 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2569 if (VBOX_SUCCESS(rc))
2570 rc = VINF_SUCCESS;
2571 }
2572 }
2573 }
2574 return rc;
2575
2576#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2577
2578 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2579 return VERR_INTERNAL_ERROR;
2580#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2581}
2582
2583
2584
2585
2586/**
2587 * Syncs a page during a PGMVerifyAccess() call.
2588 *
2589 * @returns VBox status code (informational included).
2590 * @param GCPtrPage The address of the page to sync.
2591 * @param fPage The effective guest page flags.
2592 * @param uErr The trap error code.
2593 */
2594PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2595{
2596 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2597
2598#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2599
2600# ifndef IN_RING0
2601 if (!(fPage & X86_PTE_US))
2602 {
2603 /*
2604 * Mark this page as safe.
2605 */
2606 /** @todo not correct for pages that contain both code and data!! */
2607 Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2608 CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2609 }
2610# endif
2611 /*
2612 * Get guest PD and index.
2613 */
2614
2615# if PGM_WITH_PAGING(PGM_GST_TYPE)
2616# if PGM_GST_TYPE == PGM_TYPE_32BIT
2617 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2618 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2619# else /* PAE */
2620 unsigned iPDSrc;
2621 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2622
2623 if (pPDSrc)
2624 {
2625 Log(("PGMVerifyAccess: access violation for %VGv due to non-present PDPTR\n", GCPtrPage));
2626 return VINF_EM_RAW_GUEST_TRAP;
2627 }
2628# endif
2629# else
2630 PGSTPD pPDSrc = NULL;
2631 const unsigned iPDSrc = 0;
2632# endif
2633 int rc = VINF_SUCCESS;
2634
2635 /*
2636 * First check if the shadow pd is present.
2637 */
2638# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2639 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2640# else
2641 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2642# endif
2643 if (!pPdeDst->n.u1Present)
2644 {
2645 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2646 AssertRC(rc);
2647 if (rc != VINF_SUCCESS)
2648 return rc;
2649 }
2650
2651# if PGM_WITH_PAGING(PGM_GST_TYPE)
2652 /* Check for dirty bit fault */
2653 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2654 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2655 Log(("PGMVerifyAccess: success (dirty)\n"));
2656 else
2657 {
2658 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2659#else
2660 {
2661 GSTPDE PdeSrc;
2662 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2663 PdeSrc.n.u1Present = 1;
2664 PdeSrc.n.u1Write = 1;
2665 PdeSrc.n.u1Accessed = 1;
2666 PdeSrc.n.u1User = 1;
2667
2668#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2669 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2670 if (uErr & X86_TRAP_PF_US)
2671 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2672 else /* supervisor */
2673 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2674
2675 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2676 if (VBOX_SUCCESS(rc))
2677 {
2678 /* Page was successfully synced */
2679 Log2(("PGMVerifyAccess: success (sync)\n"));
2680 rc = VINF_SUCCESS;
2681 }
2682 else
2683 {
2684 Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2685 return VINF_EM_RAW_GUEST_TRAP;
2686 }
2687 }
2688 return rc;
2689
2690#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2691
2692 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2693 return VERR_INTERNAL_ERROR;
2694#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2695}
2696
2697
2698#if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE
2699# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE
2700/**
2701 * Figures out which kind of shadow page this guest PDE warrants.
2702 *
2703 * @returns Shadow page kind.
2704 * @param pPdeSrc The guest PDE in question.
2705 * @param cr4 The current guest cr4 value.
2706 */
2707DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
2708{
2709 if (!pPdeSrc->n.u1Size || !(cr4 & X86_CR4_PSE))
2710 return BTH_PGMPOOLKIND_PT_FOR_PT;
2711 //switch (pPdeSrc->u & (X86_PDE4M_RW | X86_PDE4M_US /*| X86_PDE4M_PAE_NX*/))
2712 //{
2713 // case 0:
2714 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2715 // case X86_PDE4M_RW:
2716 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2717 // case X86_PDE4M_US:
2718 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2719 // case X86_PDE4M_RW | X86_PDE4M_US:
2720 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2721# if 0
2722 // case X86_PDE4M_PAE_NX:
2723 // return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2724 // case X86_PDE4M_RW | X86_PDE4M_PAE_NX:
2725 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2726 // case X86_PDE4M_US | X86_PDE4M_PAE_NX:
2727 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2728 // case X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PAE_NX:
2729 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2730# endif
2731 return BTH_PGMPOOLKIND_PT_FOR_BIG;
2732 //}
2733}
2734# endif
2735#endif
2736
2737#undef MY_STAM_COUNTER_INC
2738#define MY_STAM_COUNTER_INC(a) do { } while (0)
2739
2740
2741/**
2742 * Syncs the paging hierarchy starting at CR3.
2743 *
2744 * @returns VBox status code, no specials.
2745 * @param pVM The virtual machine.
2746 * @param cr0 Guest context CR0 register
2747 * @param cr3 Guest context CR3 register
2748 * @param cr4 Guest context CR4 register
2749 * @param fGlobal Including global page directories or not
2750 */
2751PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2752{
2753 if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2754 fGlobal = true; /* Change this CR3 reload to be a global one. */
2755
2756 /*
2757 * Update page access handlers.
2758 * The virtual are always flushed, while the physical are only on demand.
2759 * WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2760 * have to look into that later because it will have a bad influence on the performance.
2761 * @note SvL: There's no need for that. Just invalidate the virtual range(s).
2762 * bird: Yes, but that won't work for aliases.
2763 */
2764 /** @todo this MUST go away. See #1557. */
2765 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2766 PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2767 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2768
2769#ifdef PGMPOOL_WITH_MONITORING
2770 /*
2771 * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2772 * Occationally we will have to clear all the shadow page tables because we wanted
2773 * to monitor a page which was mapped by too many shadowed page tables. This operation
2774 * sometimes refered to as a 'lightweight flush'.
2775 */
2776 if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2777 pgmPoolMonitorModifiedClearAll(pVM);
2778 else
2779 {
2780# ifdef IN_RING3
2781 pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2782 pgmPoolClearAll(pVM);
2783# else
2784 LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2785 return VINF_PGM_SYNC_CR3;
2786# endif
2787 }
2788#endif
2789
2790 Assert(fGlobal || (cr4 & X86_CR4_PGE));
2791 MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2792
2793#if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE
2794 /*
2795 * Get page directory addresses.
2796 */
2797# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2798 PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2799# else /* PGM_SHW_TYPE == PGM_TYPE_PAE */
2800# if PGM_GST_TYPE == PGM_TYPE_32BIT
2801 PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2802# endif
2803# endif
2804
2805# if PGM_GST_TYPE == PGM_TYPE_32BIT
2806 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2807 Assert(pPDSrc);
2808# ifndef IN_GC
2809 Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & GST_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2810# endif
2811# endif
2812
2813 /*
2814 * Iterate the page directory.
2815 */
2816 PPGMMAPPING pMapping;
2817 unsigned iPdNoMapping;
2818 const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2819 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2820
2821 /* Only check mappings if they are supposed to be put into the shadow page table. */
2822 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2823 {
2824 pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2825 iPdNoMapping = (pMapping) ? (pMapping->GCPtr >> GST_PD_SHIFT) : ~0U;
2826 }
2827 else
2828 {
2829 pMapping = 0;
2830 iPdNoMapping = ~0U;
2831 }
2832# if PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
2833 for (unsigned iPDPTE = 0; iPDPTE < GST_PDPE_ENTRIES; iPDPTE++)
2834 {
2835 unsigned iPDSrc;
2836# if PGM_SHW_TYPE == PGM_TYPE_PAE
2837 PX86PDPAE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2838# else
2839 AssertFailed(); /* @todo */
2840 PX86PDPE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[iPDPTE * X86_PG_AMD64_ENTRIES];
2841# endif
2842 PX86PDEPAE pPDEDst = &pPDPAE->a[iPDPTE * X86_PG_PAE_ENTRIES];
2843 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, iPDPTE << X86_PDPT_SHIFT, &iPDSrc);
2844
2845 if (pPDSrc == NULL)
2846 {
2847 /* PDPT not present */
2848 if (pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
2849 {
2850 for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2851 {
2852 if ( pPDEDst[iPD].n.u1Present
2853 && !(pPDEDst[iPD].u & PGM_PDFLAGS_MAPPING))
2854 {
2855 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst[iPD].u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPDPTE * X86_PG_PAE_ENTRIES + iPD);
2856 pPDEDst[iPD].u = 0;
2857 }
2858 }
2859 }
2860 if (!(pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].u & PGM_PLXFLAGS_MAPPING))
2861 pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 0;
2862 continue;
2863 }
2864# else /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2865 {
2866# endif /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2867 for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2868 {
2869# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2870 Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2871# elif PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2872 AssertMsg(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst, ("%p vs %p\n", &pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512], pPDEDst));
2873# endif
2874 register GSTPDE PdeSrc = pPDSrc->a[iPD];
2875 if ( PdeSrc.n.u1Present
2876 && (PdeSrc.n.u1User || fRawR0Enabled))
2877 {
2878# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2879 || PGM_GST_TYPE == PGM_TYPE_PAE) \
2880 && !defined(PGM_WITHOUT_MAPPINGS)
2881
2882 /*
2883 * Check for conflicts with GC mappings.
2884 */
2885# if PGM_GST_TYPE == PGM_TYPE_PAE
2886 if (iPD + iPDPTE * X86_PG_PAE_ENTRIES == iPdNoMapping)
2887# else
2888 if (iPD == iPdNoMapping)
2889# endif
2890 {
2891 if (pVM->pgm.s.fMappingsFixed)
2892 {
2893 /* It's fixed, just skip the mapping. */
2894 const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
2895 iPD += cPTs - 1;
2896 pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
2897 pMapping = pMapping->CTXALLSUFF(pNext);
2898 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2899 continue;
2900 }
2901# ifdef IN_RING3
2902# if PGM_GST_TYPE == PGM_TYPE_32BIT
2903 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
2904# elif PGM_GST_TYPE == PGM_TYPE_PAE
2905 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
2906# endif
2907 if (VBOX_FAILURE(rc))
2908 return rc;
2909
2910 /*
2911 * Update iPdNoMapping and pMapping.
2912 */
2913 pMapping = pVM->pgm.s.pMappingsR3;
2914 while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
2915 pMapping = pMapping->pNextR3;
2916 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2917# else
2918 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2919 return VINF_PGM_SYNC_CR3;
2920# endif
2921 }
2922# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
2923 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2924# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
2925 /*
2926 * Sync page directory entry.
2927 *
2928 * The current approach is to allocated the page table but to set
2929 * the entry to not-present and postpone the page table synching till
2930 * it's actually used.
2931 */
2932# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2933 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2934# elif PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
2935 const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
2936# else
2937 const unsigned iPdShw = iPD; NOREF(iPdShw);
2938# endif
2939 {
2940 SHWPDE PdeDst = *pPDEDst;
2941 if (PdeDst.n.u1Present)
2942 {
2943 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2944 RTGCPHYS GCPhys;
2945 if ( !PdeSrc.b.u1Size
2946 || !(cr4 & X86_CR4_PSE))
2947 {
2948 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2949# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2950 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2951 GCPhys |= i * (PAGE_SIZE / 2);
2952# endif
2953 }
2954 else
2955 {
2956 GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2957# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2958 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2959 GCPhys |= i * X86_PAGE_2M_SIZE;
2960# endif
2961 }
2962
2963 if ( pShwPage->GCPhys == GCPhys
2964 && pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2965 && ( pShwPage->fCached
2966 || ( !fGlobal
2967 && ( false
2968# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2969 || ( (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
2970 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE)) /* global 2/4MB page. */
2971 || ( !pShwPage->fSeenNonGlobal
2972 && (cr4 & X86_CR4_PGE))
2973# endif
2974 )
2975 )
2976 )
2977 && ( (PdeSrc.u & (X86_PDE_US | X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US | X86_PDE_RW))
2978 || ( (cr4 & X86_CR4_PSE)
2979 && ((PdeSrc.u & (X86_PDE_US | X86_PDE4M_PS | X86_PDE4M_D)) | PGM_PDFLAGS_TRACK_DIRTY)
2980 == ((PdeDst.u & (X86_PDE_US | X86_PDE_RW | PGM_PDFLAGS_TRACK_DIRTY)) | X86_PDE4M_PS))
2981 )
2982 )
2983 {
2984# ifdef VBOX_WITH_STATISTICS
2985 if ( !fGlobal
2986 && (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
2987 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE))
2988 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2989 else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2990 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2991 else
2992 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2993# endif /* VBOX_WITH_STATISTICS */
2994 /** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2995 * The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2996 //# ifdef PGMPOOL_WITH_CACHE
2997 // pgmPoolCacheUsed(pPool, pShwPage);
2998 //# endif
2999 }
3000 else
3001 {
3002 pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
3003 pPDEDst->u = 0;
3004 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
3005 }
3006 }
3007 else
3008 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
3009 pPDEDst++;
3010 }
3011 }
3012# if PGM_GST_TYPE == PGM_TYPE_PAE
3013 else if (iPD + iPDPTE * X86_PG_PAE_ENTRIES != iPdNoMapping)
3014# else
3015 else if (iPD != iPdNoMapping)
3016# endif
3017 {
3018 /*
3019 * Check if there is any page directory to mark not present here.
3020 */
3021# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3022 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
3023# elif PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3024 const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
3025# else
3026 const unsigned iPdShw = iPD; NOREF(iPdShw);
3027# endif
3028 {
3029 if (pPDEDst->n.u1Present)
3030 {
3031 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
3032 pPDEDst->u = 0;
3033 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
3034 }
3035 pPDEDst++;
3036 }
3037 }
3038 else
3039 {
3040# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3041 || PGM_GST_TYPE == PGM_TYPE_PAE) \
3042 && !defined(PGM_WITHOUT_MAPPINGS)
3043
3044 const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
3045
3046 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3047 if (pVM->pgm.s.fMappingsFixed)
3048 {
3049 /* It's fixed, just skip the mapping. */
3050 pMapping = pMapping->CTXALLSUFF(pNext);
3051 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3052 }
3053 else
3054 {
3055 /*
3056 * Check for conflicts for subsequent pagetables
3057 * and advance to the next mapping.
3058 */
3059 iPdNoMapping = ~0U;
3060 unsigned iPT = cPTs;
3061 while (iPT-- > 1)
3062 {
3063 if ( pPDSrc->a[iPD + iPT].n.u1Present
3064 && (pPDSrc->a[iPD + iPT].n.u1User || fRawR0Enabled))
3065 {
3066# ifdef IN_RING3
3067# if PGM_GST_TYPE == PGM_TYPE_32BIT
3068 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
3069# elif PGM_GST_TYPE == PGM_TYPE_PAE
3070 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
3071# endif
3072 if (VBOX_FAILURE(rc))
3073 return rc;
3074
3075 /*
3076 * Update iPdNoMapping and pMapping.
3077 */
3078 pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
3079 while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
3080 pMapping = pMapping->CTXALLSUFF(pNext);
3081 iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3082 break;
3083# else
3084 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3085 return VINF_PGM_SYNC_CR3;
3086# endif
3087 }
3088 }
3089 if (iPdNoMapping == ~0U && pMapping)
3090 {
3091 pMapping = pMapping->CTXALLSUFF(pNext);
3092 if (pMapping)
3093 iPdNoMapping = pMapping->GCPtr >> GST_PD_SHIFT;
3094 }
3095 }
3096
3097 /* advance. */
3098 iPD += cPTs - 1;
3099 pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3100# if PGM_GST_TYPE != PGM_SHW_TYPE
3101 AssertCompile(PGM_GST_TYPE == PGM_TYPE_32BIT && PGM_SHW_TYPE == PGM_TYPE_PAE);
3102# endif
3103# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3104 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3105# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) || PGM_WITHOUT_MAPPINGS */
3106 }
3107
3108 } /* for iPD */
3109 } /* for each PDPTE (PAE) */
3110
3111 return VINF_SUCCESS;
3112
3113#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3114//# error not implemented
3115 return VERR_INTERNAL_ERROR;
3116#else /* guest real and protected mode */
3117 return VINF_SUCCESS;
3118#endif
3119}
3120
3121
3122
3123
3124#ifdef VBOX_STRICT
3125#ifdef IN_GC
3126# undef AssertMsgFailed
3127# define AssertMsgFailed Log
3128#endif
3129#ifdef IN_RING3
3130# include <VBox/dbgf.h>
3131
3132/**
3133 * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3134 *
3135 * @returns VBox status code (VINF_SUCCESS).
3136 * @param pVM The VM handle.
3137 * @param cr3 The root of the hierarchy.
3138 * @param crr The cr4, only PAE and PSE is currently used.
3139 * @param fLongMode Set if long mode, false if not long mode.
3140 * @param cMaxDepth Number of levels to dump.
3141 * @param pHlp Pointer to the output functions.
3142 */
3143__BEGIN_DECLS
3144PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3145__END_DECLS
3146
3147#endif
3148
3149/**
3150 * Checks that the shadow page table is in sync with the guest one.
3151 *
3152 * @returns The number of errors.
3153 * @param pVM The virtual machine.
3154 * @param cr3 Guest context CR3 register
3155 * @param cr4 Guest context CR4 register
3156 * @param GCPtr Where to start. Defaults to 0.
3157 * @param cb How much to check. Defaults to everything.
3158 */
3159PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
3160{
3161 unsigned cErrors = 0;
3162
3163#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3164 || PGM_GST_TYPE == PGM_TYPE_PAE
3165
3166 PPGM pPGM = &pVM->pgm.s;
3167 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3168 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3169# ifndef IN_RING0
3170 RTHCPHYS HCPhys; /* general usage. */
3171# endif
3172 int rc;
3173
3174 /*
3175 * Check that the Guest CR3 and all its mappings are correct.
3176 */
3177 AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3178 ("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3179 false);
3180# ifndef IN_RING0
3181# if PGM_GST_TYPE == PGM_TYPE_32BIT
3182 rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
3183# else
3184 rc = PGMShwGetPage(pVM, pPGM->pGstPaePDPTGC, NULL, &HCPhysShw);
3185# endif
3186 AssertRCReturn(rc, 1);
3187 HCPhys = NIL_RTHCPHYS;
3188 rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3189 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
3190# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3191 RTGCPHYS GCPhys;
3192 rc = PGMR3DbgHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
3193 AssertRCReturn(rc, 1);
3194 AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
3195# endif
3196#endif /* !IN_RING0 */
3197
3198# if PGM_GST_TYPE == PGM_TYPE_32BIT
3199 const GSTPD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
3200# endif
3201
3202 /*
3203 * Get and check the Shadow CR3.
3204 */
3205# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3206 const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
3207 unsigned cPDEs = ELEMENTS(pPDDst->a);
3208# else
3209 const X86PDPAE *pPDDst = pPGM->CTXMID(ap,PaePDs[0]); /* use it as a 2048 entry PD */
3210 unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
3211# endif
3212 if (cb != ~(RTGCUINTPTR)0)
3213 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3214
3215/** @todo call the other two PGMAssert*() functions. */
3216
3217# if PGM_GST_TYPE == PGM_TYPE_PAE
3218 /*
3219 * Check the 4 PDPTs too.
3220 */
3221 for (unsigned i = 0; i < 4; i++)
3222 {
3223 RTHCPTR HCPtr;
3224 RTHCPHYS HCPhys;
3225 RTGCPHYS GCPhys = pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[i].u & X86_PDPE_PG_MASK;
3226 int rc2 = pgmRamGCPhys2HCPtrAndHCPhysWithFlags(&pVM->pgm.s, GCPhys, &HCPtr, &HCPhys);
3227 if (VBOX_SUCCESS(rc2))
3228 {
3229 AssertMsg( pVM->pgm.s.apGstPaePDsHC[i] == (R3R0PTRTYPE(PX86PDPAE))HCPtr
3230 && pVM->pgm.s.aGCPhysGstPaePDs[i] == GCPhys,
3231 ("idx %d apGstPaePDsHC %VHv vs %VHv aGCPhysGstPaePDs %VGp vs %VGp\n",
3232 i, pVM->pgm.s.apGstPaePDsHC[i], HCPtr, pVM->pgm.s.aGCPhysGstPaePDs[i], GCPhys));
3233 }
3234 }
3235# endif
3236
3237 /*
3238 * Iterate the shadow page directory.
3239 */
3240 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3241 unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
3242 cPDEs += iPDDst;
3243 for (;
3244 iPDDst < cPDEs;
3245 iPDDst++, GCPtr += _4G / cPDEs)
3246 {
3247# if PGM_GST_TYPE == PGM_TYPE_PAE
3248 uint32_t iPDSrc;
3249 PGSTPD pPDSrc = pgmGstGetPaePDPtr(pPGM, (RTGCUINTPTR)GCPtr, &iPDSrc);
3250 if (!pPDSrc)
3251 {
3252 AssertMsg(!pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].n.u1Present, ("Guest PDTPR not present, shadow PDPTR %VX64\n", pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].u));
3253 continue;
3254 }
3255#endif
3256
3257 const SHWPDE PdeDst = pPDDst->a[iPDDst];
3258 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3259 {
3260 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3261 if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3262 {
3263 AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3264 cErrors++;
3265 continue;
3266 }
3267 }
3268 else if ( (PdeDst.u & X86_PDE_P)
3269 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3270 )
3271 {
3272 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3273 PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3274 if (!pPoolPage)
3275 {
3276 AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
3277 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3278 cErrors++;
3279 continue;
3280 }
3281 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3282
3283 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
3284 {
3285 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
3286 GCPtr, (uint64_t)PdeDst.u));
3287 cErrors++;
3288 }
3289
3290 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
3291 {
3292 AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
3293 GCPtr, (uint64_t)PdeDst.u));
3294 cErrors++;
3295 }
3296
3297 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3298 if (!PdeSrc.n.u1Present)
3299 {
3300 AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3301 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3302 cErrors++;
3303 continue;
3304 }
3305
3306 if ( !PdeSrc.b.u1Size
3307 || !(cr4 & X86_CR4_PSE))
3308 {
3309 GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3310# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3311 GCPhysGst |= (iPDDst & 1) * (PAGE_SIZE / 2);
3312# endif
3313 }
3314 else
3315 {
3316# if PGM_GST_TYPE == PGM_TYPE_32BIT
3317 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3318 {
3319 AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3320 GCPtr, (uint64_t)PdeSrc.u));
3321 cErrors++;
3322 continue;
3323 }
3324# endif
3325 GCPhysGst = PdeSrc.u & GST_PDE_BIG_PG_MASK;
3326# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3327 GCPhysGst |= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3328# endif
3329 }
3330
3331 if ( pPoolPage->enmKind
3332 != (!PdeSrc.b.u1Size || !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3333 {
3334 AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
3335 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3336 cErrors++;
3337 }
3338
3339 PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3340 if (!pPhysPage)
3341 {
3342 AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3343 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3344 cErrors++;
3345 continue;
3346 }
3347
3348 if (GCPhysGst != pPoolPage->GCPhys)
3349 {
3350 AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
3351 GCPhysGst, pPoolPage->GCPhys, GCPtr));
3352 cErrors++;
3353 continue;
3354 }
3355
3356 if ( !PdeSrc.b.u1Size
3357 || !(cr4 & X86_CR4_PSE))
3358 {
3359 /*
3360 * Page Table.
3361 */
3362 const GSTPT *pPTSrc;
3363 rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3364 if (VBOX_FAILURE(rc))
3365 {
3366 AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3367 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3368 cErrors++;
3369 continue;
3370 }
3371 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
3372 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
3373 {
3374 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3375 // (This problem will go away when/if we shadow multiple CR3s.)
3376 AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3377 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3378 cErrors++;
3379 continue;
3380 }
3381 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3382 {
3383 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
3384 GCPtr, (uint64_t)PdeDst.u));
3385 cErrors++;
3386 continue;
3387 }
3388
3389 /* iterate the page table. */
3390# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3391 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3392 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3393# else
3394 const unsigned offPTSrc = 0;
3395# endif
3396 for (unsigned iPT = 0, off = 0;
3397 iPT < ELEMENTS(pPTDst->a);
3398 iPT++, off += PAGE_SIZE)
3399 {
3400 const SHWPTE PteDst = pPTDst->a[iPT];
3401
3402 /* skip not-present entries. */
3403 if (!(PteDst.u & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3404 continue;
3405 Assert(PteDst.n.u1Present);
3406
3407 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3408 if (!PteSrc.n.u1Present)
3409 {
3410#ifdef IN_RING3
3411 PGMAssertHandlerAndFlagsInSync(pVM);
3412 PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3413#endif
3414 AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
3415 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3416 (PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3417 cErrors++;
3418 continue;
3419 }
3420
3421 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
3422# if 1 /** @todo sync accessed bit properly... */
3423 fIgnoreFlags |= X86_PTE_A;
3424# endif
3425
3426 /* match the physical addresses */
3427 HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3428 GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3429
3430# ifdef IN_RING3
3431 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3432 if (VBOX_FAILURE(rc))
3433 {
3434 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3435 {
3436 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3437 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3438 cErrors++;
3439 continue;
3440 }
3441 }
3442 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3443 {
3444 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3445 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3446 cErrors++;
3447 continue;
3448 }
3449# endif
3450
3451 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3452 if (!pPhysPage)
3453 {
3454# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3455 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3456 {
3457 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3458 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3459 cErrors++;
3460 continue;
3461 }
3462# endif
3463 if (PteDst.n.u1Write)
3464 {
3465 AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3466 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3467 cErrors++;
3468 }
3469 fIgnoreFlags |= X86_PTE_RW;
3470 }
3471 else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
3472 {
3473 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3474 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3475 cErrors++;
3476 continue;
3477 }
3478
3479 /* flags */
3480 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3481 {
3482 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3483 {
3484 if (PteDst.n.u1Write)
3485 {
3486 AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
3487 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3488 cErrors++;
3489 continue;
3490 }
3491 fIgnoreFlags |= X86_PTE_RW;
3492 }
3493 else
3494 {
3495 if (PteDst.n.u1Present)
3496 {
3497 AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
3498 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3499 cErrors++;
3500 continue;
3501 }
3502 fIgnoreFlags |= X86_PTE_P;
3503 }
3504 }
3505 else
3506 {
3507 if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3508 {
3509 if (PteDst.n.u1Write)
3510 {
3511 AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3512 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3513 cErrors++;
3514 continue;
3515 }
3516 if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3517 {
3518 AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3519 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3520 cErrors++;
3521 continue;
3522 }
3523 if (PteDst.n.u1Dirty)
3524 {
3525 AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3526 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3527 cErrors++;
3528 }
3529# if 0 /** @todo sync access bit properly... */
3530 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3531 {
3532 AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3533 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3534 cErrors++;
3535 }
3536 fIgnoreFlags |= X86_PTE_RW;
3537# else
3538 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
3539# endif
3540 }
3541 else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3542 {
3543 /* access bit emulation (not implemented). */
3544 if (PteSrc.n.u1Accessed || PteDst.n.u1Present)
3545 {
3546 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3547 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3548 cErrors++;
3549 continue;
3550 }
3551 if (!PteDst.n.u1Accessed)
3552 {
3553 AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3554 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3555 cErrors++;
3556 }
3557 fIgnoreFlags |= X86_PTE_P;
3558 }
3559# ifdef DEBUG_sandervl
3560 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
3561# endif
3562 }
3563
3564 if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3565 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3566 )
3567 {
3568 AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3569 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3570 fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3571 cErrors++;
3572 continue;
3573 }
3574 } /* foreach PTE */
3575 }
3576 else
3577 {
3578 /*
3579 * Big Page.
3580 */
3581 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
3582 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3583 {
3584 if (PdeDst.n.u1Write)
3585 {
3586 AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3587 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3588 cErrors++;
3589 continue;
3590 }
3591 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3592 {
3593 AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3594 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3595 cErrors++;
3596 continue;
3597 }
3598# if 0 /** @todo sync access bit properly... */
3599 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3600 {
3601 AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3602 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3603 cErrors++;
3604 }
3605 fIgnoreFlags |= X86_PTE_RW;
3606# else
3607 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
3608# endif
3609 }
3610 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3611 {
3612 /* access bit emulation (not implemented). */
3613 if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
3614 {
3615 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3616 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3617 cErrors++;
3618 continue;
3619 }
3620 if (!PdeDst.n.u1Accessed)
3621 {
3622 AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3623 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3624 cErrors++;
3625 }
3626 fIgnoreFlags |= X86_PTE_P;
3627 }
3628
3629 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3630 {
3631 AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3632 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3633 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3634 cErrors++;
3635 }
3636
3637 /* iterate the page table. */
3638 for (unsigned iPT = 0, off = 0;
3639 iPT < ELEMENTS(pPTDst->a);
3640 iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3641 {
3642 const SHWPTE PteDst = pPTDst->a[iPT];
3643
3644 if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3645 {
3646 AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3647 GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3648 cErrors++;
3649 }
3650
3651 /* skip not-present entries. */
3652 if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3653 continue;
3654
3655 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
3656
3657 /* match the physical addresses */
3658 HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3659
3660# ifdef IN_RING3
3661 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3662 if (VBOX_FAILURE(rc))
3663 {
3664 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3665 {
3666 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3667 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3668 cErrors++;
3669 }
3670 }
3671 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3672 {
3673 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3674 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3675 cErrors++;
3676 continue;
3677 }
3678# endif
3679
3680 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3681 if (!pPhysPage)
3682 {
3683# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3684 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3685 {
3686 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3687 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3688 cErrors++;
3689 continue;
3690 }
3691# endif
3692 if (PteDst.n.u1Write)
3693 {
3694 AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3695 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3696 cErrors++;
3697 }
3698 fIgnoreFlags |= X86_PTE_RW;
3699 }
3700 else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
3701 {
3702 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3703 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3704 cErrors++;
3705 continue;
3706 }
3707
3708 /* flags */
3709 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3710 {
3711 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3712 {
3713 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
3714 {
3715 if (PteDst.n.u1Write)
3716 {
3717 AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3718 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3719 cErrors++;
3720 continue;
3721 }
3722 fIgnoreFlags |= X86_PTE_RW;
3723 }
3724 }
3725 else
3726 {
3727 if (PteDst.n.u1Present)
3728 {
3729 AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3730 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3731 cErrors++;
3732 continue;
3733 }
3734 fIgnoreFlags |= X86_PTE_P;
3735 }
3736 }
3737
3738 if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3739 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3740 )
3741 {
3742 AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3743 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3744 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3745 cErrors++;
3746 continue;
3747 }
3748 } /* foreach PTE */
3749 }
3750 }
3751 /* not present */
3752
3753 } /* forearch PDE */
3754
3755# ifdef DEBUG
3756 if (cErrors)
3757 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3758# endif
3759
3760#elif PGM_GST_TYPE == PGM_TYPE_PAE
3761//# error not implemented
3762
3763
3764#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3765//# error not implemented
3766
3767/*#else: guest real and protected mode */
3768#endif
3769 return cErrors;
3770}
3771#endif /* VBOX_STRICT */
3772
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette