VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 6537

Last change on this file since 6537 was 5999, checked in by vboxsync, 17 years ago

The Giant CDDL Dual-License Header Change.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 151.4 KB
Line 
1/* $Id: PGMAllBth.h 5999 2007-12-07 15:05:06Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * This file is a big challenge!
6 */
7
8/*
9 * Copyright (C) 2006-2007 innotek GmbH
10 *
11 * This file is part of VirtualBox Open Source Edition (OSE), as
12 * available from http://www.virtualbox.org. This file is free software;
13 * you can redistribute it and/or modify it under the terms of the GNU
14 * General Public License (GPL) as published by the Free Software
15 * Foundation, in version 2 as it comes in the "COPYING" file of the
16 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18 */
19
20/*******************************************************************************
21* Internal Functions *
22*******************************************************************************/
23__BEGIN_DECLS
24PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
25PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
26PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
27PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
28PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
29PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
30PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
31PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal);
32#ifdef VBOX_STRICT
33PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
34#endif
35#ifdef PGMPOOL_WITH_USER_TRACKING
36DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
37#endif
38__END_DECLS
39
40
41/**
42 * #PF Handler for raw-mode guest execution.
43 *
44 * @returns VBox status code (appropriate for trap handling and GC return).
45 * @param pVM VM Handle.
46 * @param uErr The trap error code.
47 * @param pRegFrame Trap register frame.
48 * @param pvFault The fault address.
49 */
50PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
51{
52#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
53
54# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
55# error "32-bit guest mode is only implemented for 32-bit and PAE shadow modes."
56# endif
57
58# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
59 /*
60 * Hide the instruction fetch trap indicator for now.
61 */
62 /** @todo NXE will change this and we must fix NXE in the switcher too! */
63 if (uErr & X86_TRAP_PF_ID)
64 {
65 uErr &= ~X86_TRAP_PF_ID;
66 TRPMSetErrorCode(pVM, uErr);
67 }
68# endif
69
70 /*
71 * Get PDs.
72 */
73 int rc;
74# if PGM_WITH_PAGING(PGM_GST_TYPE)
75# if PGM_GST_TYPE == PGM_TYPE_32BIT
76 const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
77 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
78# else /* PAE */
79 unsigned iPDSrc;
80 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc);
81# endif
82# else
83 PGSTPD pPDSrc = NULL;
84 const unsigned iPDSrc = 0;
85# endif
86
87 const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
88# if PGM_SHW_TYPE == PGM_TYPE_32BIT
89 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
90# else /* PAE */
91 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries. */
92# endif
93
94# if PGM_WITH_PAGING(PGM_GST_TYPE)
95 /* Determine current privilege level */
96 uint32_t cpl = CPUMGetGuestCPL(pVM, pRegFrame);
97
98# ifdef PGM_SYNC_DIRTY_BIT
99 /*
100 * If we successfully correct the write protection fault due to dirty bit
101 * tracking, or this page fault is a genuine one, then return immediately.
102 */
103 STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
104 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
105 STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
106 if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
107 || rc == VINF_EM_RAW_GUEST_TRAP)
108 {
109 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
110 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
111 LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
112 return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
113 }
114# endif
115
116 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
117# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
118
119 /*
120 * A common case is the not-present error caused by lazy page table syncing.
121 *
122 * It is IMPORTANT that we weed out any access to non-present shadow PDEs here
123 * so we can safely assume that the shadow PT is present when calling SyncPage later.
124 *
125 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
126 * of mapping conflict and defer to SyncCR3 in R3.
127 * (Again, we do NOT support access handlers for non-present guest pages.)
128 *
129 */
130# if PGM_WITH_PAGING(PGM_GST_TYPE)
131 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
132# else
133 GSTPDE PdeSrc;
134 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
135 PdeSrc.n.u1Present = 1;
136 PdeSrc.n.u1Write = 1;
137 PdeSrc.n.u1Accessed = 1;
138 PdeSrc.n.u1User = 1;
139# endif
140 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
141 && !pPDDst->a[iPDDst].n.u1Present
142 && PdeSrc.n.u1Present
143 )
144
145 {
146 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
147 STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
148 LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
149 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
150 if (VBOX_SUCCESS(rc))
151 {
152 STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
153 return rc;
154 }
155 Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
156 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
157 STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
158 return VINF_PGM_SYNC_CR3;
159 }
160
161# if PGM_WITH_PAGING(PGM_GST_TYPE)
162 /*
163 * Check if this address is within any of our mappings.
164 *
165 * This is *very* fast and it's gonna save us a bit of effort below and prevent
166 * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
167 * (BTW, it's impossible to have physical access handlers in a mapping.)
168 */
169 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
170 {
171 STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
172 PPGMMAPPING pMapping = CTXALLSUFF(pVM->pgm.s.pMappings);
173 for ( ; pMapping; pMapping = CTXALLSUFF(pMapping->pNext))
174 {
175 if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
176 break;
177 if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
178 {
179 /*
180 * The first thing we check is if we've got an undetected conflict.
181 */
182 if (!pVM->pgm.s.fMappingsFixed)
183 {
184 unsigned iPT = pMapping->cPTs;
185 while (iPT-- > 0)
186 if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
187 {
188 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
189 Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
190 VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
191 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
192 return VINF_PGM_SYNC_CR3;
193 }
194 }
195
196 /*
197 * Check if the fault address is in a virtual page access handler range.
198 */
199 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
200 if ( pCur
201 && pCur->enmType != PGMVIRTHANDLERTYPE_EIP
202 && (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
203 && ( uErr & X86_TRAP_PF_RW
204 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
205 && pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** r=bird: <- this is probably wrong. */
206 {
207# ifdef IN_GC
208 STAM_PROFILE_START(&pCur->Stat, h);
209 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
210 STAM_PROFILE_STOP(&pCur->Stat, h);
211# else
212 AssertFailed();
213 rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
214# endif
215 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
216 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
217 return rc;
218 }
219
220 /*
221 * Check if the EIP is in a virtual page access handler range.
222 */
223 if (cpl == 0)
224 {
225 RTGCPTR pvEIP;
226 rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &pvEIP);
227 if (VBOX_SUCCESS(rc))
228 {
229 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvEIP);
230 if ( pCur
231 && pCur->enmType == PGMVIRTHANDLERTYPE_EIP
232 && (RTGCUINTPTR)pvEIP - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
233 {
234# ifdef IN_GC
235 STAM_PROFILE_START(&pCur->Stat, h);
236 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvEIP - (RTGCUINTPTR)pCur->GCPtr);
237 STAM_PROFILE_STOP(&pCur->Stat, h);
238# else
239 AssertFailed();
240 rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
241# endif
242 STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
243 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
244 return rc;
245 }
246 }
247 }
248
249 /*
250 * Pretend we're not here and let the guest handle the trap.
251 */
252 TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
253 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
254 LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
255 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
256 return VINF_EM_RAW_GUEST_TRAP;
257 }
258 }
259 STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
260 } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
261# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
262
263 /*
264 * Check if this fault address is flagged for special treatment,
265 * which means we'll have to figure out the physical address and
266 * check flags associated with it.
267 *
268 * ASSUME that we can limit any special access handling to pages
269 * in page tables which the guest believes to be present.
270 */
271 if (PdeSrc.n.u1Present)
272 {
273 RTGCPHYS GCPhys = ~0U;
274
275# if PGM_WITH_PAGING(PGM_GST_TYPE)
276 uint32_t cr4 = CPUMGetGuestCR4(pVM);
277 if ( PdeSrc.b.u1Size
278 && (cr4 & X86_CR4_PSE))
279 GCPhys = (PdeSrc.u & GST_PDE4M_PG_MASK)
280 | ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
281 else
282 {
283 PVBOXPT pPTSrc;
284# ifdef IN_GC
285 rc = PGMGCDynMapGCPage(pVM, PdeSrc.u & GST_PDE_PG_MASK, (void **)&pPTSrc);
286# else
287 pPTSrc = (PVBOXPT)MMPhysGCPhys2HCVirt(pVM, PdeSrc.u & GST_PDE_PG_MASK, sizeof(*pPTSrc));
288 if (pPTSrc == 0)
289 rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
290# endif
291 if (VBOX_SUCCESS(rc))
292 {
293 unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> PAGE_SHIFT) & GST_PT_MASK;
294 if (pPTSrc->a[iPTESrc].n.u1Present)
295 GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
296 }
297 }
298# else
299 /* No paging so the fault address is the physical address */
300 GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
301# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
302
303 /*
304 * If we have a GC address we'll check if it has any flags set.
305 */
306 if (GCPhys != ~0U)
307 {
308 STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
309
310 PPGMPAGE pPage;
311 rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
312 if (VBOX_SUCCESS(rc))
313 {
314 if (pPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_HANDLER | MM_RAM_FLAGS_VIRTUAL_HANDLER)) /** @todo PAGE FLAGS */
315 {
316 if (pPage->HCPhys & MM_RAM_FLAGS_PHYSICAL_HANDLER) /** @todo PAGE FLAGS */
317 {
318 /*
319 * Physical page access handler.
320 */
321 const RTGCPHYS GCPhysFault = GCPhys | ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
322 PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
323 if (pCur)
324 {
325# ifdef PGM_SYNC_N_PAGES
326 /*
327 * If the region is write protected and we got a page not present fault, then sync
328 * the pages. If the fault was caused by a read, then restart the instruction.
329 * In case of write access continue to the GC write handler.
330 *
331 * ASSUMES that there is only one handler per page or that they have similar write properties.
332 */
333 if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
334 && !(uErr & X86_TRAP_PF_P))
335 {
336 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
337 if ( VBOX_FAILURE(rc)
338 || !(uErr & X86_TRAP_PF_RW)
339 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
340 {
341 AssertRC(rc);
342 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
343 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
344 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
345 return rc;
346 }
347 }
348# endif
349
350 AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
351 || (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
352 ("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
353
354#if defined(IN_GC) || defined(IN_RING0)
355 if (CTXALLSUFF(pCur->pfnHandler))
356 {
357 STAM_PROFILE_START(&pCur->Stat, h);
358 rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
359 STAM_PROFILE_STOP(&pCur->Stat, h);
360 }
361 else
362#endif
363 rc = VINF_EM_RAW_EMULATE_INSTR;
364 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
365 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
366 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
367 return rc;
368 }
369 }
370# if PGM_WITH_PAGING(PGM_GST_TYPE)
371 else
372 {
373# ifdef PGM_SYNC_N_PAGES
374 /*
375 * If the region is write protected and we got a page not present fault, then sync
376 * the pages. If the fault was caused by a read, then restart the instruction.
377 * In case of write access continue to the GC write handler.
378 */
379 if ( (pPage->HCPhys & (MM_RAM_FLAGS_VIRTUAL_WRITE | MM_RAM_FLAGS_VIRTUAL_ALL)) == MM_RAM_FLAGS_VIRTUAL_WRITE /** @todo PAGE FLAGS */
380 && !(uErr & X86_TRAP_PF_P))
381 {
382 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
383 if ( VBOX_FAILURE(rc)
384 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
385 || !(uErr & X86_TRAP_PF_RW))
386 {
387 AssertRC(rc);
388 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
389 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
390 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
391 return rc;
392 }
393 }
394# endif
395 /*
396 * Ok, it's an virtual page access handler.
397 *
398 * Since it's faster to search by address, we'll do that first
399 * and then retry by GCPhys if that fails.
400 */
401 /** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
402 /** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
403 * page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
404 */
405 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
406 if (pCur)
407 {
408 AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
409 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
410 || !(uErr & X86_TRAP_PF_P)
411 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
412 ("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
413
414 if ( pCur->enmType != PGMVIRTHANDLERTYPE_EIP
415 && (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
416 && ( uErr & X86_TRAP_PF_RW
417 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
418 && pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** @todo r=bird: _HYPERVISOR is impossible here because of mapping check. */
419 {
420# ifdef IN_GC
421 STAM_PROFILE_START(&pCur->Stat, h);
422 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
423 STAM_PROFILE_STOP(&pCur->Stat, h);
424# else
425 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
426# endif
427 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
428 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
429 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
430 return rc;
431 }
432 /* Unhandled part of a monitored page */
433 }
434 else
435 {
436 /* Check by physical address. */
437 PPGMVIRTHANDLER pCur;
438 unsigned iPage;
439 rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
440 &pCur, &iPage);
441 Assert(VBOX_SUCCESS(rc) || !pCur);
442 if ( pCur
443 && pCur->enmType != PGMVIRTHANDLERTYPE_EIP
444 && ( uErr & X86_TRAP_PF_RW
445 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
446 && pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) )
447 {
448 Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
449# ifdef IN_GC
450 RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->GCPtr & PAGE_OFFSET_MASK);
451 Assert(off < pCur->cb);
452 STAM_PROFILE_START(&pCur->Stat, h);
453 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
454 STAM_PROFILE_STOP(&pCur->Stat, h);
455# else
456 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
457# endif
458 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
459 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
460 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
461 return rc;
462 }
463 }
464 }
465# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
466
467 /*
468 * There is a handled area of the page, but this fault doesn't belong to it.
469 * We must emulate the instruction.
470 *
471 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
472 * we first check if this was a page-not-present fault for a page with only
473 * write access handlers. Restart the instruction if it wasn't a write access.
474 */
475 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
476
477 if ( !(pPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL)) /** @todo PAGE FLAGS */
478 && !(uErr & X86_TRAP_PF_P))
479 {
480 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
481 if ( VBOX_FAILURE(rc)
482 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
483 || !(uErr & X86_TRAP_PF_RW))
484 {
485 AssertRC(rc);
486 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
487 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
488 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
489 return rc;
490 }
491 }
492
493 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
494 * It's writing to an unhandled part of the LDT page several million times.
495 */
496 rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
497 LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%VHp%s%s\n",
498 rc, pPage->HCPhys, pPage->HCPhys & MM_RAM_FLAGS_PHYSICAL_HANDLER ? " phys" : "",
499 pPage->HCPhys & MM_RAM_FLAGS_VIRTUAL_HANDLER ? " virt" : ""));
500 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
501 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
502 return rc;
503 } /* if any kind of handler */
504
505# if PGM_WITH_PAGING(PGM_GST_TYPE)
506 if (uErr & X86_TRAP_PF_P)
507 {
508 /*
509 * The page isn't marked, but it might still be monitored by a virtual page access handler.
510 * (ASSUMES no temporary disabling of virtual handlers.)
511 */
512 /** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
513 * we should correct both the shadow page table and physical memory flags, and not only check for
514 * accesses within the handler region but for access to pages with virtual handlers. */
515 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
516 if (pCur)
517 {
518 AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
519 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
520 || !(uErr & X86_TRAP_PF_P)
521 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
522 ("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
523
524 if ( pCur->enmType != PGMVIRTHANDLERTYPE_EIP
525 && (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
526 && ( uErr & X86_TRAP_PF_RW
527 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
528 && pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR) ) ) /** @todo r=bird: _HYPERVISOR is impossible here because of mapping check. */
529 {
530# ifdef IN_GC
531 STAM_PROFILE_START(&pCur->Stat, h);
532 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
533 STAM_PROFILE_STOP(&pCur->Stat, h);
534# else
535 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
536# endif
537 STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
538 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
539 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
540 return rc;
541 }
542 }
543 }
544# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
545 }
546 STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
547
548# ifdef PGM_OUT_OF_SYNC_IN_GC
549 /*
550 * We are here only if page is present in Guest page tables and trap is not handled
551 * by our handlers.
552 * Check it for page out-of-sync situation.
553 */
554 STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
555
556 if (!(uErr & X86_TRAP_PF_P))
557 {
558 /*
559 * Page is not present in our page tables.
560 * Try to sync it!
561 * BTW, fPageShw is invalid in this branch!
562 */
563 if (uErr & X86_TRAP_PF_US)
564 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
565 else /* supervisor */
566 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
567
568# if defined(LOG_ENABLED) && !defined(IN_RING0)
569 RTGCPHYS GCPhys;
570 uint64_t fPageGst;
571 PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
572 Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
573 pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
574# endif /* LOG_ENABLED */
575
576# if PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0)
577 if (cpl == 0)
578 {
579 uint64_t fPageGst;
580 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
581 if ( VBOX_SUCCESS(rc)
582 && !(fPageGst & X86_PTE_US))
583 {
584 /* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
585 if ( pvFault == (RTGCPTR)pRegFrame->eip
586 || (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
587# ifdef CSAM_DETECT_NEW_CODE_PAGES
588 || ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
589 && CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
590# endif /* CSAM_DETECT_NEW_CODE_PAGES */
591 )
592 {
593 LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
594 rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
595 if (rc != VINF_SUCCESS)
596 {
597 /*
598 * CSAM needs to perform a job in ring 3.
599 *
600 * Sync the page before going to the host context; otherwise we'll end up in a loop if
601 * CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
602 */
603 LogFlow(("CSAM ring 3 job\n"));
604 int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
605 AssertRC(rc2);
606
607 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
608 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
609 return rc;
610 }
611 }
612# ifdef CSAM_DETECT_NEW_CODE_PAGES
613 else
614 if ( uErr == X86_TRAP_PF_RW
615 && pRegFrame->ecx >= 0x100 /* early check for movswd count */
616 && pRegFrame->ecx < 0x10000
617 )
618 {
619 /* In case of a write to a non-present supervisor shadow page, we'll take special precautions
620 * to detect loading of new code pages.
621 */
622
623 /*
624 * Decode the instruction.
625 */
626 RTGCPTR PC;
627 rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
628 if (rc == VINF_SUCCESS)
629 {
630 DISCPUSTATE Cpu;
631 uint32_t cbOp;
632 rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
633
634 /* For now we'll restrict this to rep movsw/d instructions */
635 if ( rc == VINF_SUCCESS
636 && Cpu.pCurInstr->opcode == OP_MOVSWD
637 && (Cpu.prefix & PREFIX_REP))
638 {
639 CSAMMarkPossibleCodePage(pVM, pvFault);
640 }
641 }
642 }
643# endif /* CSAM_DETECT_NEW_CODE_PAGES */
644
645 /*
646 * Mark this page as safe.
647 */
648 /** @todo not correct for pages that contain both code and data!! */
649 Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
650 CSAMMarkPage(pVM, pvFault, true);
651 }
652 }
653# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
654 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
655 if (VBOX_SUCCESS(rc))
656 {
657 /* The page was successfully synced, return to the guest. */
658 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
659 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
660 return VINF_SUCCESS;
661 }
662 }
663 else
664 {
665 /*
666 * A side effect of not flushing global PDEs are out of sync pages due
667 * to physical monitored regions, that are no longer valid.
668 * Assume for now it only applies to the read/write flag
669 */
670 if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
671 {
672 if (uErr & X86_TRAP_PF_US)
673 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
674 else /* supervisor */
675 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
676
677
678 /*
679 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
680 */
681 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
682 if (VBOX_SUCCESS(rc))
683 {
684 /*
685 * Page was successfully synced, return to guest.
686 */
687# ifdef VBOX_STRICT
688 RTGCPHYS GCPhys;
689 uint64_t fPageGst;
690 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
691 Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
692 LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
693
694 uint64_t fPageShw;
695 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
696 Assert(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW);
697# endif /* VBOX_STRICT */
698 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
699 STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
700 return VINF_SUCCESS;
701 }
702 }
703
704# if PGM_WITH_PAGING(PGM_GST_TYPE)
705# ifdef VBOX_STRICT
706 /*
707 * Check for VMM page flags vs. Guest page flags consistency.
708 * Currently only for debug purposes.
709 */
710 if (VBOX_SUCCESS(rc))
711 {
712 /* Get guest page flags. */
713 uint64_t fPageGst;
714 rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
715 if (VBOX_SUCCESS(rc))
716 {
717 uint64_t fPageShw;
718 rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
719
720 /*
721 * Compare page flags.
722 * Note: we have AVL, A, D bits desynched.
723 */
724 AssertMsg((fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)),
725 ("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
726 }
727 else
728 AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
729 }
730 else
731 AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
732# endif /* VBOX_STRICT */
733# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
734 }
735 STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
736# endif /* PGM_OUT_OF_SYNC_IN_GC */
737 }
738 else
739 {
740 /*
741 * Page not present in Guest OS or invalid page table address.
742 * This is potential virtual page access handler food.
743 *
744 * For the present we'll say that our access handlers don't
745 * work for this case - we've already discarded the page table
746 * not present case which is identical to this.
747 *
748 * When we perchance find we need this, we will probably have AVL
749 * trees (offset based) to operate on and we can measure their speed
750 * agains mapping a page table and probably rearrange this handling
751 * a bit. (Like, searching virtual ranges before checking the
752 * physical address.)
753 */
754 }
755 }
756
757
758# if PGM_WITH_PAGING(PGM_GST_TYPE)
759 /*
760 * Check if it's in a EIP based virtual page access handler range.
761 * This is only used for supervisor pages in flat mode.
762 */
763 /** @todo this stuff is completely broken by the out-of-sync stuff. since we don't use this stuff, that's not really a problem yet. */
764 STAM_PROFILE_START(&pVM->pgm.s.StatEIPHandlers, d);
765 if (cpl == 0)
766 {
767 RTGCPTR pvEIP;
768 rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &pvEIP);
769 if ( VBOX_SUCCESS(rc)
770 && pvEIP == (RTGCPTR)pRegFrame->eip)
771 {
772 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvEIP);
773 if ( pCur
774 && pCur->enmType == PGMVIRTHANDLERTYPE_EIP
775 && (RTGCUINTPTR)pvEIP - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
776 {
777 LogFlow(("EIP handler\n"));
778# ifdef IN_GC
779 STAM_PROFILE_START(&pCur->Stat, h);
780 rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvEIP - (RTGCUINTPTR)pCur->GCPtr);
781 STAM_PROFILE_STOP(&pCur->Stat, h);
782# else
783 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
784# endif
785 STAM_PROFILE_STOP(&pVM->pgm.s.StatEIPHandlers, d);
786 return rc;
787 }
788 }
789 }
790 STAM_PROFILE_STOP(&pVM->pgm.s.StatEIPHandlers, d);
791
792 /*
793 * Conclusion, this is a guest trap.
794 */
795 LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
796 STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
797 return VINF_EM_RAW_GUEST_TRAP;
798# else
799 /* present, but not a monitored page; perhaps the guest is probing physical memory */
800 return VINF_EM_RAW_EMULATE_INSTR;
801# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
802
803
804#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
805
806 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
807 return VERR_INTERNAL_ERROR;
808#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
809}
810
811
812/**
813 * Emulation of the invlpg instruction.
814 *
815 *
816 * @returns VBox status code.
817 *
818 * @param pVM VM handle.
819 * @param GCPtrPage Page to invalidate.
820 *
821 * @remark ASSUMES that the guest is updating before invalidating. This order
822 * isn't required by the CPU, so this is speculative and could cause
823 * trouble.
824 *
825 * @todo Flush page or page directory only if necessary!
826 * @todo Add a #define for simply invalidating the page.
827 */
828PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
829{
830#if PGM_GST_TYPE == PGM_TYPE_32BIT
831
832 LogFlow(("InvalidatePage %x\n", GCPtrPage));
833# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE
834 /*
835 * Get the shadow PD entry and skip out if this PD isn't present.
836 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
837 */
838 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
839# if PGM_SHW_TYPE == PGM_TYPE_32BIT
840 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
841# else
842 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
843# endif
844 const SHWPDE PdeDst = *pPdeDst;
845 if (!PdeDst.n.u1Present)
846 {
847 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
848 return VINF_SUCCESS;
849 }
850
851 /*
852 * Get the guest PD entry and calc big page.
853 */
854# if PGM_GST_TYPE == PGM_TYPE_32BIT
855 PX86PD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
856 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
857 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
858# else /* PAE */
859 unsigned iPDSrc;
860 PX86PD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
861
862 GSTPDE PdeSrc;;
863 PdeSrc.u = pgmGstGetPaePDE(&pVM->pgm.s, GCPtrPage);
864# endif
865
866 const uint32_t cr4 = CPUMGetGuestCR4(pVM);
867 const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
868
869# ifdef IN_RING3
870 /*
871 * If a CR3 Sync is pending we may ignore the invalidate page operation
872 * depending on the kind of sync and if it's a global page or not.
873 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
874 */
875# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
876 if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
877 || ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
878 && fIsBigPage
879 && PdeSrc.b.u1Global
880 && (cr4 & X86_CR4_PGE)
881 )
882 )
883# else
884 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
885# endif
886 {
887 STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
888 return VINF_SUCCESS;
889 }
890# endif /* IN_RING3 */
891
892
893 /*
894 * Deal with the Guest PDE.
895 */
896 int rc = VINF_SUCCESS;
897 if (PdeSrc.n.u1Present)
898 {
899 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
900 {
901 /*
902 * Conflict - Let SyncPT deal with it to avoid duplicate code.
903 */
904 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
905 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
906 }
907 else if ( PdeSrc.n.u1User != PdeDst.n.u1User
908 || (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
909 {
910 /*
911 * Mark not present so we can resync the PDE when it's used.
912 */
913 LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
914 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
915 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
916 pPdeDst->u = 0;
917 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
918 PGM_INVL_GUEST_TLBS();
919 }
920# ifdef PGM_SYNC_ACCESSED_BIT
921 else if (!PdeSrc.n.u1Accessed)
922 {
923 /*
924 * Mark not present so we can set the accessed bit.
925 */
926 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
927 pPdeDst->u = 0;
928 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
929 PGM_INVL_GUEST_TLBS();
930 }
931# endif
932 else if (!fIsBigPage)
933 {
934 /*
935 * 4KB - page.
936 */
937 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
938 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
939# if PGM_SHW_TYPE != PGM_TYPE_32BIT
940 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
941# endif
942 if (pShwPage->GCPhys == GCPhys)
943 {
944#if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
945 const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
946 PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
947 if (pPT->a[iPTEDst].n.u1Present)
948 {
949# ifdef PGMPOOL_WITH_USER_TRACKING
950 /* This is very unlikely with caching/monitoring enabled. */
951 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
952# endif
953 pPT->a[iPTEDst].u = 0;
954 }
955#else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
956 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
957 if (VBOX_SUCCESS(rc))
958 rc = VINF_SUCCESS;
959#endif
960 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
961 PGM_INVL_PG(GCPtrPage);
962 }
963 else
964 {
965 /*
966 * The page table address changed.
967 */
968 LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
969 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
970 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
971 pPdeDst->u = 0;
972 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
973 PGM_INVL_GUEST_TLBS();
974 }
975 }
976 else
977 {
978 /*
979 * 4MB - page.
980 */
981 /* Before freeing the page, check if anything really changed. */
982 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
983 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
984# if PGM_SHW_TYPE != PGM_TYPE_32BIT
985 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
986# endif
987 if ( pShwPage->GCPhys == GCPhys
988 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
989 {
990 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
991 /** @todo PAT */
992# ifdef PGM_SYNC_DIRTY_BIT
993 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
994 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
995 && ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
996 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
997# else
998 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
999 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD)))
1000# endif
1001 {
1002 LogFlow(("Skipping flush for big page containing %VGv (PD=%X)-> nothing has changed!\n", GCPtrPage, iPDSrc));
1003 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
1004 return VINF_SUCCESS;
1005 }
1006 }
1007
1008 /*
1009 * Ok, the page table is present and it's been changed in the guest.
1010 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1011 * We could do this for some flushes in GC too, but we need an algorithm for
1012 * deciding which 4MB pages containing code likely to be executed very soon.
1013 */
1014 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1015 pPdeDst->u = 0;
1016 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
1017 DUMP_PDE_BIG("PGMInvalidatePage", iPDSrc, PdeSrc);
1018 PGM_INVL_BIG_PG(GCPtrPage);
1019 }
1020 }
1021 else
1022 {
1023 /*
1024 * Page directory is not present, mark shadow PDE not present.
1025 */
1026 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1027 {
1028 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1029 pPdeDst->u = 0;
1030 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1031 PGM_INVL_PG(GCPtrPage);
1032 }
1033 else
1034 {
1035 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1036 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
1037 }
1038 }
1039
1040 return rc;
1041
1042# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1043# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
1044# endif
1045 return VINF_SUCCESS;
1046
1047#elif PGM_GST_TYPE == PGM_TYPE_PAE
1048# if PGM_SHW_TYPE == PGM_TYPE_PAE
1049//# error not implemented
1050 return VERR_INTERNAL_ERROR;
1051
1052# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1053# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
1054# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1055
1056#elif PGM_GST_TYPE == PGM_TYPE_AMD64
1057# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1058//# error not implemented
1059 return VERR_INTERNAL_ERROR;
1060
1061# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1062# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
1063# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
1064
1065#else /* guest real and protected mode */
1066 /* There's no such thing when paging is disabled. */
1067 return VINF_SUCCESS;
1068#endif
1069}
1070
1071
1072#ifdef PGMPOOL_WITH_USER_TRACKING
1073/**
1074 * Update the tracking of shadowed pages.
1075 *
1076 * @param pVM The VM handle.
1077 * @param pShwPage The shadow page.
1078 * @param HCPhys The physical page we is being dereferenced.
1079 */
1080DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1081{
1082# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1083 STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1084 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1085
1086 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1087 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1088 * 2. write protect all shadowed pages. I.e. implement caching.
1089 */
1090 /*
1091 * Find the guest address.
1092 */
1093 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTXSUFF(pRamRanges);
1094 pRam;
1095 pRam = pRam->CTXSUFF(pNext))
1096 {
1097 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1098 while (iPage-- > 0)
1099 {
1100 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1101 {
1102 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1103 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1104 pShwPage->cPresent--;
1105 pPool->cPresent--;
1106 STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1107 return;
1108 }
1109 }
1110 }
1111
1112 for (;;)
1113 AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1114# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1115 pShwPage->cPresent--;
1116 pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1117# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1118}
1119
1120
1121/**
1122 * Update the tracking of shadowed pages.
1123 *
1124 * @param pVM The VM handle.
1125 * @param pShwPage The shadow page.
1126 * @param u16 The top 16-bit of the pPage->HCPhys.
1127 * @param pPage Pointer to the guest page. this will be modified.
1128 * @param iPTDst The index into the shadow table.
1129 */
1130DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1131{
1132# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1133 /*
1134 * We're making certain assumptions about the placement of cRef and idx.
1135 */
1136 Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1137 Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1138
1139 /*
1140 * Just deal with the simple first time here.
1141 */
1142 if (!u16)
1143 {
1144 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1145 u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) | pShwPage->idx;
1146 }
1147 else
1148 u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1149
1150 /* write back, trying to be clever... */
1151 Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%VHp->%VHp iPTDst=%#x\n",
1152 u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) | ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1153 *((uint16_t *)&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1154# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1155
1156 /* update statistics. */
1157 pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1158 pShwPage->cPresent++;
1159 if (pShwPage->iFirstPresent > iPTDst)
1160 pShwPage->iFirstPresent = iPTDst;
1161}
1162#endif /* PGMPOOL_WITH_USER_TRACKING */
1163
1164
1165/**
1166 * Creates a 4K shadow page for a guest page.
1167 *
1168 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1169 * physical address. The PdeSrc argument only the flags are used. No page structured
1170 * will be mapped in this function.
1171 *
1172 * @param pVM VM handle.
1173 * @param pPteDst Destination page table entry.
1174 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1175 * Can safely assume that only the flags are being used.
1176 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1177 * @param pShwPage Pointer to the shadow page.
1178 * @param iPTDst The index into the shadow table.
1179 *
1180 * @remark Not used for 2/4MB pages!
1181 */
1182DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, VBOXPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1183{
1184 if (PteSrc.n.u1Present)
1185 {
1186 /*
1187 * Find the ram range.
1188 */
1189 PPGMPAGE pPage;
1190 int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1191 if (VBOX_SUCCESS(rc))
1192 {
1193 /** @todo investiage PWT, PCD and PAT. */
1194 /*
1195 * Make page table entry.
1196 */
1197 const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1198 SHWPTE PteDst;
1199 if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE))
1200 {
1201 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1202 if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL)))
1203 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1204 | (HCPhys & X86_PTE_PAE_PG_MASK);
1205 else
1206 {
1207 LogFlow(("SyncPageWorker: monitored page (%VGp) -> mark not present\n", (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL))));
1208 PteDst.u = 0;
1209 }
1210 /** @todo count these two kinds. */
1211 }
1212 else
1213 {
1214#ifdef PGM_SYNC_DIRTY_BIT
1215# ifdef PGM_SYNC_ACCESSED_BIT
1216 /*
1217 * If the page or page directory entry is not marked accessed,
1218 * we mark the page not present.
1219 */
1220 if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
1221 {
1222 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1223 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1224 PteDst.u = 0;
1225 }
1226 else
1227# endif
1228 /*
1229 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1230 * when the page is modified.
1231 */
1232 if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1233 {
1234 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1235 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1236 | (HCPhys & X86_PTE_PAE_PG_MASK)
1237 | PGM_PTFLAGS_TRACK_DIRTY;
1238 }
1239 else
1240 {
1241 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1242 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1243 | (HCPhys & X86_PTE_PAE_PG_MASK);
1244 }
1245#endif
1246 }
1247
1248#ifdef PGMPOOL_WITH_USER_TRACKING
1249 /*
1250 * Keep user track up to date.
1251 */
1252 if (PteDst.n.u1Present)
1253 {
1254 if (!pPteDst->n.u1Present)
1255 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1256 else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1257 {
1258 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1259 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1260 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1261 }
1262 }
1263 else if (pPteDst->n.u1Present)
1264 {
1265 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1266 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1267 }
1268#endif /* PGMPOOL_WITH_USER_TRACKING */
1269
1270 /*
1271 * Update statistics and commit the entry.
1272 */
1273 if (!PteSrc.n.u1Global)
1274 pShwPage->fSeenNonGlobal = true;
1275 *pPteDst = PteDst;
1276 }
1277 /* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1278 /** @todo count these. */
1279 }
1280 else
1281 {
1282 /*
1283 * Page not-present.
1284 */
1285 LogFlow(("SyncPageWorker: page not present in Pte\n"));
1286#ifdef PGMPOOL_WITH_USER_TRACKING
1287 /* Keep user track up to date. */
1288 if (pPteDst->n.u1Present)
1289 {
1290 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1291 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1292 }
1293#endif /* PGMPOOL_WITH_USER_TRACKING */
1294 pPteDst->u = 0;
1295 /** @todo count these. */
1296 }
1297}
1298
1299
1300/**
1301 * Syncs a guest OS page.
1302 *
1303 * There are no conflicts at this point, neither is there any need for
1304 * page table allocations.
1305 *
1306 * @returns VBox status code.
1307 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1308 * @param pVM VM handle.
1309 * @param PdeSrc Page directory entry of the guest.
1310 * @param GCPtrPage Guest context page address.
1311 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1312 * @param uErr Fault error (X86_TRAP_PF_*).
1313 */
1314PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1315{
1316 LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1317
1318#if PGM_GST_TYPE == PGM_TYPE_32BIT
1319
1320# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1321# error "Invalid shadow mode for 32-bit guest mode!"
1322# endif
1323
1324 /*
1325 * Assert preconditions.
1326 */
1327# if GC_ARCH_BITS != 32
1328 Assert(GCPtrPage < _4G); //???
1329# endif
1330 STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> X86_PD_SHIFT) & GST_PD_MASK]);
1331 Assert(PdeSrc.n.u1Present);
1332 Assert(cPages);
1333
1334 /*
1335 * Get the shadow PDE, find the shadow page table in the pool.
1336 */
1337 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1338# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1339 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1340# else /* PAE */
1341 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1342# endif
1343 Assert(PdeDst.n.u1Present);
1344 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1345
1346 /*
1347 * Check that the page is present and that the shadow PDE isn't out of sync.
1348 */
1349 const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1350 RTGCPHYS GCPhys;
1351 if (!fBigPage)
1352 {
1353 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1354# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1355 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1356# endif
1357 }
1358 else
1359 {
1360 GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1361# if PGM_SHW_TYPE != PGM_TYPE_32BIT
1362 GCPhys |= GCPtrPage & X86_PAGE_2M_SIZE;
1363# endif
1364 }
1365 if ( pShwPage->GCPhys == GCPhys
1366 && PdeSrc.n.u1Present
1367 && (PdeSrc.n.u1User == PdeDst.n.u1User)
1368 && (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
1369 )
1370 {
1371# ifdef PGM_SYNC_ACCESSED_BIT
1372 /*
1373 * Check that the PDE is marked accessed already.
1374 * Since we set the accessed bit *before* getting here on a #PF, this
1375 * check is only meant for dealing with non-#PF'ing paths.
1376 */
1377 if (PdeSrc.n.u1Accessed)
1378# endif
1379 {
1380 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1381 if (!fBigPage)
1382 {
1383 /*
1384 * 4KB Page - Map the guest page table.
1385 */
1386 PVBOXPT pPTSrc;
1387 int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1388 if (VBOX_SUCCESS(rc))
1389 {
1390# ifdef PGM_SYNC_N_PAGES
1391 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1392 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1393 {
1394 /*
1395 * This code path is currently only taken when the caller is PGMTrap0eHandler
1396 * for non-present pages!
1397 *
1398 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1399 * deal with locality.
1400 */
1401 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1402# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1403 const unsigned offPTSrc = 0;
1404# else
1405 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1406# endif
1407 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1408 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1409 iPTDst = 0;
1410 else
1411 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1412 for (; iPTDst < iPTDstEnd; iPTDst++)
1413 {
1414 if (!pPTDst->a[iPTDst].n.u1Present)
1415 {
1416 VBOXPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1417 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1418 NOREF(GCPtrCurPage);
1419#ifndef IN_RING0
1420 /*
1421 * Assuming kernel code will be marked as supervisor - and not as user level
1422 * and executed using a conforming code selector - And marked as readonly.
1423 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
1424 */
1425 if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW | X86_PTE_US))
1426 || iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1427 || !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1428 || pgmRamTestFlags(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK,
1429 MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)
1430 )
1431#endif
1432 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1433 Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1434 GCPtrCurPage, PteSrc.n.u1Present,
1435 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1436 PteSrc.n.u1User & PdeSrc.n.u1User,
1437 (uint64_t)PteSrc.u,
1438 (uint64_t)pPTDst->a[iPTDst].u,
1439 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1440 }
1441 }
1442 }
1443 else
1444# endif /* PGM_SYNC_N_PAGES */
1445 {
1446 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1447 VBOXPTE PteSrc = pPTSrc->a[iPTSrc];
1448 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1449 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1450 Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1451 GCPtrPage, PteSrc.n.u1Present,
1452 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1453 PteSrc.n.u1User & PdeSrc.n.u1User,
1454 (uint64_t)PteSrc.u,
1455 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1456 }
1457 }
1458 else /* MMIO or invalid page: emulated in #PF handler. */
1459 {
1460 LogFlow(("PGM_GCPHYS_2_PTR %VGp failed with %Vrc\n", GCPhys, rc));
1461 Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1462 }
1463 }
1464 else
1465 {
1466 /*
1467 * 4/2MB page - lazy syncing shadow 4K pages.
1468 * (There are many causes of getting here, it's no longer only CSAM.)
1469 */
1470 /* Calculate the GC physical address of this 4KB shadow page. */
1471 RTGCPHYS GCPhys = (PdeSrc.u & X86_PDE4M_PAE_PG_MASK) | ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1472 /* Find ram range. */
1473 PPGMPAGE pPage;
1474 int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1475 if (VBOX_SUCCESS(rc))
1476 {
1477 /*
1478 * Make shadow PTE entry.
1479 */
1480 RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1481 SHWPTE PteDst;
1482 PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1483 | (HCPhys & X86_PTE_PAE_PG_MASK);
1484 if (HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE))
1485 {
1486 if (!(HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL)))
1487 PteDst.n.u1Write = 0;
1488 else
1489 PteDst.u = 0;
1490 }
1491 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1492# ifdef PGMPOOL_WITH_USER_TRACKING
1493 if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1494 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1495# endif
1496 pPTDst->a[iPTDst] = PteDst;
1497
1498
1499# ifdef PGM_SYNC_DIRTY_BIT
1500 /*
1501 * If the page is not flagged as dirty and is writable, then make it read-only
1502 * at PD level, so we can set the dirty bit when the page is modified.
1503 *
1504 * ASSUMES that page access handlers are implemented on page table entry level.
1505 * Thus we will first catch the dirty access and set PDE.D and restart. If
1506 * there is an access handler, we'll trap again and let it work on the problem.
1507 */
1508 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1509 * As for invlpg, it simply frees the whole shadow PT.
1510 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1511 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1512 {
1513 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1514 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
1515 PdeDst.n.u1Write = 0;
1516 }
1517 else
1518 {
1519 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1520 PdeDst.n.u1Write = PdeSrc.n.u1Write;
1521 }
1522# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1523 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1524# else /* PAE */
1525 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1526# endif
1527# endif /* PGM_SYNC_DIRTY_BIT */
1528 Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1529 GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1530 PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1531 }
1532 else
1533 LogFlow(("PGM_GCPHYS_2_PTR %VGp (big) failed with %Vrc\n", GCPhys, rc));
1534 }
1535 return VINF_SUCCESS;
1536 }
1537# ifdef PGM_SYNC_ACCESSED_BIT
1538 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1539#endif
1540 }
1541 else
1542 {
1543 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1544 Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1545 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1546 }
1547
1548 /*
1549 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1550 * Yea, I'm lazy.
1551 */
1552 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1553# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1554 pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1555# else /* PAE */
1556 pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1557# endif
1558 PGM_INVL_GUEST_TLBS();
1559 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1560
1561#elif PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT
1562
1563# ifdef PGM_SYNC_N_PAGES
1564 /*
1565 * Get the shadow PDE, find the shadow page table in the pool.
1566 */
1567 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1568# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1569 X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1570# else /* PAE */
1571 X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1572# endif
1573 Assert(PdeDst.n.u1Present);
1574 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1575 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1576
1577# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1578 const unsigned offPTSrc = 0;
1579# else
1580 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1581# endif
1582
1583 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1584 if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1585 {
1586 /*
1587 * This code path is currently only taken when the caller is PGMTrap0eHandler
1588 * for non-present pages!
1589 *
1590 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1591 * deal with locality.
1592 */
1593 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1594 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1595 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1596 iPTDst = 0;
1597 else
1598 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1599 for (; iPTDst < iPTDstEnd; iPTDst++)
1600 {
1601 if (!pPTDst->a[iPTDst].n.u1Present)
1602 {
1603 VBOXPTE PteSrc;
1604
1605 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1606
1607 /* Fake the page table entry */
1608 PteSrc.u = GCPtrCurPage;
1609 PteSrc.n.u1Present = 1;
1610 PteSrc.n.u1Dirty = 1;
1611 PteSrc.n.u1Accessed = 1;
1612 PteSrc.n.u1Write = 1;
1613 PteSrc.n.u1User = 1;
1614
1615 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1616
1617 Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1618 GCPtrCurPage, PteSrc.n.u1Present,
1619 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1620 PteSrc.n.u1User & PdeSrc.n.u1User,
1621 (uint64_t)PteSrc.u,
1622 (uint64_t)pPTDst->a[iPTDst].u,
1623 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1624 }
1625 }
1626 }
1627 else
1628# endif /* PGM_SYNC_N_PAGES */
1629 {
1630 VBOXPTE PteSrc;
1631 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1632 RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1633
1634 /* Fake the page table entry */
1635 PteSrc.u = GCPtrCurPage;
1636 PteSrc.n.u1Present = 1;
1637 PteSrc.n.u1Dirty = 1;
1638 PteSrc.n.u1Accessed = 1;
1639 PteSrc.n.u1Write = 1;
1640 PteSrc.n.u1User = 1;
1641 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1642
1643 Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1644 GCPtrPage, PteSrc.n.u1Present,
1645 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1646 PteSrc.n.u1User & PdeSrc.n.u1User,
1647 (uint64_t)PteSrc.u,
1648 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1649 }
1650 return VINF_SUCCESS;
1651
1652#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1653 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1654 return VERR_INTERNAL_ERROR;
1655#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1656}
1657
1658
1659
1660#if PGM_WITH_PAGING(PGM_GST_TYPE)
1661
1662# ifdef PGM_SYNC_DIRTY_BIT
1663
1664/**
1665 * Investigate page fault and handle write protection page faults caused by
1666 * dirty bit tracking.
1667 *
1668 * @returns VBox status code.
1669 * @param pVM VM handle.
1670 * @param uErr Page fault error code.
1671 * @param pPdeDst Shadow page directory entry.
1672 * @param pPdeSrc Guest page directory entry.
1673 * @param GCPtrPage Guest context page address.
1674 */
1675PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1676{
1677 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1678 LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1679
1680 /*
1681 * Real page fault?
1682 */
1683 if ( (uErr & X86_TRAP_PF_RSVD)
1684 || !pPdeSrc->n.u1Present
1685#if PGM_GST_TYPE == PGM_TYPE_PAE
1686 || ((uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
1687#endif
1688 || ((uErr & X86_TRAP_PF_RW) && !pPdeSrc->n.u1Write)
1689 || ((uErr & X86_TRAP_PF_US) && !pPdeSrc->n.u1User) )
1690 {
1691# ifdef IN_GC
1692 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1693# endif
1694 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1695 LogFlow(("CheckPageFault: real page fault at %VGv (1)\n", GCPtrPage));
1696
1697 if (pPdeSrc->n.u1Present)
1698 {
1699 /* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1700 * See the 2nd case below as well.
1701 */
1702 if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1703 {
1704 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
1705 }
1706 else
1707 {
1708 /*
1709 * Map the guest page table.
1710 */
1711 PGSTPT pPTSrc;
1712 int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1713 if (VBOX_SUCCESS(rc))
1714 {
1715 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & GST_PT_MASK];
1716 const GSTPTE PteSrc = *pPteSrc;
1717 if (pPteSrc->n.u1Present)
1718 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
1719 }
1720 AssertRC(rc);
1721 }
1722 }
1723 return VINF_EM_RAW_GUEST_TRAP;
1724 }
1725
1726 /*
1727 * First check the easy case where the page directory has been marked read-only to track
1728 * the dirty bit of an emulated BIG page
1729 */
1730 if (pPdeSrc->b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE))
1731 {
1732 /* Mark guest page directory as accessed */
1733 pPdeSrc->b.u1Accessed = 1;
1734
1735 /*
1736 * Only write protection page faults are relevant here.
1737 */
1738 if (uErr & X86_TRAP_PF_RW)
1739 {
1740 /* Mark guest page directory as dirty (BIG page only). */
1741 pPdeSrc->b.u1Dirty = 1;
1742
1743 if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1744 {
1745 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1746
1747 Assert(pPdeSrc->b.u1Write);
1748
1749 pPdeDst->n.u1Write = 1;
1750 pPdeDst->n.u1Accessed = 1;
1751 pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1752 PGM_INVL_BIG_PG(GCPtrPage);
1753 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1754 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1755 }
1756 }
1757 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1758 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1759 }
1760 /* else: 4KB page table */
1761
1762 /*
1763 * Map the guest page table.
1764 */
1765 PGSTPT pPTSrc;
1766 int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1767 if (VBOX_SUCCESS(rc))
1768 {
1769 /*
1770 * Real page fault?
1771 */
1772 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> PAGE_SHIFT) & GST_PT_MASK];
1773 const GSTPTE PteSrc = *pPteSrc;
1774 if ( !PteSrc.n.u1Present
1775#if PGM_GST_TYPE == PGM_TYPE_PAE
1776 || ((uErr & X86_TRAP_PF_ID) && !PteSrc.n.u1NoExecute)
1777#endif
1778 || ((uErr & X86_TRAP_PF_RW) && !PteSrc.n.u1Write)
1779 || ((uErr & X86_TRAP_PF_US) && !PteSrc.n.u1User)
1780 )
1781 {
1782# ifdef IN_GC
1783 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1784# endif
1785 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1786 LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1787
1788 /* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1789 * See the 2nd case above as well.
1790 */
1791 if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1792 TRPMSetErrorCode(pVM, uErr | X86_TRAP_PF_P); /* page-level protection violation */
1793
1794 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1795 return VINF_EM_RAW_GUEST_TRAP;
1796 }
1797 LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1798
1799 /*
1800 * Set the accessed bits in the page directory and the page table.
1801 */
1802 pPdeSrc->n.u1Accessed = 1;
1803 pPteSrc->n.u1Accessed = 1;
1804
1805 /*
1806 * Only write protection page faults are relevant here.
1807 */
1808 if (uErr & X86_TRAP_PF_RW)
1809 {
1810 /* Write access, so mark guest entry as dirty. */
1811# if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1812 if (!pPteSrc->n.u1Dirty)
1813 STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1814 else
1815 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1816# endif
1817 pPteSrc->n.u1Dirty = 1;
1818
1819 if (pPdeDst->n.u1Present)
1820 {
1821 /* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1822 * Our individual shadow handlers will provide more information and force a fatal exit.
1823 */
1824 if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1825 {
1826 LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1827 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1828 return VINF_SUCCESS;
1829 }
1830
1831 /*
1832 * Map shadow page table.
1833 */
1834 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1835 if (pShwPage)
1836 {
1837 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1838 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1839 if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1840 && (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1841 {
1842 LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1843# ifdef VBOX_STRICT
1844 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
1845 if (pPage)
1846 AssertMsg(!(pPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)), /** @todo PAGE FLAGS */
1847 ("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1848# endif
1849 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1850
1851 Assert(pPteSrc->n.u1Write);
1852
1853 pPteDst->n.u1Write = 1;
1854 pPteDst->n.u1Dirty = 1;
1855 pPteDst->n.u1Accessed = 1;
1856 pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1857 PGM_INVL_PG(GCPtrPage);
1858
1859 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1860 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1861 }
1862 }
1863 else
1864 AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
1865 }
1866 }
1867/** @todo Optimize accessed bit emulation? */
1868# ifdef VBOX_STRICT
1869 /*
1870 * Sanity check.
1871 */
1872 else if ( !pPteSrc->n.u1Dirty
1873 && (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
1874 && pPdeDst->n.u1Present)
1875 {
1876 PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1877 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1878 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1879 if ( pPteDst->n.u1Present
1880 && pPteDst->n.u1Write)
1881 LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
1882 }
1883# endif /* VBOX_STRICT */
1884 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1885 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1886 }
1887 AssertRC(rc);
1888 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1889 return rc;
1890}
1891
1892# endif
1893
1894#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
1895
1896
1897/**
1898 * Sync a shadow page table.
1899 *
1900 * The shadow page table is not present. This includes the case where
1901 * there is a conflict with a mapping.
1902 *
1903 * @returns VBox status code.
1904 * @param pVM VM handle.
1905 * @param iPD Page directory index.
1906 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
1907 * Assume this is a temporary mapping.
1908 * @param GCPtrPage GC Pointer of the page that caused the fault
1909 */
1910PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
1911{
1912 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1913 STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
1914 LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
1915
1916#if PGM_GST_TYPE == PGM_TYPE_32BIT
1917
1918# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
1919# error "Invalid shadow mode for 32-bit guest mode!"
1920# endif
1921
1922 /*
1923 * Validate input a little bit.
1924 */
1925 Assert(iPDSrc == (GCPtrPage >> GST_PD_SHIFT));
1926# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1927 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
1928# else
1929 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
1930# endif
1931 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1932 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
1933 SHWPDE PdeDst = *pPdeDst;
1934
1935 /*
1936 * Check for conflicts.
1937 * GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
1938 * HC: Simply resolve the conflict.
1939 */
1940 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1941 {
1942 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1943# ifndef IN_RING3
1944 Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
1945 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1946 return VERR_ADDRESS_CONFLICT;
1947# else
1948 PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
1949 Assert(pMapping);
1950 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPDSrc);
1951 if (VBOX_FAILURE(rc))
1952 {
1953 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1954 return rc;
1955 }
1956 PdeDst = *pPdeDst;
1957# endif
1958 }
1959 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
1960
1961 /*
1962 * Sync page directory entry.
1963 */
1964 int rc = VINF_SUCCESS;
1965 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1966 if (PdeSrc.n.u1Present)
1967 {
1968 /*
1969 * Allocate & map the page table.
1970 */
1971 PSHWPT pPTDst;
1972 const bool fPageTable = !PdeSrc.b.u1Size || !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1973 PPGMPOOLPAGE pShwPage;
1974 RTGCPHYS GCPhys;
1975 if (fPageTable)
1976 {
1977 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1978# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1979 GCPhys |= (iPDDst & 1) * (PAGE_SIZE / 2);
1980# endif
1981 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1982 }
1983 else
1984 {
1985 GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
1986# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1987 GCPhys |= GCPtrPage & RT_BIT(X86_PAGE_2M_SHIFT);
1988# endif
1989 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
1990 }
1991 if (rc == VINF_SUCCESS)
1992 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1993 else if (rc == VINF_PGM_CACHED_PAGE)
1994 {
1995 /*
1996 * The PT was cached, just hook it up.
1997 */
1998 if (fPageTable)
1999 PdeDst.u = pShwPage->Core.Key
2000 | (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2001 else
2002 {
2003 PdeDst.u = pShwPage->Core.Key
2004 | (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2005# ifdef PGM_SYNC_DIRTY_BIT /* (see explanation and assumtions further down.) */
2006 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2007 {
2008 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2009 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2010 PdeDst.b.u1Write = 0;
2011 }
2012# endif
2013 }
2014 *pPdeDst = PdeDst;
2015 return VINF_SUCCESS;
2016 }
2017 else if (rc == VERR_PGM_POOL_FLUSHED)
2018 return VINF_PGM_SYNC_CR3;
2019 else
2020 AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2021 PdeDst.u &= X86_PDE_AVL_MASK;
2022 PdeDst.u |= pShwPage->Core.Key;
2023
2024# ifdef PGM_SYNC_DIRTY_BIT
2025 /*
2026 * Page directory has been accessed (this is a fault situation, remember).
2027 */
2028 pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2029# endif
2030 if (fPageTable)
2031 {
2032 /*
2033 * Page table - 4KB.
2034 *
2035 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2036 */
2037 Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2038 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2039 PGSTPT pPTSrc;
2040 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2041 if (VBOX_SUCCESS(rc))
2042 {
2043 /*
2044 * Start by syncing the page directory entry so CSAM's TLB trick works.
2045 */
2046 PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK | X86_PDE_AVL_MASK))
2047 | (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2048 *pPdeDst = PdeDst;
2049
2050 /*
2051 * Directory/page user or supervisor privilege: (same goes for read/write)
2052 *
2053 * Directory Page Combined
2054 * U/S U/S U/S
2055 * 0 0 0
2056 * 0 1 0
2057 * 1 0 0
2058 * 1 1 1
2059 *
2060 * Simple AND operation. Table listed for completeness.
2061 *
2062 */
2063 STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
2064# ifdef PGM_SYNC_N_PAGES
2065 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2066 unsigned iPTDst = iPTBase;
2067 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
2068 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2069 iPTDst = 0;
2070 else
2071 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2072# else /* !PGM_SYNC_N_PAGES */
2073 unsigned iPTDst = 0;
2074 const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
2075# endif /* !PGM_SYNC_N_PAGES */
2076# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2077 const unsigned offPTSrc = 0;
2078# else
2079 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2080# endif
2081 for (; iPTDst < iPTDstEnd; iPTDst++)
2082 {
2083 const unsigned iPTSrc = iPTDst + offPTSrc;
2084 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2085
2086 if (PteSrc.n.u1Present) /* we've already cleared it above */
2087 {
2088#ifndef IN_RING0
2089 /*
2090 * Assuming kernel code will be marked as supervisor - and not as user level
2091 * and executed using a conforming code selector - And marked as readonly.
2092 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
2093 */
2094 if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW | X86_PTE_US))
2095 || !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)))
2096 || pgmRamTestFlags(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK,
2097 MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)
2098 )
2099#endif
2100 PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2101 Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
2102 (RTGCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)),
2103 PteSrc.n.u1Present,
2104 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2105 PteSrc.n.u1User & PdeSrc.n.u1User,
2106 (uint64_t)PteSrc.u,
2107 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2108 (PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
2109 }
2110 } /* for PTEs */
2111 }
2112 }
2113 else
2114 {
2115 /*
2116 * Big page - 2/4MB.
2117 *
2118 * We'll walk the ram range list in parallel and optimize lookups.
2119 * We will only sync on shadow page table at a time.
2120 */
2121 STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
2122
2123 /**
2124 * @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2125 */
2126
2127 /*
2128 * Start by syncing the page directory entry.
2129 */
2130 PdeDst.u = (PdeDst.u & (X86_PDE_PAE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2131 | (PdeSrc.u & ~(X86_PDE_PAE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2132
2133# ifdef PGM_SYNC_DIRTY_BIT
2134 /*
2135 * If the page is not flagged as dirty and is writable, then make it read-only
2136 * at PD level, so we can set the dirty bit when the page is modified.
2137 *
2138 * ASSUMES that page access handlers are implemented on page table entry level.
2139 * Thus we will first catch the dirty access and set PDE.D and restart. If
2140 * there is an access handler, we'll trap again and let it work on the problem.
2141 */
2142 /** @todo move the above stuff to a section in the PGM documentation. */
2143 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2144 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2145 {
2146 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2147 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2148 PdeDst.b.u1Write = 0;
2149 }
2150# endif /* PGM_SYNC_DIRTY_BIT */
2151 *pPdeDst = PdeDst;
2152
2153 /*
2154 * Fill the shadow page table.
2155 */
2156 /* Get address and flags from the source PDE. */
2157 SHWPTE PteDstBase;
2158 PteDstBase.u = PdeSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT);
2159
2160 /* Loop thru the entries in the shadow PT. */
2161 const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2162 Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2163 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2164 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2165 PPGMRAMRANGE pRam = CTXSUFF(pVM->pgm.s.pRamRanges);
2166 unsigned iPTDst = 0;
2167 while (iPTDst < ELEMENTS(pPTDst->a))
2168 {
2169 /* Advance ram range list. */
2170 while (pRam && GCPhys > pRam->GCPhysLast)
2171 pRam = CTXSUFF(pRam->pNext);
2172 if (pRam && GCPhys >= pRam->GCPhys)
2173 {
2174 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2175 do
2176 {
2177 /* Make shadow PTE. */
2178 PPGMPAGE pPage = &pRam->aPages[iHCPage];
2179 SHWPTE PteDst;
2180
2181 /* Make sure the RAM has already been allocated. */
2182 if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2183 {
2184 if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2185 {
2186# ifdef IN_RING3
2187 int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2188# else
2189 int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2190# endif
2191 if (rc != VINF_SUCCESS)
2192 return rc;
2193 }
2194 }
2195
2196 if (pPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)) /** @todo PAGE FLAGS */
2197 {
2198 if (!(pPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL))) /** @todo PAGE FLAGS */
2199 {
2200 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2201 PteDst.n.u1Write = 0;
2202 }
2203 else
2204 PteDst.u = 0;
2205 }
2206# ifndef IN_RING0
2207 /*
2208 * Assuming kernel code will be marked as supervisor and not as user level and executed
2209 * using a conforming code selector. Don't check for readonly, as that implies the whole
2210 * 4MB can be code or readonly data. Linux enables write access for its large pages.
2211 */
2212 else if ( !PdeSrc.n.u1User
2213 && CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))))
2214 PteDst.u = 0;
2215# endif
2216 else
2217 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2218# ifdef PGMPOOL_WITH_USER_TRACKING
2219 if (PteDst.n.u1Present)
2220 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2221# endif
2222 /* commit it */
2223 pPTDst->a[iPTDst] = PteDst;
2224 Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2225 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2226 PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2227
2228 /* advance */
2229 GCPhys += PAGE_SIZE;
2230 iHCPage++;
2231 iPTDst++;
2232 } while ( iPTDst < ELEMENTS(pPTDst->a)
2233 && GCPhys <= pRam->GCPhysLast);
2234 }
2235 else if (pRam)
2236 {
2237 Log(("Invalid pages at %VGp\n", GCPhys));
2238 do
2239 {
2240 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2241 GCPhys += PAGE_SIZE;
2242 iPTDst++;
2243 } while ( iPTDst < ELEMENTS(pPTDst->a)
2244 && GCPhys < pRam->GCPhys);
2245 }
2246 else
2247 {
2248 Log(("Invalid pages at %VGp (2)\n", GCPhys));
2249 for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2250 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2251 }
2252 } /* while more PTEs */
2253 } /* 4KB / 4MB */
2254 }
2255 else
2256 AssertRelease(!PdeDst.n.u1Present);
2257
2258 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2259# ifdef IN_GC
2260 if (VBOX_FAILURE(rc))
2261 STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2262# endif
2263 return rc;
2264
2265#elif PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT
2266
2267 int rc = VINF_SUCCESS;
2268
2269 /*
2270 * Validate input a little bit.
2271 */
2272# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2273 PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2274# else
2275 PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2276# endif
2277 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2278 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2279 SHWPDE PdeDst = *pPdeDst;
2280
2281 Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2282 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2283
2284 GSTPDE PdeSrc;
2285 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2286 PdeSrc.n.u1Present = 1;
2287 PdeSrc.n.u1Write = 1;
2288 PdeSrc.n.u1Accessed = 1;
2289 PdeSrc.n.u1User = 1;
2290
2291 /*
2292 * Allocate & map the page table.
2293 */
2294 PSHWPT pPTDst;
2295 PPGMPOOLPAGE pShwPage;
2296 RTGCPHYS GCPhys;
2297
2298 /* Virtual address = physical address */
2299 GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK_32;
2300 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2301
2302 if ( rc == VINF_SUCCESS
2303 || rc == VINF_PGM_CACHED_PAGE)
2304 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2305 else
2306 AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2307
2308 PdeDst.u &= X86_PDE_AVL_MASK;
2309 PdeDst.u |= pShwPage->Core.Key;
2310 PdeDst.n.u1Present = 1;
2311 *pPdeDst = PdeDst;
2312
2313 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2314 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2315 return rc;
2316
2317#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2318
2319 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2320 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2321 return VERR_INTERNAL_ERROR;
2322#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2323}
2324
2325
2326
2327/**
2328 * Prefetch a page/set of pages.
2329 *
2330 * Typically used to sync commonly used pages before entering raw mode
2331 * after a CR3 reload.
2332 *
2333 * @returns VBox status code.
2334 * @param pVM VM handle.
2335 * @param GCPtrPage Page to invalidate.
2336 */
2337PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2338{
2339#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2340
2341# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2342# error "Invalid shadow mode for 32-bit guest mode!"
2343# endif
2344
2345 /*
2346 * Check that all Guest levels thru the PDE are present, getting the
2347 * PD and PDE in the processes.
2348 */
2349 int rc = VINF_SUCCESS;
2350# if PGM_WITH_PAGING(PGM_GST_TYPE)
2351# if PGM_GST_TYPE == PGM_TYPE_32BIT
2352 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2353 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2354# else /* PAE */
2355 unsigned iPDSrc
2356 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2357# endif
2358# else
2359 PGSTPD pPDSrc = NULL;
2360 const unsigned iPDSrc = 0;
2361# endif
2362
2363# if PGM_WITH_PAGING(PGM_GST_TYPE)
2364 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2365# else
2366 GSTPDE PdeSrc;
2367 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2368 PdeSrc.n.u1Present = 1;
2369 PdeSrc.n.u1Write = 1;
2370 PdeSrc.n.u1Accessed = 1;
2371 PdeSrc.n.u1User = 1;
2372# endif
2373
2374# ifdef PGM_SYNC_ACCESSED_BIT
2375 if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2376# else
2377 if (PdeSrc.n.u1Present)
2378# endif
2379 {
2380# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2381 const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2382# else
2383 const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2384# endif
2385 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2386 {
2387 if (!PdeDst.n.u1Present)
2388 /** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2389 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2390 else
2391 {
2392 /** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2393 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2394 * makes no sense to prefetch more than one page.
2395 */
2396 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2397 if (VBOX_SUCCESS(rc))
2398 rc = VINF_SUCCESS;
2399 }
2400 }
2401 }
2402 return rc;
2403
2404#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2405
2406 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2407 return VERR_INTERNAL_ERROR;
2408#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2409}
2410
2411
2412
2413
2414/**
2415 * Syncs a page during a PGMVerifyAccess() call.
2416 *
2417 * @returns VBox status code (informational included).
2418 * @param GCPtrPage The address of the page to sync.
2419 * @param fPage The effective guest page flags.
2420 * @param uErr The trap error code.
2421 */
2422PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2423{
2424 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2425
2426#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2427
2428# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2429# error "Invalid shadow mode for 32-bit guest mode!"
2430# endif
2431
2432#ifndef IN_RING0
2433 if (!(fPage & X86_PTE_US))
2434 {
2435 /*
2436 * Mark this page as safe.
2437 */
2438 /** @todo not correct for pages that contain both code and data!! */
2439 Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2440 CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2441 }
2442#endif
2443 /*
2444 * Get guest PD and index.
2445 */
2446
2447# if PGM_WITH_PAGING(PGM_GST_TYPE)
2448# if PGM_GST_TYPE == PGM_TYPE_32BIT
2449 const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2450 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2451# else /* PAE */
2452 unsigned iPDSrc;
2453 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2454# endif
2455# else
2456 PGSTPD pPDSrc = NULL;
2457 const unsigned iPDSrc = 0;
2458# endif
2459 int rc = VINF_SUCCESS;
2460
2461 /*
2462 * First check if the shadow pd is present.
2463 */
2464# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2465 PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2466# else
2467 PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2468# endif
2469 if (!pPdeDst->n.u1Present)
2470 {
2471 rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2472 AssertRC(rc);
2473 if (rc != VINF_SUCCESS)
2474 return rc;
2475 }
2476
2477# if PGM_WITH_PAGING(PGM_GST_TYPE)
2478 /* Check for dirty bit fault */
2479 rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2480 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2481 Log(("PGMVerifyAccess: success (dirty)\n"));
2482 else
2483 {
2484 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2485#else
2486 {
2487 GSTPDE PdeSrc;
2488 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2489 PdeSrc.n.u1Present = 1;
2490 PdeSrc.n.u1Write = 1;
2491 PdeSrc.n.u1Accessed = 1;
2492 PdeSrc.n.u1User = 1;
2493
2494#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2495 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2496 if (uErr & X86_TRAP_PF_US)
2497 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2498 else /* supervisor */
2499 STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2500
2501 rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2502 if (VBOX_SUCCESS(rc))
2503 {
2504 /* Page was successfully synced */
2505 Log2(("PGMVerifyAccess: success (sync)\n"));
2506 rc = VINF_SUCCESS;
2507 }
2508 else
2509 {
2510 Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2511 return VINF_EM_RAW_GUEST_TRAP;
2512 }
2513 }
2514 return rc;
2515
2516#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2517
2518 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2519 return VERR_INTERNAL_ERROR;
2520#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2521}
2522
2523
2524#if PGM_GST_TYPE == PGM_TYPE_32BIT
2525# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE
2526/**
2527 * Figures out which kind of shadow page this guest PDE warrants.
2528 *
2529 * @returns Shadow page kind.
2530 * @param pPdeSrc The guest PDE in question.
2531 * @param cr4 The current guest cr4 value.
2532 */
2533DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
2534{
2535 if (!pPdeSrc->n.u1Size || !(cr4 & X86_CR4_PSE))
2536 return BTH_PGMPOOLKIND_PT_FOR_PT;
2537 //switch (pPdeSrc->u & (X86_PDE4M_RW | X86_PDE4M_US /*| X86_PDE4M_PAE_NX*/))
2538 //{
2539 // case 0:
2540 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2541 // case X86_PDE4M_RW:
2542 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2543 // case X86_PDE4M_US:
2544 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2545 // case X86_PDE4M_RW | X86_PDE4M_US:
2546 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2547# if 0
2548 // case X86_PDE4M_PAE_NX:
2549 // return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2550 // case X86_PDE4M_RW | X86_PDE4M_PAE_NX:
2551 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2552 // case X86_PDE4M_US | X86_PDE4M_PAE_NX:
2553 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2554 // case X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PAE_NX:
2555 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2556# endif
2557 return BTH_PGMPOOLKIND_PT_FOR_BIG;
2558 //}
2559}
2560# endif
2561#endif
2562
2563#undef MY_STAM_COUNTER_INC
2564#define MY_STAM_COUNTER_INC(a) do { } while (0)
2565
2566
2567/**
2568 * Syncs the paging hierarchy starting at CR3.
2569 *
2570 * @returns VBox status code, no specials.
2571 * @param pVM The virtual machine.
2572 * @param cr0 Guest context CR0 register
2573 * @param cr3 Guest context CR3 register
2574 * @param cr4 Guest context CR4 register
2575 * @param fGlobal Including global page directories or not
2576 */
2577PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint32_t cr0, uint32_t cr3, uint32_t cr4, bool fGlobal)
2578{
2579#if PGM_GST_TYPE == PGM_TYPE_32BIT
2580# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE
2581 if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2582 fGlobal = true; /* Change this CR3 reload to be a global one. */
2583# endif
2584#endif
2585
2586 /*
2587 * Update page access handlers.
2588 * The virtual are always flushed, while the physical are only on demand.
2589 * WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2590 * have to look into that later because it will have a bad influence on the performance.
2591 * @note SvL: There's no need for that. Just invalidate the virtual range(s).
2592 * bird: Yes, but that won't work for aliases.
2593 */
2594 /** @todo this MUST go away. See #1557. */
2595 STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2596 PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2597 STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2598
2599#ifdef PGMPOOL_WITH_MONITORING
2600 /*
2601 * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2602 * Occationally we will have to clear all the shadow page tables because we wanted
2603 * to monitor a page which was mapped by too many shadowed page tables. This operation
2604 * sometimes refered to as a 'lightweight flush'.
2605 */
2606 if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2607 pgmPoolMonitorModifiedClearAll(pVM);
2608 else
2609 {
2610# ifdef IN_RING3
2611 pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2612 pgmPoolClearAll(pVM);
2613# else
2614 LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2615 return VINF_PGM_SYNC_CR3;
2616# endif
2617 }
2618#endif
2619
2620 Assert(fGlobal || (cr4 & X86_CR4_PGE));
2621 MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2622
2623#if PGM_GST_TYPE == PGM_TYPE_32BIT
2624# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE
2625 /*
2626 * Get page directory addresses.
2627 */
2628# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2629 PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2630# else
2631 PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2632# endif
2633
2634# if PGM_GST_TYPE == PGM_TYPE_32BIT
2635 PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2636# else /* PAE */
2637 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, 0);
2638# endif
2639
2640 Assert(pPDSrc);
2641#ifndef IN_GC
2642 Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2643#endif
2644
2645 /*
2646 * Iterate the page directory.
2647 */
2648 PPGMMAPPING pMapping;
2649 unsigned iPdNoMapping;
2650 const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2651 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2652
2653 /* Only check mappings if they are supposed to be put into the shadow page table. */
2654 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2655 {
2656 pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2657 iPdNoMapping = (pMapping) ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U; /** PAE todo */
2658 }
2659 else
2660 {
2661 pMapping = 0;
2662 iPdNoMapping = ~0U;
2663 }
2664
2665 for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2666 {
2667# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2668 Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2669# else
2670 Assert(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst);
2671# endif
2672 register GSTPDE PdeSrc = pPDSrc->a[iPD];
2673 if ( PdeSrc.n.u1Present
2674 && (PdeSrc.n.u1User || fRawR0Enabled))
2675 {
2676 /*
2677 * Check for conflicts with GC mappings.
2678 */
2679 if (iPD == iPdNoMapping)
2680 {
2681 if (pVM->pgm.s.fMappingsFixed)
2682 {
2683 /* It's fixed, just skip the mapping. */
2684 const unsigned cPTs = pMapping->cPTs;
2685 iPD += cPTs - 1;
2686 pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2687 pMapping = pMapping->CTXALLSUFF(pNext);
2688 iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2689 continue;
2690 }
2691
2692#ifdef IN_RING3
2693 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2694 if (VBOX_FAILURE(rc))
2695 return rc;
2696
2697 /*
2698 * Update iPdNoMapping and pMapping.
2699 */
2700 pMapping = pVM->pgm.s.pMappingsR3;
2701 while (pMapping && pMapping->GCPtr < (iPD << X86_PD_SHIFT))
2702 pMapping = pMapping->pNextR3;
2703 iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2704#else
2705 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2706 return VINF_PGM_SYNC_CR3;
2707#endif
2708 }
2709
2710 /*
2711 * Sync page directory entry.
2712 *
2713 * The current approach is to allocated the page table but to set
2714 * the entry to not-present and postpone the page table synching till
2715 * it's actually used.
2716 */
2717# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2718 const unsigned iPdShw = iPD; NOREF(iPdShw);
2719# else
2720 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2721# endif
2722 {
2723 SHWPDE PdeDst = *pPDEDst;
2724 if (PdeDst.n.u1Present)
2725 {
2726 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2727 RTGCPHYS GCPhys;
2728 if ( !PdeSrc.b.u1Size
2729 || !(cr4 & X86_CR4_PSE))
2730 {
2731 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2732# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2733 GCPhys |= i * (PAGE_SIZE / 2);
2734# endif
2735 }
2736 else
2737 {
2738 GCPhys = PdeSrc.u & GST_PDE4M_PG_MASK;
2739# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2740 GCPhys |= i * X86_PAGE_2M_SIZE;
2741# endif
2742 }
2743
2744 if ( pShwPage->GCPhys == GCPhys
2745 && pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2746 && ( pShwPage->fCached
2747 || ( !fGlobal
2748 && ( false
2749# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2750 || ( (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
2751 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE)) /* global 2/4MB page. */
2752 || ( !pShwPage->fSeenNonGlobal
2753 && (cr4 & X86_CR4_PGE))
2754# endif
2755 )
2756 )
2757 )
2758 && ( (PdeSrc.u & (X86_PDE_US | X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US | X86_PDE_RW))
2759 || ( (cr4 & X86_CR4_PSE)
2760 && ((PdeSrc.u & (X86_PDE_US | X86_PDE4M_PS | X86_PDE4M_D)) | PGM_PDFLAGS_TRACK_DIRTY)
2761 == ((PdeDst.u & (X86_PDE_US | X86_PDE_RW | PGM_PDFLAGS_TRACK_DIRTY)) | X86_PDE4M_PS))
2762 )
2763 )
2764 {
2765# ifdef VBOX_WITH_STATISTICS
2766 if ( !fGlobal
2767 && (PdeSrc.u & (X86_PDE4M_PS | X86_PDE4M_G)) == (X86_PDE4M_PS | X86_PDE4M_G)
2768 && (cr4 & (X86_CR4_PGE | X86_CR4_PSE)) == (X86_CR4_PGE | X86_CR4_PSE))
2769 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2770 else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2771 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2772 else
2773 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2774# endif /* VBOX_WITH_STATISTICS */
2775/** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2776 * The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2777//# ifdef PGMPOOL_WITH_CACHE
2778// pgmPoolCacheUsed(pPool, pShwPage);
2779//# endif
2780 }
2781 else
2782 {
2783 pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
2784 pPDEDst->u = 0;
2785 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
2786 }
2787 }
2788 else
2789 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
2790 pPDEDst++;
2791 }
2792 }
2793 else if (iPD != iPdNoMapping)
2794 {
2795 /*
2796 * Check if there is any page directory to mark not present here.
2797 */
2798# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2799 const unsigned iPdShw = iPD; NOREF(iPdShw);
2800# else
2801 for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2802# endif
2803 {
2804 if (pPDEDst->n.u1Present)
2805 {
2806 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
2807 pPDEDst->u = 0;
2808 MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
2809 }
2810 pPDEDst++;
2811 }
2812 }
2813 else
2814 {
2815 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2816 const unsigned cPTs = pMapping->cPTs;
2817 if (pVM->pgm.s.fMappingsFixed)
2818 {
2819 /* It's fixed, just skip the mapping. */
2820 pMapping = pMapping->CTXALLSUFF(pNext);
2821 iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2822 }
2823 else
2824 {
2825 /*
2826 * Check for conflicts for subsequent pagetables
2827 * and advance to the next mapping.
2828 */
2829 iPdNoMapping = ~0U;
2830 unsigned iPT = cPTs;
2831 while (iPT-- > 1)
2832 {
2833 if ( pPDSrc->a[iPD + iPT].n.u1Present
2834 && (pPDSrc->a[iPD + iPT].n.u1User || fRawR0Enabled))
2835 {
2836# ifdef IN_RING3
2837 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD);
2838 if (VBOX_FAILURE(rc))
2839 return rc;
2840
2841 /*
2842 * Update iPdNoMapping and pMapping.
2843 */
2844 pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2845 while (pMapping && pMapping->GCPtr < (iPD << X86_PD_SHIFT))
2846 pMapping = pMapping->CTXALLSUFF(pNext);
2847 iPdNoMapping = pMapping ? pMapping->GCPtr >> X86_PD_SHIFT : ~0U;
2848 break;
2849# else
2850 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2851 return VINF_PGM_SYNC_CR3;
2852# endif
2853 }
2854 }
2855 if (iPdNoMapping == ~0U && pMapping)
2856 {
2857 pMapping = pMapping->CTXALLSUFF(pNext);
2858 if (pMapping)
2859 iPdNoMapping = pMapping->GCPtr >> X86_PD_SHIFT;
2860 }
2861 }
2862 /* advance. */
2863 iPD += cPTs - 1;
2864 pPDEDst += cPTs + (PGM_SHW_TYPE != PGM_TYPE_32BIT) * cPTs;
2865 }
2866
2867 } /* for iPD */
2868# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2869# error "Guest 32-bit mode and shadow AMD64 mode doesn't add up!"
2870# endif
2871
2872 return VINF_SUCCESS;
2873
2874#elif PGM_GST_TYPE == PGM_TYPE_PAE
2875# if PGM_SHW_TYPE == PGM_TYPE_PAE
2876//# error not implemented
2877 return VERR_INTERNAL_ERROR;
2878
2879# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2880# error "Guest PAE mode, but not the shadow mode ; 32bit - maybe, but amd64 no."
2881# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2882
2883#elif PGM_GST_TYPE == PGM_TYPE_AMD64
2884# if PGM_SHW_TYPE == PGM_TYPE_AMD64
2885//# error not implemented
2886 return VERR_INTERNAL_ERROR;
2887
2888# else /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2889# error "Guest AMD64 mode, but not the shadow mode - that can't be right!"
2890# endif /* PGM_SHW_TYPE != PGM_TYPE_AMD64 */
2891
2892#else /* guest real and protected mode */
2893
2894 return VINF_SUCCESS;
2895#endif
2896}
2897
2898
2899
2900
2901#ifdef VBOX_STRICT
2902#ifdef IN_GC
2903# undef AssertMsgFailed
2904# define AssertMsgFailed Log
2905#endif
2906#ifdef IN_RING3
2907# include <VBox/dbgf.h>
2908
2909/**
2910 * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
2911 *
2912 * @returns VBox status code (VINF_SUCCESS).
2913 * @param pVM The VM handle.
2914 * @param cr3 The root of the hierarchy.
2915 * @param crr The cr4, only PAE and PSE is currently used.
2916 * @param fLongMode Set if long mode, false if not long mode.
2917 * @param cMaxDepth Number of levels to dump.
2918 * @param pHlp Pointer to the output functions.
2919 */
2920__BEGIN_DECLS
2921PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
2922__END_DECLS
2923
2924#endif
2925
2926/**
2927 * Checks that the shadow page table is in sync with the guest one.
2928 *
2929 * @returns The number of errors.
2930 * @param pVM The virtual machine.
2931 * @param cr3 Guest context CR3 register
2932 * @param cr4 Guest context CR4 register
2933 * @param GCPtr Where to start. Defaults to 0.
2934 * @param cb How much to check. Defaults to everything.
2935 */
2936PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint32_t cr3, uint32_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
2937{
2938 unsigned cErrors = 0;
2939
2940#if PGM_GST_TYPE == PGM_TYPE_32BIT
2941
2942# if PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2943# error "Invalid shadow mode for 32-bit guest paging."
2944# endif
2945
2946 PPGM pPGM = &pVM->pgm.s;
2947 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
2948 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
2949 RTHCPHYS HCPhys; /* general usage. */
2950 int rc;
2951
2952 /*
2953 * Check that the Guest CR3 and all it's mappings are correct.
2954 */
2955 AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & X86_CR3_PAGE_MASK),
2956 ("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
2957 false);
2958 rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
2959 AssertRCReturn(rc, 1);
2960 HCPhys = NIL_RTHCPHYS;
2961 rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & X86_CR3_PAGE_MASK, &HCPhys);
2962 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
2963# ifdef IN_RING3
2964 RTGCPHYS GCPhys;
2965 rc = PGMR3DbgHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
2966 AssertRCReturn(rc, 1);
2967 AssertMsgReturn((cr3 & X86_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
2968# endif
2969 const X86PD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
2970
2971 /*
2972 * Get and check the Shadow CR3.
2973 */
2974# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2975 const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
2976 unsigned cPDEs = ELEMENTS(pPDDst->a);
2977# else
2978 const X86PDPAE *pPDDst = pPGM->CTXMID(ap,PaePDs[0]); /* use it as a 2048 entry PD */
2979 unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
2980# endif
2981 if (cb != ~(RTGCUINTPTR)0)
2982 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
2983
2984/** @todo call the other two PGMAssert*() functions. */
2985
2986 /*
2987 * Iterate the shadow page directory.
2988 */
2989 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
2990 unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
2991 cPDEs += iPDDst;
2992 for (;
2993 iPDDst < cPDEs;
2994 iPDDst++, GCPtr += _4G / cPDEs)
2995 {
2996 const SHWPDE PdeDst = pPDDst->a[iPDDst];
2997 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2998 {
2999 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3000 if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3001 {
3002 AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3003 cErrors++;
3004 continue;
3005 }
3006 }
3007 else if ( (PdeDst.u & X86_PDE_P)
3008 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3009 )
3010 {
3011 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3012 PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3013 if (!pPoolPage)
3014 {
3015 AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
3016 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3017 cErrors++;
3018 continue;
3019 }
3020 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3021
3022 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
3023 {
3024 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
3025 GCPtr, (uint64_t)PdeDst.u));
3026 cErrors++;
3027 }
3028
3029 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
3030 {
3031 AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
3032 GCPtr, (uint64_t)PdeDst.u));
3033 cErrors++;
3034 }
3035
3036 const X86PDE PdeSrc = pPDSrc->a[iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)];
3037 if (!PdeSrc.n.u1Present)
3038 {
3039 AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3040 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3041 cErrors++;
3042 continue;
3043 }
3044
3045 if ( !PdeSrc.b.u1Size
3046 || !(cr4 & X86_CR4_PSE))
3047 {
3048 GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3049# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3050 GCPhysGst |= (iPDDst & 1) * (PAGE_SIZE / 2);
3051# endif
3052 }
3053 else
3054 {
3055 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3056 {
3057 AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3058 GCPtr, (uint64_t)PdeSrc.u));
3059 cErrors++;
3060 continue;
3061 }
3062 GCPhysGst = PdeSrc.u & GST_PDE4M_PG_MASK;
3063# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3064 GCPhysGst |= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3065# endif
3066 }
3067
3068 if ( pPoolPage->enmKind
3069 != (!PdeSrc.b.u1Size || !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3070 {
3071 AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
3072 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3073 cErrors++;
3074 }
3075
3076 PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3077 if (!pPhysPage)
3078 {
3079 AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3080 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3081 cErrors++;
3082 continue;
3083 }
3084
3085 if (GCPhysGst != pPoolPage->GCPhys)
3086 {
3087 AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
3088 GCPhysGst, pPoolPage->GCPhys, GCPtr));
3089 cErrors++;
3090 continue;
3091 }
3092
3093 if ( !PdeSrc.b.u1Size
3094 || !(cr4 & X86_CR4_PSE))
3095 {
3096 /*
3097 * Page Table.
3098 */
3099 const GSTPT *pPTSrc;
3100 rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3101 if (VBOX_FAILURE(rc))
3102 {
3103 AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3104 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3105 cErrors++;
3106 continue;
3107 }
3108 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
3109 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
3110 {
3111 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3112 // (This problem will go away when/if we shadow multiple CR3s.)
3113 AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3114 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3115 cErrors++;
3116 continue;
3117 }
3118 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3119 {
3120 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
3121 GCPtr, (uint64_t)PdeDst.u));
3122 cErrors++;
3123 continue;
3124 }
3125
3126 /* iterate the page table. */
3127# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3128 const unsigned offPTSrc = 0;
3129# else
3130 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3131# endif
3132 for (unsigned iPT = 0, off = 0;
3133 iPT < ELEMENTS(pPTDst->a);
3134 iPT++, off += PAGE_SIZE)
3135 {
3136 const SHWPTE PteDst = pPTDst->a[iPT];
3137
3138 /* skip not-present entries. */
3139 if (!(PteDst.u & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3140 continue;
3141 Assert(PteDst.n.u1Present);
3142
3143 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3144 if (!PteSrc.n.u1Present)
3145 {
3146#ifdef IN_RING3
3147 PGMAssertHandlerAndFlagsInSync(pVM);
3148 PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3149#endif
3150 AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
3151 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3152 (PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3153 cErrors++;
3154 continue;
3155 }
3156
3157 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
3158# if 1 /** @todo sync accessed bit properly... */
3159 fIgnoreFlags |= X86_PTE_A;
3160# endif
3161
3162 /* match the physical addresses */
3163 HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3164 GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3165
3166# ifdef IN_RING3
3167 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3168 if (VBOX_FAILURE(rc))
3169 {
3170 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3171 {
3172 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3173 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3174 cErrors++;
3175 continue;
3176 }
3177 }
3178 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3179 {
3180 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3181 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3182 cErrors++;
3183 continue;
3184 }
3185# endif
3186
3187 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3188 if (!pPhysPage)
3189 {
3190# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3191 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3192 {
3193 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3194 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3195 cErrors++;
3196 continue;
3197 }
3198# endif
3199 if (PteDst.n.u1Write)
3200 {
3201 AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3202 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3203 cErrors++;
3204 }
3205 fIgnoreFlags |= X86_PTE_RW;
3206 }
3207 else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
3208 {
3209 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3210 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3211 cErrors++;
3212 continue;
3213 }
3214
3215 /* flags */
3216 if (pPhysPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)) /** @todo PAGE FLAGS */
3217 {
3218 if (pPhysPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)) /** @todo PAGE FLAGS */
3219 {
3220 if (PteDst.n.u1Write)
3221 {
3222 AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
3223 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3224 cErrors++;
3225 continue;
3226 }
3227 fIgnoreFlags |= X86_PTE_RW;
3228 }
3229 else
3230 {
3231 if (PteDst.n.u1Present)
3232 {
3233 AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
3234 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3235 cErrors++;
3236 continue;
3237 }
3238 fIgnoreFlags |= X86_PTE_P;
3239 }
3240 }
3241 else
3242 {
3243 if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3244 {
3245 if (PteDst.n.u1Write)
3246 {
3247 AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3248 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3249 cErrors++;
3250 continue;
3251 }
3252 if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3253 {
3254 AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3255 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3256 cErrors++;
3257 continue;
3258 }
3259 if (PteDst.n.u1Dirty)
3260 {
3261 AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3262 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3263 cErrors++;
3264 }
3265# if 0 /** @todo sync access bit properly... */
3266 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3267 {
3268 AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3269 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3270 cErrors++;
3271 }
3272 fIgnoreFlags |= X86_PTE_RW;
3273# else
3274 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
3275# endif
3276 }
3277 else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3278 {
3279 /* access bit emulation (not implemented). */
3280 if (PteSrc.n.u1Accessed || PteDst.n.u1Present)
3281 {
3282 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3283 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3284 cErrors++;
3285 continue;
3286 }
3287 if (!PteDst.n.u1Accessed)
3288 {
3289 AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3290 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3291 cErrors++;
3292 }
3293 fIgnoreFlags |= X86_PTE_P;
3294 }
3295# ifdef DEBUG_sandervl
3296 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
3297# endif
3298 }
3299
3300 if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3301 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3302 )
3303 {
3304 AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3305 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3306 fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3307 cErrors++;
3308 continue;
3309 }
3310 } /* foreach PTE */
3311 }
3312 else
3313 {
3314 /*
3315 * Big Page.
3316 */
3317 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | X86_PDE_PAE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
3318 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3319 {
3320 if (PdeDst.n.u1Write)
3321 {
3322 AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3323 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3324 cErrors++;
3325 continue;
3326 }
3327 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3328 {
3329 AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3330 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3331 cErrors++;
3332 continue;
3333 }
3334# if 0 /** @todo sync access bit properly... */
3335 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3336 {
3337 AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3338 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3339 cErrors++;
3340 }
3341 fIgnoreFlags |= X86_PTE_RW;
3342# else
3343 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
3344# endif
3345 }
3346 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3347 {
3348 /* access bit emulation (not implemented). */
3349 if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
3350 {
3351 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3352 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3353 cErrors++;
3354 continue;
3355 }
3356 if (!PdeDst.n.u1Accessed)
3357 {
3358 AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3359 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3360 cErrors++;
3361 }
3362 fIgnoreFlags |= X86_PTE_P;
3363 }
3364
3365 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3366 {
3367 AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3368 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3369 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3370 cErrors++;
3371 }
3372
3373 /* iterate the page table. */
3374 for (unsigned iPT = 0, off = 0;
3375 iPT < ELEMENTS(pPTDst->a);
3376 iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3377 {
3378 const SHWPTE PteDst = pPTDst->a[iPT];
3379
3380 if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3381 {
3382 AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3383 GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3384 cErrors++;
3385 }
3386
3387 /* skip not-present entries. */
3388 if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3389 continue;
3390
3391 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
3392
3393 /* match the physical addresses */
3394 HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3395
3396# ifdef IN_RING3
3397 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3398 if (VBOX_FAILURE(rc))
3399 {
3400 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3401 {
3402 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3403 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3404 cErrors++;
3405 }
3406 }
3407 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3408 {
3409 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3410 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3411 cErrors++;
3412 continue;
3413 }
3414# endif
3415
3416 pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3417 if (!pPhysPage)
3418 {
3419# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3420 if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3421 {
3422 AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3423 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3424 cErrors++;
3425 continue;
3426 }
3427# endif
3428 if (PteDst.n.u1Write)
3429 {
3430 AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3431 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3432 cErrors++;
3433 }
3434 fIgnoreFlags |= X86_PTE_RW;
3435 }
3436 else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
3437 {
3438 AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3439 GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3440 cErrors++;
3441 continue;
3442 }
3443
3444 /* flags */
3445 if (pPhysPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_ALL | MM_RAM_FLAGS_VIRTUAL_ALL | MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)) /** @todo PAGE FLAGS */
3446 {
3447 if (pPhysPage->HCPhys & (MM_RAM_FLAGS_PHYSICAL_WRITE | MM_RAM_FLAGS_VIRTUAL_WRITE)) /** @todo PAGE FLAGS */
3448 {
3449 if (!(pPhysPage->HCPhys & MM_RAM_FLAGS_PHYSICAL_TEMP_OFF)) /** @todo PAGE FLAGS */
3450 {
3451 if (PteDst.n.u1Write)
3452 {
3453 AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3454 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3455 cErrors++;
3456 continue;
3457 }
3458 fIgnoreFlags |= X86_PTE_RW;
3459 }
3460 }
3461 else
3462 {
3463 if (PteDst.n.u1Present)
3464 {
3465 AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3466 GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3467 cErrors++;
3468 continue;
3469 }
3470 fIgnoreFlags |= X86_PTE_P;
3471 }
3472 }
3473
3474 if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3475 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3476 )
3477 {
3478 AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3479 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3480 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3481 cErrors++;
3482 continue;
3483 }
3484 } /* foreach PTE */
3485 }
3486 }
3487 /* not present */
3488
3489 } /* forearch PDE */
3490
3491# ifdef DEBUG
3492 if (cErrors)
3493 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3494# endif
3495
3496#elif PGM_GST_TYPE == PGM_TYPE_PAE
3497//# error not implemented
3498
3499
3500#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3501//# error not implemented
3502
3503/*#else: guest real and protected mode */
3504#endif
3505 return cErrors;
3506}
3507#endif /* VBOX_STRICT */
3508
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette