VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 20065

Last change on this file since 20065 was 20065, checked in by vboxsync, 16 years ago

Logging
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 194.9 KB
Line 
1/* $Id: PGMAllBth.h 20065 2009-05-27 11:14:30Z vboxsync $ */
2/** @file
3 * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4 *
5 * This file is a big challenge!
6 */
7
8/*
9 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
10 *
11 * This file is part of VirtualBox Open Source Edition (OSE), as
12 * available from http://www.virtualbox.org. This file is free software;
13 * you can redistribute it and/or modify it under the terms of the GNU
14 * General Public License (GPL) as published by the Free Software
15 * Foundation, in version 2 as it comes in the "COPYING" file of the
16 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20 * Clara, CA 95054 USA or visit http://www.sun.com if you need
21 * additional information or have any questions.
22 */
23
24/*******************************************************************************
25* Internal Functions *
26*******************************************************************************/
27__BEGIN_DECLS
28PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
30PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
31PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage);
32PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
33PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
34PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
35PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36#ifdef VBOX_STRICT
37PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
38#endif
39#ifdef PGMPOOL_WITH_USER_TRACKING
40DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41#endif
42PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
43PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
44__END_DECLS
45
46
47/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
48#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
49# error "Invalid combination; PAE guest implies PAE shadow"
50#endif
51
52#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
53 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
54# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
55#endif
56
57#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
58 && !(PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT)
59# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
60#endif
61
62#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
63 || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
64# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
65#endif
66
67#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
68# define PGM_WITHOUT_MAPPINGS
69#endif
70
71
72#ifndef IN_RING3
73/**
74 * #PF Handler for raw-mode guest execution.
75 *
76 * @returns VBox status code (appropriate for trap handling and GC return).
77 *
78 * @param pVCpu VMCPU Handle.
79 * @param uErr The trap error code.
80 * @param pRegFrame Trap register frame.
81 * @param pvFault The fault address.
82 */
83PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
84{
85 PVM pVM = pVCpu->CTX_SUFF(pVM);
86
87# if defined(IN_RC) && defined(VBOX_STRICT)
88 PGMDynCheckLocks(pVM);
89# endif
90
91# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
92 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
93 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
94
95# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
96 /*
97 * Hide the instruction fetch trap indicator for now.
98 */
99 /** @todo NXE will change this and we must fix NXE in the switcher too! */
100 if (uErr & X86_TRAP_PF_ID)
101 {
102 uErr &= ~X86_TRAP_PF_ID;
103 TRPMSetErrorCode(pVCpu, uErr);
104 }
105# endif
106
107 /*
108 * Get PDs.
109 */
110 int rc;
111# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
112# if PGM_GST_TYPE == PGM_TYPE_32BIT
113 const unsigned iPDSrc = pvFault >> GST_PD_SHIFT;
114 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
115
116# elif PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
117
118# if PGM_GST_TYPE == PGM_TYPE_PAE
119 unsigned iPDSrc;
120 X86PDPE PdpeSrc;
121 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, pvFault, &iPDSrc, &PdpeSrc);
122
123# elif PGM_GST_TYPE == PGM_TYPE_AMD64
124 unsigned iPDSrc;
125 PX86PML4E pPml4eSrc;
126 X86PDPE PdpeSrc;
127 PGSTPD pPDSrc;
128
129 pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
130 Assert(pPml4eSrc);
131# endif
132
133 /* Quick check for a valid guest trap. (PAE & AMD64) */
134 if (!pPDSrc)
135 {
136# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
137 LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)((pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
138# else
139 LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
140# endif
141 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
142 TRPMSetErrorCode(pVCpu, uErr);
143 return VINF_EM_RAW_GUEST_TRAP;
144 }
145# endif
146
147# else /* !PGM_WITH_PAGING */
148 PGSTPD pPDSrc = NULL;
149 const unsigned iPDSrc = 0;
150# endif /* !PGM_WITH_PAGING */
151
152 /* Fetch the guest PDE */
153# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
154 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
155# else
156 GSTPDE PdeSrc;
157 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
158 PdeSrc.n.u1Present = 1;
159 PdeSrc.n.u1Write = 1;
160 PdeSrc.n.u1Accessed = 1;
161 PdeSrc.n.u1User = 1;
162# endif
163
164 pgmLock(pVM);
165 { /* Force the shadow pointers to go out of scope after releasing the lock. */
166# if PGM_SHW_TYPE == PGM_TYPE_32BIT
167 const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
168 PX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
169
170# elif PGM_SHW_TYPE == PGM_TYPE_PAE
171 const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
172
173 PX86PDPAE pPDDst;
174# if PGM_GST_TYPE != PGM_TYPE_PAE
175 X86PDPE PdpeSrc;
176
177 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
178 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
179# endif
180 rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, &PdpeSrc, &pPDDst);
181 if (rc != VINF_SUCCESS)
182 {
183 pgmUnlock(pVM);
184 AssertRC(rc);
185 return rc;
186 }
187 Assert(pPDDst);
188
189# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
190 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
191 PX86PDPAE pPDDst;
192# if PGM_GST_TYPE == PGM_TYPE_PROT
193 /* AMD-V nested paging */
194 X86PML4E Pml4eSrc;
195 X86PDPE PdpeSrc;
196 PX86PML4E pPml4eSrc = &Pml4eSrc;
197
198 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
199 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
200 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
201# endif
202
203 rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
204 if (rc != VINF_SUCCESS)
205 {
206 pgmUnlock(pVM);
207 AssertRC(rc);
208 return rc;
209 }
210 Assert(pPDDst);
211
212# elif PGM_SHW_TYPE == PGM_TYPE_EPT
213 const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
214 PEPTPD pPDDst;
215
216 rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
217 if (rc != VINF_SUCCESS)
218 {
219 pgmUnlock(pVM);
220 AssertRC(rc);
221 return rc;
222 }
223 Assert(pPDDst);
224# endif
225
226# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
227 /*
228 * If we successfully correct the write protection fault due to dirty bit
229 * tracking, or this page fault is a genuine one, then return immediately.
230 */
231 STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
232 rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], pvFault);
233 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
234 if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
235 || rc == VINF_EM_RAW_GUEST_TRAP)
236 {
237 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
238 = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVCpu->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
239 LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
240 pgmUnlock(pVM);
241 return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
242 }
243
244 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
245# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
246
247 /*
248 * A common case is the not-present error caused by lazy page table syncing.
249 *
250 * It is IMPORTANT that we weed out any access to non-present shadow PDEs here
251 * so we can safely assume that the shadow PT is present when calling SyncPage later.
252 *
253 * On failure, we ASSUME that SyncPT is out of memory or detected some kind
254 * of mapping conflict and defer to SyncCR3 in R3.
255 * (Again, we do NOT support access handlers for non-present guest pages.)
256 *
257 */
258 if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
259 && !pPDDst->a[iPDDst].n.u1Present
260 && PdeSrc.n.u1Present
261 )
262 {
263 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2SyncPT; });
264 STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
265 LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
266 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, pvFault);
267 pgmUnlock(pVM);
268 if (RT_SUCCESS(rc))
269 {
270 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
271 return rc;
272 }
273 Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
274 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
275 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
276 return VINF_PGM_SYNC_CR3;
277 }
278 pgmUnlock(pVM);
279 }
280
281# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
282 /*
283 * Check if this address is within any of our mappings.
284 *
285 * This is *very* fast and it's gonna save us a bit of effort below and prevent
286 * us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
287 * (BTW, it's impossible to have physical access handlers in a mapping.)
288 */
289 if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
290 {
291 STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
292 PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
293 for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
294 {
295 if (pvFault < pMapping->GCPtr)
296 break;
297 if (pvFault - pMapping->GCPtr < pMapping->cb)
298 {
299 /*
300 * The first thing we check is if we've got an undetected conflict.
301 */
302 if (!pVM->pgm.s.fMappingsFixed)
303 {
304 unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
305 while (iPT-- > 0)
306 if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
307 {
308 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eConflicts);
309 Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
310 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
311 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
312 return VINF_PGM_SYNC_CR3;
313 }
314 }
315
316 /*
317 * Check if the fault address is in a virtual page access handler range.
318 */
319 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
320 if ( pCur
321 && pvFault - pCur->Core.Key < pCur->cb
322 && uErr & X86_TRAP_PF_RW)
323 {
324# ifdef IN_RC
325 STAM_PROFILE_START(&pCur->Stat, h);
326 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
327 STAM_PROFILE_STOP(&pCur->Stat, h);
328# else
329 AssertFailed();
330 rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
331# endif
332 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersMapping);
333 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
334 return rc;
335 }
336
337 /*
338 * Pretend we're not here and let the guest handle the trap.
339 */
340 TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
341 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFMapping);
342 LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
343 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
344 return VINF_EM_RAW_GUEST_TRAP;
345 }
346 }
347 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
348 } /* pgmAreMappingsEnabled(&pVM->pgm.s) */
349# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
350
351 /*
352 * Check if this fault address is flagged for special treatment,
353 * which means we'll have to figure out the physical address and
354 * check flags associated with it.
355 *
356 * ASSUME that we can limit any special access handling to pages
357 * in page tables which the guest believes to be present.
358 */
359 if (PdeSrc.n.u1Present)
360 {
361 RTGCPHYS GCPhys = NIL_RTGCPHYS;
362
363# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
364# if PGM_GST_TYPE == PGM_TYPE_AMD64
365 bool fBigPagesSupported = true;
366# else
367 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
368# endif
369 if ( PdeSrc.b.u1Size
370 && fBigPagesSupported)
371 GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc)
372 | ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
373 else
374 {
375 PGSTPT pPTSrc;
376 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
377 if (RT_SUCCESS(rc))
378 {
379 unsigned iPTESrc = (pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
380 if (pPTSrc->a[iPTESrc].n.u1Present)
381 GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
382 }
383 }
384# else
385 /* No paging so the fault address is the physical address */
386 GCPhys = (RTGCPHYS)(pvFault & ~PAGE_OFFSET_MASK);
387# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
388
389 /*
390 * If we have a GC address we'll check if it has any flags set.
391 */
392 if (GCPhys != NIL_RTGCPHYS)
393 {
394 STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
395
396 PPGMPAGE pPage;
397 rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
398 if (RT_SUCCESS(rc)) /** just handle the failure immediate (it returns) and make things easier to read. */
399 {
400 if ( PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage)
401 || PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage))
402 {
403 if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
404 {
405 /*
406 * Physical page access handler.
407 */
408 const RTGCPHYS GCPhysFault = GCPhys | (pvFault & PAGE_OFFSET_MASK);
409 PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
410 if (pCur)
411 {
412# ifdef PGM_SYNC_N_PAGES
413 /*
414 * If the region is write protected and we got a page not present fault, then sync
415 * the pages. If the fault was caused by a read, then restart the instruction.
416 * In case of write access continue to the GC write handler.
417 *
418 * ASSUMES that there is only one handler per page or that they have similar write properties.
419 */
420 if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
421 && !(uErr & X86_TRAP_PF_P))
422 {
423 pgmLock(pVM);
424 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
425 pgmUnlock(pVM);
426 if ( RT_FAILURE(rc)
427 || !(uErr & X86_TRAP_PF_RW)
428 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
429 {
430 AssertRC(rc);
431 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
432 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
433 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
434 return rc;
435 }
436 }
437# endif
438
439 AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
440 || (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
441 ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
442
443# if defined(IN_RC) || defined(IN_RING0)
444 if (pCur->CTX_SUFF(pfnHandler))
445 {
446 STAM_PROFILE_START(&pCur->Stat, h);
447 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pCur->CTX_SUFF(pvUser));
448 STAM_PROFILE_STOP(&pCur->Stat, h);
449 }
450 else
451# endif
452 rc = VINF_EM_RAW_EMULATE_INSTR;
453 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersPhysical);
454 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
455 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndPhys; });
456 return rc;
457 }
458 }
459# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
460 else
461 {
462# ifdef PGM_SYNC_N_PAGES
463 /*
464 * If the region is write protected and we got a page not present fault, then sync
465 * the pages. If the fault was caused by a read, then restart the instruction.
466 * In case of write access continue to the GC write handler.
467 */
468 if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
469 && !(uErr & X86_TRAP_PF_P))
470 {
471 pgmLock(pVM);
472 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
473 pgmUnlock(pVM);
474 if ( RT_FAILURE(rc)
475 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
476 || !(uErr & X86_TRAP_PF_RW))
477 {
478 AssertRC(rc);
479 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
480 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
481 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
482 return rc;
483 }
484 }
485# endif
486 /*
487 * Ok, it's an virtual page access handler.
488 *
489 * Since it's faster to search by address, we'll do that first
490 * and then retry by GCPhys if that fails.
491 */
492 /** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
493 /** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
494 * page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
495 */
496 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
497 if (pCur)
498 {
499 AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
500 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
501 || !(uErr & X86_TRAP_PF_P)
502 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
503 ("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
504
505 if ( pvFault - pCur->Core.Key < pCur->cb
506 && ( uErr & X86_TRAP_PF_RW
507 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
508 {
509# ifdef IN_RC
510 STAM_PROFILE_START(&pCur->Stat, h);
511 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
512 STAM_PROFILE_STOP(&pCur->Stat, h);
513# else
514 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
515# endif
516 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtual);
517 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
518 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
519 return rc;
520 }
521 /* Unhandled part of a monitored page */
522 }
523 else
524 {
525 /* Check by physical address. */
526 PPGMVIRTHANDLER pCur;
527 unsigned iPage;
528 rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + (pvFault & PAGE_OFFSET_MASK),
529 &pCur, &iPage);
530 Assert(RT_SUCCESS(rc) || !pCur);
531 if ( pCur
532 && ( uErr & X86_TRAP_PF_RW
533 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
534 {
535 Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
536# ifdef IN_RC
537 RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
538 Assert(off < pCur->cb);
539 STAM_PROFILE_START(&pCur->Stat, h);
540 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
541 STAM_PROFILE_STOP(&pCur->Stat, h);
542# else
543 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
544# endif
545 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualByPhys);
546 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
547 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
548 return rc;
549 }
550 }
551 }
552# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
553
554 /*
555 * There is a handled area of the page, but this fault doesn't belong to it.
556 * We must emulate the instruction.
557 *
558 * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
559 * we first check if this was a page-not-present fault for a page with only
560 * write access handlers. Restart the instruction if it wasn't a write access.
561 */
562 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersUnhandled);
563
564 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
565 && !(uErr & X86_TRAP_PF_P))
566 {
567 pgmLock(pVM);
568 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
569 pgmUnlock(pVM);
570 if ( RT_FAILURE(rc)
571 || rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
572 || !(uErr & X86_TRAP_PF_RW))
573 {
574 AssertRC(rc);
575 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
576 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
577 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
578 return rc;
579 }
580 }
581
582 /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
583 * It's writing to an unhandled part of the LDT page several million times.
584 */
585 rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
586 LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
587 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
588 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndUnhandled; });
589 return rc;
590 } /* if any kind of handler */
591
592# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
593 if (uErr & X86_TRAP_PF_P)
594 {
595 /*
596 * The page isn't marked, but it might still be monitored by a virtual page access handler.
597 * (ASSUMES no temporary disabling of virtual handlers.)
598 */
599 /** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
600 * we should correct both the shadow page table and physical memory flags, and not only check for
601 * accesses within the handler region but for access to pages with virtual handlers. */
602 PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
603 if (pCur)
604 {
605 AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
606 || ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
607 || !(uErr & X86_TRAP_PF_P)
608 || (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
609 ("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
610
611 if ( pvFault - pCur->Core.Key < pCur->cb
612 && ( uErr & X86_TRAP_PF_RW
613 || pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
614 {
615# ifdef IN_RC
616 STAM_PROFILE_START(&pCur->Stat, h);
617 rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
618 STAM_PROFILE_STOP(&pCur->Stat, h);
619# else
620 rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
621# endif
622 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualUnmarked);
623 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
624 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
625 return rc;
626 }
627 }
628 }
629# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
630 }
631 else
632 {
633 /*
634 * When the guest accesses invalid physical memory (e.g. probing
635 * of RAM or accessing a remapped MMIO range), then we'll fall
636 * back to the recompiler to emulate the instruction.
637 */
638 LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
639 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersInvalid);
640 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
641 return VINF_EM_RAW_EMULATE_INSTR;
642 }
643
644 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
645
646# ifdef PGM_OUT_OF_SYNC_IN_GC /** @todo remove this bugger. */
647 /*
648 * We are here only if page is present in Guest page tables and
649 * trap is not handled by our handlers.
650 *
651 * Check it for page out-of-sync situation.
652 */
653 STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
654
655 if (!(uErr & X86_TRAP_PF_P))
656 {
657 /*
658 * Page is not present in our page tables.
659 * Try to sync it!
660 * BTW, fPageShw is invalid in this branch!
661 */
662 if (uErr & X86_TRAP_PF_US)
663 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
664 else /* supervisor */
665 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
666
667# if defined(LOG_ENABLED) && !defined(IN_RING0)
668 RTGCPHYS GCPhys;
669 uint64_t fPageGst;
670 PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
671 Log(("Page out of sync: %RGv eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%RGp scan=%d\n",
672 pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
673# endif /* LOG_ENABLED */
674
675# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
676 if (CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
677 {
678 uint64_t fPageGst;
679 rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
680 if ( RT_SUCCESS(rc)
681 && !(fPageGst & X86_PTE_US))
682 {
683 /* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
684 if ( pvFault == (RTGCPTR)pRegFrame->eip
685 || pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
686# ifdef CSAM_DETECT_NEW_CODE_PAGES
687 || ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
688 && CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
689# endif /* CSAM_DETECT_NEW_CODE_PAGES */
690 )
691 {
692 LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
693 rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
694 if (rc != VINF_SUCCESS)
695 {
696 /*
697 * CSAM needs to perform a job in ring 3.
698 *
699 * Sync the page before going to the host context; otherwise we'll end up in a loop if
700 * CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
701 */
702 LogFlow(("CSAM ring 3 job\n"));
703 pgmLock(pVM);
704 int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
705 pgmUnlock(pVM);
706 AssertRC(rc2);
707
708 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
709 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2CSAM; });
710 return rc;
711 }
712 }
713# ifdef CSAM_DETECT_NEW_CODE_PAGES
714 else if ( uErr == X86_TRAP_PF_RW
715 && pRegFrame->ecx >= 0x100 /* early check for movswd count */
716 && pRegFrame->ecx < 0x10000)
717 {
718 /* In case of a write to a non-present supervisor shadow page, we'll take special precautions
719 * to detect loading of new code pages.
720 */
721
722 /*
723 * Decode the instruction.
724 */
725 RTGCPTR PC;
726 rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
727 if (rc == VINF_SUCCESS)
728 {
729 DISCPUSTATE Cpu;
730 uint32_t cbOp;
731 rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, &Cpu, &cbOp);
732
733 /* For now we'll restrict this to rep movsw/d instructions */
734 if ( rc == VINF_SUCCESS
735 && Cpu.pCurInstr->opcode == OP_MOVSWD
736 && (Cpu.prefix & PREFIX_REP))
737 {
738 CSAMMarkPossibleCodePage(pVM, pvFault);
739 }
740 }
741 }
742# endif /* CSAM_DETECT_NEW_CODE_PAGES */
743
744 /*
745 * Mark this page as safe.
746 */
747 /** @todo not correct for pages that contain both code and data!! */
748 Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
749 CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
750 }
751 }
752# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
753 pgmLock(pVM);
754 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
755 pgmUnlock(pVM);
756 if (RT_SUCCESS(rc))
757 {
758 /* The page was successfully synced, return to the guest. */
759 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
760 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSync; });
761 return VINF_SUCCESS;
762 }
763 }
764 else /* uErr & X86_TRAP_PF_P: */
765 {
766 /*
767 * Write protected pages are make writable when the guest makes the first
768 * write to it. This happens for pages that are shared, write monitored
769 * and not yet allocated.
770 *
771 * Also, a side effect of not flushing global PDEs are out of sync pages due
772 * to physical monitored regions, that are no longer valid.
773 * Assume for now it only applies to the read/write flag.
774 */
775 if (RT_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
776 {
777 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
778 {
779 Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n",
780 GCPhys, pPage, pvFault, uErr));
781 rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
782 if (rc != VINF_SUCCESS)
783 {
784 AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
785 return rc;
786 }
787 if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
788 return VINF_EM_NO_MEMORY;
789 }
790 /// @todo count the above case; else
791 if (uErr & X86_TRAP_PF_US)
792 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
793 else /* supervisor */
794 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
795
796 /*
797 * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
798 * page is not present, which is not true in this case.
799 */
800 pgmLock(pVM);
801 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
802 pgmUnlock(pVM);
803 if (RT_SUCCESS(rc))
804 {
805 /*
806 * Page was successfully synced, return to guest.
807 */
808# ifdef VBOX_STRICT
809 RTGCPHYS GCPhys;
810 uint64_t fPageGst;
811 rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
812 Assert(RT_SUCCESS(rc) && fPageGst & X86_PTE_RW);
813 LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
814
815 uint64_t fPageShw;
816 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
817 AssertMsg((pVM->cCPUs > 1 && rc == VERR_PAGE_NOT_PRESENT) || (RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)), ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
818# endif /* VBOX_STRICT */
819 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
820 STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
821 return VINF_SUCCESS;
822 }
823
824 /* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
825 if ( CPUMGetGuestCPL(pVCpu, pRegFrame) == 0
826 && ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG)
827 && (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P))
828 {
829 uint64_t fPageGst;
830 rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
831 if ( RT_SUCCESS(rc)
832 && !(fPageGst & X86_PTE_RW))
833 {
834 rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
835 if (RT_SUCCESS(rc))
836 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulInRZ);
837 else
838 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulToR3);
839 return rc;
840 }
841 AssertMsgFailed(("Unexpected r/w page %RGv flag=%x rc=%Rrc\n", pvFault, (uint32_t)fPageGst, rc));
842 }
843 }
844
845# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
846# ifdef VBOX_STRICT
847 /*
848 * Check for VMM page flags vs. Guest page flags consistency.
849 * Currently only for debug purposes.
850 */
851 if (RT_SUCCESS(rc))
852 {
853 /* Get guest page flags. */
854 uint64_t fPageGst;
855 rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
856 if (RT_SUCCESS(rc))
857 {
858 uint64_t fPageShw;
859 rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
860
861 /*
862 * Compare page flags.
863 * Note: we have AVL, A, D bits desynched.
864 */
865 AssertMsg((fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)),
866 ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
867 }
868 else
869 AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
870 }
871 else
872 AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
873# endif /* VBOX_STRICT */
874# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
875 }
876 STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
877# endif /* PGM_OUT_OF_SYNC_IN_GC */
878 }
879 else /* GCPhys == NIL_RTGCPHYS */
880 {
881 /*
882 * Page not present in Guest OS or invalid page table address.
883 * This is potential virtual page access handler food.
884 *
885 * For the present we'll say that our access handlers don't
886 * work for this case - we've already discarded the page table
887 * not present case which is identical to this.
888 *
889 * When we perchance find we need this, we will probably have AVL
890 * trees (offset based) to operate on and we can measure their speed
891 * agains mapping a page table and probably rearrange this handling
892 * a bit. (Like, searching virtual ranges before checking the
893 * physical address.)
894 */
895 }
896 }
897 /* else: !present (guest) */
898
899
900# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
901 /*
902 * Conclusion, this is a guest trap.
903 */
904 LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
905 STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFUnh);
906 return VINF_EM_RAW_GUEST_TRAP;
907# else
908 /* present, but not a monitored page; perhaps the guest is probing physical memory */
909 return VINF_EM_RAW_EMULATE_INSTR;
910# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
911
912
913# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
914
915 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
916 return VERR_INTERNAL_ERROR;
917# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
918}
919#endif /* !IN_RING3 */
920
921
922/**
923 * Emulation of the invlpg instruction.
924 *
925 *
926 * @returns VBox status code.
927 *
928 * @param pVCpu The VMCPU handle.
929 * @param GCPtrPage Page to invalidate.
930 *
931 * @remark ASSUMES that the guest is updating before invalidating. This order
932 * isn't required by the CPU, so this is speculative and could cause
933 * trouble.
934 * @remark No TLB shootdown is done on any other VCPU as we assume that
935 * invlpg emulation is the *only* reason for calling this function.
936 * (The guest has to shoot down TLB entries on other CPUs itself)
937 * Currently true, but keep in mind!
938 *
939 * @todo Flush page or page directory only if necessary!
940 * @todo Add a #define for simply invalidating the page.
941 */
942PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
943{
944#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
945 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
946 && PGM_SHW_TYPE != PGM_TYPE_EPT
947 int rc;
948 PVM pVM = pVCpu->CTX_SUFF(pVM);
949 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
950
951 Assert(PGMIsLockOwner(pVM));
952
953 LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
954 /*
955 * Get the shadow PD entry and skip out if this PD isn't present.
956 * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
957 */
958# if PGM_SHW_TYPE == PGM_TYPE_32BIT
959 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
960 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
961
962 /* Fetch the pgm pool shadow descriptor. */
963 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
964 Assert(pShwPde);
965
966# elif PGM_SHW_TYPE == PGM_TYPE_PAE
967 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
968 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
969
970 /* If the shadow PDPE isn't present, then skip the invalidate. */
971 if (!pPdptDst->a[iPdpt].n.u1Present)
972 {
973 Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
974 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
975 return VINF_SUCCESS;
976 }
977
978 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
979 PPGMPOOLPAGE pShwPde = NULL;
980 PX86PDPAE pPDDst;
981
982 /* Fetch the pgm pool shadow descriptor. */
983 rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
984 AssertRCSuccessReturn(rc, rc);
985 Assert(pShwPde);
986
987 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
988 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
989
990# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
991 /* PML4 */
992 const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
993 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
994 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
995 PX86PDPAE pPDDst;
996 PX86PDPT pPdptDst;
997 PX86PML4E pPml4eDst;
998 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
999 if (rc != VINF_SUCCESS)
1000 {
1001 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1002 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1003 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1004 PGM_INVL_VCPU_TLBS(pVCpu);
1005 return VINF_SUCCESS;
1006 }
1007 Assert(pPDDst);
1008
1009 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1010 PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1011
1012 if (!pPdpeDst->n.u1Present)
1013 {
1014 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1015 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1016 PGM_INVL_VCPU_TLBS(pVCpu);
1017 return VINF_SUCCESS;
1018 }
1019
1020# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1021
1022 const SHWPDE PdeDst = *pPdeDst;
1023 if (!PdeDst.n.u1Present)
1024 {
1025 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1026 return VINF_SUCCESS;
1027 }
1028
1029# if defined(IN_RC)
1030 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1031 PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1032# endif
1033
1034 /*
1035 * Get the guest PD entry and calc big page.
1036 */
1037# if PGM_GST_TYPE == PGM_TYPE_32BIT
1038 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
1039 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1040 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1041# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1042 unsigned iPDSrc = 0;
1043# if PGM_GST_TYPE == PGM_TYPE_PAE
1044 X86PDPE PdpeSrc;
1045 PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
1046# else /* AMD64 */
1047 PX86PML4E pPml4eSrc;
1048 X86PDPE PdpeSrc;
1049 PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
1050# endif
1051 GSTPDE PdeSrc;
1052
1053 if (pPDSrc)
1054 PdeSrc = pPDSrc->a[iPDSrc];
1055 else
1056 PdeSrc.u = 0;
1057# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1058
1059# if PGM_GST_TYPE == PGM_TYPE_AMD64
1060 const bool fIsBigPage = PdeSrc.b.u1Size;
1061# else
1062 const bool fIsBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1063# endif
1064
1065# ifdef IN_RING3
1066 /*
1067 * If a CR3 Sync is pending we may ignore the invalidate page operation
1068 * depending on the kind of sync and if it's a global page or not.
1069 * This doesn't make sense in GC/R0 so we'll skip it entirely there.
1070 */
1071# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1072 if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1073 || ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1074 && fIsBigPage
1075 && PdeSrc.b.u1Global
1076 )
1077 )
1078# else
1079 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1080# endif
1081 {
1082 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1083 return VINF_SUCCESS;
1084 }
1085# endif /* IN_RING3 */
1086
1087# if PGM_GST_TYPE == PGM_TYPE_AMD64
1088 /* Fetch the pgm pool shadow descriptor. */
1089 PPGMPOOLPAGE pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
1090 Assert(pShwPdpt);
1091
1092 /* Fetch the pgm pool shadow descriptor. */
1093 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1094 Assert(pShwPde);
1095
1096 Assert(pPml4eDst->n.u1Present && (pPml4eDst->u & SHW_PDPT_MASK));
1097 RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1098
1099 if ( !pPml4eSrc->n.u1Present
1100 || pShwPdpt->GCPhys != GCPhysPdpt)
1101 {
1102 LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1103 GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1104 pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1105 ASMAtomicWriteSize(pPml4eDst, 0);
1106 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1107 PGM_INVL_VCPU_TLBS(pVCpu);
1108 return VINF_SUCCESS;
1109 }
1110 if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
1111 || (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
1112 {
1113 /*
1114 * Mark not present so we can resync the PML4E when it's used.
1115 */
1116 LogFlow(("InvalidatePage: Out-of-sync PML4E at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1117 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1118 pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1119 ASMAtomicWriteSize(pPml4eDst, 0);
1120 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1121 PGM_INVL_VCPU_TLBS(pVCpu);
1122 }
1123 else if (!pPml4eSrc->n.u1Accessed)
1124 {
1125 /*
1126 * Mark not present so we can set the accessed bit.
1127 */
1128 LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1129 GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1130 pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1131 ASMAtomicWriteSize(pPml4eDst, 0);
1132 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1133 PGM_INVL_VCPU_TLBS(pVCpu);
1134 }
1135
1136 /* Check if the PDPT entry has changed. */
1137 Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1138 RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1139 if ( !PdpeSrc.n.u1Present
1140 || pShwPde->GCPhys != GCPhysPd)
1141 {
1142 LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1143 GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1144 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1145 ASMAtomicWriteSize(pPdpeDst, 0);
1146 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1147 PGM_INVL_VCPU_TLBS(pVCpu);
1148 return VINF_SUCCESS;
1149 }
1150 if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1151 || (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1152 {
1153 /*
1154 * Mark not present so we can resync the PDPTE when it's used.
1155 */
1156 LogFlow(("InvalidatePage: Out-of-sync PDPE at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1157 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1158 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1159 ASMAtomicWriteSize(pPdpeDst, 0);
1160 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1161 PGM_INVL_VCPU_TLBS(pVCpu);
1162 }
1163 else if (!PdpeSrc.lm.u1Accessed)
1164 {
1165 /*
1166 * Mark not present so we can set the accessed bit.
1167 */
1168 LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1169 GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1170 pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1171 ASMAtomicWriteSize(pPdpeDst, 0);
1172 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1173 PGM_INVL_VCPU_TLBS(pVCpu);
1174 }
1175# endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1176
1177 /*
1178 * Deal with the Guest PDE.
1179 */
1180 rc = VINF_SUCCESS;
1181 if (PdeSrc.n.u1Present)
1182 {
1183 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1184 {
1185 /*
1186 * Conflict - Let SyncPT deal with it to avoid duplicate code.
1187 */
1188 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1189 Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1190 pgmLock(pVM);
1191 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1192 pgmUnlock(pVM);
1193 }
1194 else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1195 || (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1196 {
1197 /*
1198 * Mark not present so we can resync the PDE when it's used.
1199 */
1200 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1201 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1202 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1203 ASMAtomicWriteSize(pPdeDst, 0);
1204 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1205 PGM_INVL_VCPU_TLBS(pVCpu);
1206 }
1207 else if (!PdeSrc.n.u1Accessed)
1208 {
1209 /*
1210 * Mark not present so we can set the accessed bit.
1211 */
1212 LogFlow(("InvalidatePage: Out-of-sync (A) at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1213 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1214 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1215 ASMAtomicWriteSize(pPdeDst, 0);
1216 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1217 PGM_INVL_VCPU_TLBS(pVCpu);
1218 }
1219 else if (!fIsBigPage)
1220 {
1221 /*
1222 * 4KB - page.
1223 */
1224 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1225 RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1226# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1227 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1228 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1229# endif
1230 if (pShwPage->GCPhys == GCPhys)
1231 {
1232# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1233 const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1234 PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1235 if (pPT->a[iPTEDst].n.u1Present)
1236 {
1237# ifdef PGMPOOL_WITH_USER_TRACKING
1238 /* This is very unlikely with caching/monitoring enabled. */
1239 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1240# endif
1241 ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1242 }
1243# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1244 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1245 if (RT_SUCCESS(rc))
1246 rc = VINF_SUCCESS;
1247# endif
1248 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1249 PGM_INVL_PG(pVCpu, GCPtrPage);
1250 }
1251 else
1252 {
1253 /*
1254 * The page table address changed.
1255 */
1256 LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1257 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1258 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1259 ASMAtomicWriteSize(pPdeDst, 0);
1260 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1261 PGM_INVL_VCPU_TLBS(pVCpu);
1262 }
1263 }
1264 else
1265 {
1266 /*
1267 * 2/4MB - page.
1268 */
1269 /* Before freeing the page, check if anything really changed. */
1270 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1271 RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1272# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1273 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1274 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1275# endif
1276 if ( pShwPage->GCPhys == GCPhys
1277 && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1278 {
1279 /* ASSUMES a the given bits are identical for 4M and normal PDEs */
1280 /** @todo PAT */
1281 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1282 == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD))
1283 && ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1284 || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1285 {
1286 LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1287 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1288# if defined(IN_RC)
1289 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1290 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1291# endif
1292 return VINF_SUCCESS;
1293 }
1294 }
1295
1296 /*
1297 * Ok, the page table is present and it's been changed in the guest.
1298 * If we're in host context, we'll just mark it as not present taking the lazy approach.
1299 * We could do this for some flushes in GC too, but we need an algorithm for
1300 * deciding which 4MB pages containing code likely to be executed very soon.
1301 */
1302 LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1303 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1304 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1305 ASMAtomicWriteSize(pPdeDst, 0);
1306 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1307 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1308 }
1309 }
1310 else
1311 {
1312 /*
1313 * Page directory is not present, mark shadow PDE not present.
1314 */
1315 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1316 {
1317 pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1318 ASMAtomicWriteSize(pPdeDst, 0);
1319 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1320 PGM_INVL_PG(pVCpu, GCPtrPage);
1321 }
1322 else
1323 {
1324 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1325 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDMappings));
1326 }
1327 }
1328# if defined(IN_RC)
1329 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1330 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1331# endif
1332 return rc;
1333
1334#else /* guest real and protected mode */
1335 /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1336 return VINF_SUCCESS;
1337#endif
1338}
1339
1340
1341#ifdef PGMPOOL_WITH_USER_TRACKING
1342/**
1343 * Update the tracking of shadowed pages.
1344 *
1345 * @param pVCpu The VMCPU handle.
1346 * @param pShwPage The shadow page.
1347 * @param HCPhys The physical page we is being dereferenced.
1348 */
1349DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1350{
1351# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1352 PVM pVM = pVCpu->CTX_SUFF(pVM);
1353
1354 STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1355 LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1356
1357 /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
1358 * 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1359 * 2. write protect all shadowed pages. I.e. implement caching.
1360 */
1361 /*
1362 * Find the guest address.
1363 */
1364 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1365 pRam;
1366 pRam = pRam->CTX_SUFF(pNext))
1367 {
1368 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1369 while (iPage-- > 0)
1370 {
1371 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1372 {
1373 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1374 pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1375 pShwPage->cPresent--;
1376 pPool->cPresent--;
1377 STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1378 return;
1379 }
1380 }
1381 }
1382
1383 for (;;)
1384 AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1385# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1386 pShwPage->cPresent--;
1387 pVM->pgm.s.CTX_SUFF(pPool)->cPresent--;
1388# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1389}
1390
1391
1392/**
1393 * Update the tracking of shadowed pages.
1394 *
1395 * @param pVCpu The VMCPU handle.
1396 * @param pShwPage The shadow page.
1397 * @param u16 The top 16-bit of the pPage->HCPhys.
1398 * @param pPage Pointer to the guest page. this will be modified.
1399 * @param iPTDst The index into the shadow table.
1400 */
1401DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1402{
1403 PVM pVM = pVCpu->CTX_SUFF(pVM);
1404# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1405 /*
1406 * Just deal with the simple first time here.
1407 */
1408 if (!u16)
1409 {
1410 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1411 u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1412 }
1413 else
1414 u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1415
1416 /* write back */
1417 Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1418 PGM_PAGE_SET_TRACKING(pPage, u16);
1419
1420# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1421
1422 /* update statistics. */
1423 pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1424 pShwPage->cPresent++;
1425 if (pShwPage->iFirstPresent > iPTDst)
1426 pShwPage->iFirstPresent = iPTDst;
1427}
1428#endif /* PGMPOOL_WITH_USER_TRACKING */
1429
1430
1431/**
1432 * Creates a 4K shadow page for a guest page.
1433 *
1434 * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1435 * physical address. The PdeSrc argument only the flags are used. No page structured
1436 * will be mapped in this function.
1437 *
1438 * @param pVCpu The VMCPU handle.
1439 * @param pPteDst Destination page table entry.
1440 * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1441 * Can safely assume that only the flags are being used.
1442 * @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1443 * @param pShwPage Pointer to the shadow page.
1444 * @param iPTDst The index into the shadow table.
1445 *
1446 * @remark Not used for 2/4MB pages!
1447 */
1448DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1449{
1450 if (PteSrc.n.u1Present)
1451 {
1452 PVM pVM = pVCpu->CTX_SUFF(pVM);
1453
1454 /*
1455 * Find the ram range.
1456 */
1457 PPGMPAGE pPage;
1458 int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1459 if (RT_SUCCESS(rc))
1460 {
1461#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1462 /* Try make the page writable if necessary. */
1463 if ( PteSrc.n.u1Write
1464 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1465 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1466 {
1467 rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1468 AssertRC(rc);
1469 }
1470#endif
1471
1472 /** @todo investiage PWT, PCD and PAT. */
1473 /*
1474 * Make page table entry.
1475 */
1476 SHWPTE PteDst;
1477 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1478 {
1479 /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1480 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1481 {
1482#if PGM_SHW_TYPE == PGM_TYPE_EPT
1483 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1484 PteDst.n.u1Present = 1;
1485 PteDst.n.u1Execute = 1;
1486 PteDst.n.u1IgnorePAT = 1;
1487 PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1488 /* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1489#else
1490 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1491 | PGM_PAGE_GET_HCPHYS(pPage);
1492#endif
1493 }
1494 else
1495 {
1496 LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", PGM_PAGE_GET_HCPHYS(pPage)));
1497 PteDst.u = 0;
1498 }
1499 /** @todo count these two kinds. */
1500 }
1501 else
1502 {
1503#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1504 /*
1505 * If the page or page directory entry is not marked accessed,
1506 * we mark the page not present.
1507 */
1508 if (!PteSrc.n.u1Accessed || !PdeSrc.n.u1Accessed)
1509 {
1510 LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1511 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,AccessedPage));
1512 PteDst.u = 0;
1513 }
1514 else
1515 /*
1516 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1517 * when the page is modified.
1518 */
1519 if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1520 {
1521 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPage));
1522 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT | X86_PTE_RW))
1523 | PGM_PAGE_GET_HCPHYS(pPage)
1524 | PGM_PTFLAGS_TRACK_DIRTY;
1525 }
1526 else
1527#endif
1528 {
1529 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageSkipped));
1530#if PGM_SHW_TYPE == PGM_TYPE_EPT
1531 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1532 PteDst.n.u1Present = 1;
1533 PteDst.n.u1Write = 1;
1534 PteDst.n.u1Execute = 1;
1535 PteDst.n.u1IgnorePAT = 1;
1536 PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1537 /* PteDst.n.u1Size = 0 */
1538#else
1539 PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1540 | PGM_PAGE_GET_HCPHYS(pPage);
1541#endif
1542 }
1543 }
1544
1545 /*
1546 * Make sure only allocated pages are mapped writable.
1547 */
1548 if ( PteDst.n.u1Write
1549 && PteDst.n.u1Present
1550 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1551 {
1552 PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. */
1553 Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1554 }
1555
1556#ifdef PGMPOOL_WITH_USER_TRACKING
1557 /*
1558 * Keep user track up to date.
1559 */
1560 if (PteDst.n.u1Present)
1561 {
1562 if (!pPteDst->n.u1Present)
1563 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1564 else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1565 {
1566 Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1567 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1568 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1569 }
1570 }
1571 else if (pPteDst->n.u1Present)
1572 {
1573 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1574 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1575 }
1576#endif /* PGMPOOL_WITH_USER_TRACKING */
1577
1578 /*
1579 * Update statistics and commit the entry.
1580 */
1581#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1582 if (!PteSrc.n.u1Global)
1583 pShwPage->fSeenNonGlobal = true;
1584#endif
1585 ASMAtomicWriteSize(pPteDst, PteDst.u);
1586 }
1587 /* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1588 /** @todo count these. */
1589 }
1590 else
1591 {
1592 /*
1593 * Page not-present.
1594 */
1595 LogFlow(("SyncPageWorker: page not present in Pte\n"));
1596#ifdef PGMPOOL_WITH_USER_TRACKING
1597 /* Keep user track up to date. */
1598 if (pPteDst->n.u1Present)
1599 {
1600 Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1601 PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1602 }
1603#endif /* PGMPOOL_WITH_USER_TRACKING */
1604 ASMAtomicWriteSize(pPteDst, 0);
1605 /** @todo count these. */
1606 }
1607}
1608
1609
1610/**
1611 * Syncs a guest OS page.
1612 *
1613 * There are no conflicts at this point, neither is there any need for
1614 * page table allocations.
1615 *
1616 * @returns VBox status code.
1617 * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1618 * @param pVCpu The VMCPU handle.
1619 * @param PdeSrc Page directory entry of the guest.
1620 * @param GCPtrPage Guest context page address.
1621 * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1622 * @param uErr Fault error (X86_TRAP_PF_*).
1623 */
1624PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1625{
1626 PVM pVM = pVCpu->CTX_SUFF(pVM);
1627 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1628 LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1629
1630 Assert(PGMIsLockOwner(pVM));
1631
1632#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1633 || PGM_GST_TYPE == PGM_TYPE_PAE \
1634 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
1635 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
1636 && PGM_SHW_TYPE != PGM_TYPE_EPT
1637
1638# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1639 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
1640# endif
1641
1642 /*
1643 * Assert preconditions.
1644 */
1645 Assert(PdeSrc.n.u1Present);
1646 Assert(cPages);
1647 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1648
1649 /*
1650 * Get the shadow PDE, find the shadow page table in the pool.
1651 */
1652# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1653 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1654 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
1655
1656 /* Fetch the pgm pool shadow descriptor. */
1657 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1658 Assert(pShwPde);
1659
1660# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1661 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1662 PPGMPOOLPAGE pShwPde = NULL;
1663 PX86PDPAE pPDDst;
1664
1665 /* Fetch the pgm pool shadow descriptor. */
1666 int rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1667 AssertRCSuccessReturn(rc, rc);
1668 Assert(pShwPde);
1669
1670 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1671 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1672
1673# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1674 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1675 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1676 PX86PDPAE pPDDst;
1677 PX86PDPT pPdptDst;
1678
1679 int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1680 AssertRCSuccessReturn(rc, rc);
1681 Assert(pPDDst && pPdptDst);
1682 PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1683# endif
1684 SHWPDE PdeDst = *pPdeDst;
1685 if (!PdeDst.n.u1Present)
1686 {
1687 AssertMsg(pVM->cCPUs > 1, ("%Unexpected missing PDE p=%llx\n", pPdeDst, (uint64_t)PdeDst.u));
1688 Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", GCPtrPage));
1689 return VINF_SUCCESS; /* force the instruction to be executed again. */
1690 }
1691
1692 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1693
1694# if PGM_GST_TYPE == PGM_TYPE_AMD64
1695 /* Fetch the pgm pool shadow descriptor. */
1696 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1697 Assert(pShwPde);
1698# endif
1699
1700# if defined(IN_RC)
1701 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1702 PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1703# endif
1704
1705 /*
1706 * Check that the page is present and that the shadow PDE isn't out of sync.
1707 */
1708# if PGM_GST_TYPE == PGM_TYPE_AMD64
1709 const bool fBigPage = PdeSrc.b.u1Size;
1710# else
1711 const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1712# endif
1713 RTGCPHYS GCPhys;
1714 if (!fBigPage)
1715 {
1716 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1717# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1718 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1719 GCPhys |= (iPDDst & 1) * (PAGE_SIZE/2);
1720# endif
1721 }
1722 else
1723 {
1724 GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1725# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1726 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1727 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1728# endif
1729 }
1730 if ( pShwPage->GCPhys == GCPhys
1731 && PdeSrc.n.u1Present
1732 && (PdeSrc.n.u1User == PdeDst.n.u1User)
1733 && (PdeSrc.n.u1Write == PdeDst.n.u1Write || !PdeDst.n.u1Write)
1734# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1735 && (!fNoExecuteBitValid || PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1736# endif
1737 )
1738 {
1739 /*
1740 * Check that the PDE is marked accessed already.
1741 * Since we set the accessed bit *before* getting here on a #PF, this
1742 * check is only meant for dealing with non-#PF'ing paths.
1743 */
1744 if (PdeSrc.n.u1Accessed)
1745 {
1746 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1747 if (!fBigPage)
1748 {
1749 /*
1750 * 4KB Page - Map the guest page table.
1751 */
1752 PGSTPT pPTSrc;
1753 int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1754 if (RT_SUCCESS(rc))
1755 {
1756# ifdef PGM_SYNC_N_PAGES
1757 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1758 if ( cPages > 1
1759 && !(uErr & X86_TRAP_PF_P)
1760 && !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1761 {
1762 /*
1763 * This code path is currently only taken when the caller is PGMTrap0eHandler
1764 * for non-present pages!
1765 *
1766 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1767 * deal with locality.
1768 */
1769 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1770# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1771 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1772 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1773# else
1774 const unsigned offPTSrc = 0;
1775# endif
1776 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1777 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1778 iPTDst = 0;
1779 else
1780 iPTDst -= PGM_SYNC_NR_PAGES / 2;
1781 for (; iPTDst < iPTDstEnd; iPTDst++)
1782 {
1783 if (!pPTDst->a[iPTDst].n.u1Present)
1784 {
1785 GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1786 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) | ((offPTSrc + iPTDst) << PAGE_SHIFT);
1787 NOREF(GCPtrCurPage);
1788#ifndef IN_RING0
1789 /*
1790 * Assuming kernel code will be marked as supervisor - and not as user level
1791 * and executed using a conforming code selector - And marked as readonly.
1792 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
1793 */
1794 PPGMPAGE pPage;
1795 if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW | X86_PTE_US))
1796 || iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1797 || !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1798 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1799 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1800 )
1801#endif /* else: CSAM not active */
1802 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1803 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1804 GCPtrCurPage, PteSrc.n.u1Present,
1805 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1806 PteSrc.n.u1User & PdeSrc.n.u1User,
1807 (uint64_t)PteSrc.u,
1808 (uint64_t)pPTDst->a[iPTDst].u,
1809 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1810 }
1811 }
1812 }
1813 else
1814# endif /* PGM_SYNC_N_PAGES */
1815 {
1816 const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1817 GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1818 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1819 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1820 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1821 GCPtrPage, PteSrc.n.u1Present,
1822 PteSrc.n.u1Write & PdeSrc.n.u1Write,
1823 PteSrc.n.u1User & PdeSrc.n.u1User,
1824 (uint64_t)PteSrc.u,
1825 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1826 }
1827 }
1828 else /* MMIO or invalid page: emulated in #PF handler. */
1829 {
1830 LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1831 Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1832 }
1833 }
1834 else
1835 {
1836 /*
1837 * 4/2MB page - lazy syncing shadow 4K pages.
1838 * (There are many causes of getting here, it's no longer only CSAM.)
1839 */
1840 /* Calculate the GC physical address of this 4KB shadow page. */
1841 RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1842 /* Find ram range. */
1843 PPGMPAGE pPage;
1844 int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1845 if (RT_SUCCESS(rc))
1846 {
1847# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1848 /* Try make the page writable if necessary. */
1849 if ( PdeSrc.n.u1Write
1850 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1851 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1852 {
1853 rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
1854 AssertRC(rc);
1855 }
1856# endif
1857
1858 /*
1859 * Make shadow PTE entry.
1860 */
1861 SHWPTE PteDst;
1862 PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1863 | PGM_PAGE_GET_HCPHYS(pPage);
1864 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1865 {
1866 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1867 PteDst.n.u1Write = 0;
1868 else
1869 PteDst.u = 0;
1870 }
1871 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1872# ifdef PGMPOOL_WITH_USER_TRACKING
1873 if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1874 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1875# endif
1876 /* Make sure only allocated pages are mapped writable. */
1877 if ( PteDst.n.u1Write
1878 && PteDst.n.u1Present
1879 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1880 {
1881 PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1882 Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1883 }
1884
1885 ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1886
1887 /*
1888 * If the page is not flagged as dirty and is writable, then make it read-only
1889 * at PD level, so we can set the dirty bit when the page is modified.
1890 *
1891 * ASSUMES that page access handlers are implemented on page table entry level.
1892 * Thus we will first catch the dirty access and set PDE.D and restart. If
1893 * there is an access handler, we'll trap again and let it work on the problem.
1894 */
1895 /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1896 * As for invlpg, it simply frees the whole shadow PT.
1897 * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1898 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1899 {
1900 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
1901 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
1902 PdeDst.n.u1Write = 0;
1903 }
1904 else
1905 {
1906 PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1907 PdeDst.n.u1Write = PdeSrc.n.u1Write;
1908 }
1909 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
1910 Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1911 GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1912 PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1913 }
1914 else
1915 LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1916 }
1917# if defined(IN_RC)
1918 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1919 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1920# endif
1921 return VINF_SUCCESS;
1922 }
1923 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDNAs));
1924 }
1925 else
1926 {
1927 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1928 Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1929 GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1930 }
1931
1932 /*
1933 * Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1934 * Yea, I'm lazy.
1935 */
1936 pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1937 ASMAtomicWriteSize(pPdeDst, 0);
1938
1939# if defined(IN_RC)
1940 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1941 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1942# endif
1943 PGM_INVL_VCPU_TLBS(pVCpu);
1944 return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1945
1946#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
1947 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
1948 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
1949 && !defined(IN_RC)
1950
1951# ifdef PGM_SYNC_N_PAGES
1952 /*
1953 * Get the shadow PDE, find the shadow page table in the pool.
1954 */
1955# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1956 X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
1957
1958# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1959 X86PDEPAE PdeDst = pgmShwGetPaePDE(&pVCpu->pgm.s, GCPtrPage);
1960
1961# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1962 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1963 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
1964 PX86PDPAE pPDDst;
1965 X86PDEPAE PdeDst;
1966 PX86PDPT pPdptDst;
1967
1968 int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1969 AssertRCSuccessReturn(rc, rc);
1970 Assert(pPDDst && pPdptDst);
1971 PdeDst = pPDDst->a[iPDDst];
1972# elif PGM_SHW_TYPE == PGM_TYPE_EPT
1973 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1974 PEPTPD pPDDst;
1975 EPTPDE PdeDst;
1976
1977 int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
1978 if (rc != VINF_SUCCESS)
1979 {
1980 AssertRC(rc);
1981 return rc;
1982 }
1983 Assert(pPDDst);
1984 PdeDst = pPDDst->a[iPDDst];
1985# endif
1986 AssertMsg(PdeDst.n.u1Present, ("%#llx\n", (uint64_t)PdeDst.u));
1987 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1988 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1989
1990 Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
1991 if ( cPages > 1
1992 && !(uErr & X86_TRAP_PF_P)
1993 && !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1994 {
1995 /*
1996 * This code path is currently only taken when the caller is PGMTrap0eHandler
1997 * for non-present pages!
1998 *
1999 * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2000 * deal with locality.
2001 */
2002 unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2003 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2004 if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2005 iPTDst = 0;
2006 else
2007 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2008 for (; iPTDst < iPTDstEnd; iPTDst++)
2009 {
2010 if (!pPTDst->a[iPTDst].n.u1Present)
2011 {
2012 GSTPTE PteSrc;
2013
2014 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT);
2015
2016 /* Fake the page table entry */
2017 PteSrc.u = GCPtrCurPage;
2018 PteSrc.n.u1Present = 1;
2019 PteSrc.n.u1Dirty = 1;
2020 PteSrc.n.u1Accessed = 1;
2021 PteSrc.n.u1Write = 1;
2022 PteSrc.n.u1User = 1;
2023
2024 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2025
2026 Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2027 GCPtrCurPage, PteSrc.n.u1Present,
2028 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2029 PteSrc.n.u1User & PdeSrc.n.u1User,
2030 (uint64_t)PteSrc.u,
2031 (uint64_t)pPTDst->a[iPTDst].u,
2032 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2033
2034 if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2035 break;
2036 }
2037 else
2038 Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2039 }
2040 }
2041 else
2042# endif /* PGM_SYNC_N_PAGES */
2043 {
2044 GSTPTE PteSrc;
2045 const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2046 RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << PAGE_SHIFT);
2047
2048 /* Fake the page table entry */
2049 PteSrc.u = GCPtrCurPage;
2050 PteSrc.n.u1Present = 1;
2051 PteSrc.n.u1Dirty = 1;
2052 PteSrc.n.u1Accessed = 1;
2053 PteSrc.n.u1Write = 1;
2054 PteSrc.n.u1User = 1;
2055 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2056
2057 Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2058 GCPtrPage, PteSrc.n.u1Present,
2059 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2060 PteSrc.n.u1User & PdeSrc.n.u1User,
2061 (uint64_t)PteSrc.u,
2062 (uint64_t)pPTDst->a[iPTDst].u,
2063 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2064 }
2065 return VINF_SUCCESS;
2066
2067#else
2068 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2069 return VERR_INTERNAL_ERROR;
2070#endif
2071}
2072
2073
2074#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2075/**
2076 * Investigate page fault and handle write protection page faults caused by
2077 * dirty bit tracking.
2078 *
2079 * @returns VBox status code.
2080 * @param pVCpu The VMCPU handle.
2081 * @param uErr Page fault error code.
2082 * @param pPdeDst Shadow page directory entry.
2083 * @param pPdeSrc Guest page directory entry.
2084 * @param GCPtrPage Guest context page address.
2085 */
2086PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage)
2087{
2088 bool fWriteProtect = !!(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP);
2089 bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
2090 bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
2091# if PGM_GST_TYPE == PGM_TYPE_AMD64
2092 bool fBigPagesSupported = true;
2093# else
2094 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2095# endif
2096# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2097 bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2098# endif
2099 unsigned uPageFaultLevel;
2100 int rc;
2101 PVM pVM = pVCpu->CTX_SUFF(pVM);
2102 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2103
2104 Assert(PGMIsLockOwner(pVM));
2105
2106 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2107 LogFlow(("CheckPageFault: GCPtrPage=%RGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
2108
2109# if PGM_GST_TYPE == PGM_TYPE_PAE \
2110 || PGM_GST_TYPE == PGM_TYPE_AMD64
2111
2112# if PGM_GST_TYPE == PGM_TYPE_AMD64
2113 PX86PML4E pPml4eSrc;
2114 PX86PDPE pPdpeSrc;
2115
2116 pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc);
2117 Assert(pPml4eSrc);
2118
2119 /*
2120 * Real page fault? (PML4E level)
2121 */
2122 if ( (uErr & X86_TRAP_PF_RSVD)
2123 || !pPml4eSrc->n.u1Present
2124 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
2125 || (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
2126 || (fUserLevelFault && !pPml4eSrc->n.u1User)
2127 )
2128 {
2129 uPageFaultLevel = 0;
2130 goto l_UpperLevelPageFault;
2131 }
2132 Assert(pPdpeSrc);
2133
2134# else /* PAE */
2135 PX86PDPE pPdpeSrc = pgmGstGetPaePDPEPtr(&pVCpu->pgm.s, GCPtrPage);
2136# endif /* PAE */
2137
2138 /*
2139 * Real page fault? (PDPE level)
2140 */
2141 if ( (uErr & X86_TRAP_PF_RSVD)
2142 || !pPdpeSrc->n.u1Present
2143# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
2144 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
2145 || (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault || fWriteProtect))
2146 || (fUserLevelFault && !pPdpeSrc->lm.u1User)
2147# endif
2148 )
2149 {
2150 uPageFaultLevel = 1;
2151 goto l_UpperLevelPageFault;
2152 }
2153# endif
2154
2155 /*
2156 * Real page fault? (PDE level)
2157 */
2158 if ( (uErr & X86_TRAP_PF_RSVD)
2159 || !pPdeSrc->n.u1Present
2160# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2161 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
2162# endif
2163 || (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault || fWriteProtect))
2164 || (fUserLevelFault && !pPdeSrc->n.u1User) )
2165 {
2166 uPageFaultLevel = 2;
2167 goto l_UpperLevelPageFault;
2168 }
2169
2170 /*
2171 * First check the easy case where the page directory has been marked read-only to track
2172 * the dirty bit of an emulated BIG page
2173 */
2174 if (pPdeSrc->b.u1Size && fBigPagesSupported)
2175 {
2176 /* Mark guest page directory as accessed */
2177# if PGM_GST_TYPE == PGM_TYPE_AMD64
2178 pPml4eSrc->n.u1Accessed = 1;
2179 pPdpeSrc->lm.u1Accessed = 1;
2180# endif
2181 pPdeSrc->b.u1Accessed = 1;
2182
2183 /*
2184 * Only write protection page faults are relevant here.
2185 */
2186 if (fWriteFault)
2187 {
2188 /* Mark guest page directory as dirty (BIG page only). */
2189 pPdeSrc->b.u1Dirty = 1;
2190
2191 if (pPdeDst->n.u1Present)
2192 {
2193 if (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY)
2194 {
2195 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2196 Assert(pPdeSrc->b.u1Write);
2197
2198 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2199 * fault again and take this path to only invalidate the entry.
2200 */
2201 pPdeDst->n.u1Write = 1;
2202 pPdeDst->n.u1Accessed = 1;
2203 pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2204 PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2205 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2206 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2207 }
2208# ifdef IN_RING0
2209 else
2210 /* Check for stale TLB entry; only applies to the SMP guest case. */
2211 if ( pVM->cCPUs > 1
2212 && pPdeDst->n.u1Write
2213 && pPdeDst->n.u1Accessed)
2214 {
2215 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2216 if (pShwPage)
2217 {
2218 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2219 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2220 if ( pPteDst->n.u1Present
2221 && pPteDst->n.u1Write)
2222 {
2223 /* Stale TLB entry. */
2224 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2225 PGM_INVL_PG(pVCpu, GCPtrPage);
2226
2227 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2228 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2229 }
2230 }
2231 }
2232# endif /* IN_RING0 */
2233 }
2234 }
2235 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2236 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2237 }
2238 /* else: 4KB page table */
2239
2240 /*
2241 * Map the guest page table.
2242 */
2243 PGSTPT pPTSrc;
2244 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2245 if (RT_SUCCESS(rc))
2246 {
2247 /*
2248 * Real page fault?
2249 */
2250 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2251 const GSTPTE PteSrc = *pPteSrc;
2252 if ( !PteSrc.n.u1Present
2253# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2254 || (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2255# endif
2256 || (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault || fWriteProtect))
2257 || (fUserLevelFault && !PteSrc.n.u1User)
2258 )
2259 {
2260 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2261 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2262 LogFlow(("CheckPageFault: real page fault at %RGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2263
2264 /* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2265 * See the 2nd case above as well.
2266 */
2267 if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2268 TRPMSetErrorCode(pVCpu, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2269
2270 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2271 return VINF_EM_RAW_GUEST_TRAP;
2272 }
2273 LogFlow(("CheckPageFault: page fault at %RGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2274
2275 /*
2276 * Set the accessed bits in the page directory and the page table.
2277 */
2278# if PGM_GST_TYPE == PGM_TYPE_AMD64
2279 pPml4eSrc->n.u1Accessed = 1;
2280 pPdpeSrc->lm.u1Accessed = 1;
2281# endif
2282 pPdeSrc->n.u1Accessed = 1;
2283 pPteSrc->n.u1Accessed = 1;
2284
2285 /*
2286 * Only write protection page faults are relevant here.
2287 */
2288 if (fWriteFault)
2289 {
2290 /* Write access, so mark guest entry as dirty. */
2291# ifdef VBOX_WITH_STATISTICS
2292 if (!pPteSrc->n.u1Dirty)
2293 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtiedPage));
2294 else
2295 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageAlreadyDirty));
2296# endif
2297
2298 pPteSrc->n.u1Dirty = 1;
2299
2300 if (pPdeDst->n.u1Present)
2301 {
2302#ifndef IN_RING0
2303 /* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2304 * Our individual shadow handlers will provide more information and force a fatal exit.
2305 */
2306 if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2307 {
2308 LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2309 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2310 return VINF_SUCCESS;
2311 }
2312#endif
2313 /*
2314 * Map shadow page table.
2315 */
2316 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2317 if (pShwPage)
2318 {
2319 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2320 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2321 if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2322 {
2323 if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2324 {
2325 LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2326# ifdef VBOX_STRICT
2327 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2328 if (pPage)
2329 AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
2330 ("Unexpected dirty bit tracking on monitored page %RGv (phys %RGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
2331# endif
2332 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2333
2334 Assert(pPteSrc->n.u1Write);
2335
2336 /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2337 * fault again and take this path to only invalidate the entry.
2338 */
2339 pPteDst->n.u1Write = 1;
2340 pPteDst->n.u1Dirty = 1;
2341 pPteDst->n.u1Accessed = 1;
2342 pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2343 PGM_INVL_PG(pVCpu, GCPtrPage);
2344
2345 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2346 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2347 }
2348# ifdef IN_RING0
2349 else
2350 /* Check for stale TLB entry; only applies to the SMP guest case. */
2351 if ( pVM->cCPUs > 1
2352 && pPteDst->n.u1Write == 1
2353 && pPteDst->n.u1Accessed == 1)
2354 {
2355 /* Stale TLB entry. */
2356 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2357 PGM_INVL_PG(pVCpu, GCPtrPage);
2358
2359 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2360 return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2361 }
2362# endif
2363 }
2364 }
2365 else
2366 AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2367 }
2368 }
2369/** @todo Optimize accessed bit emulation? */
2370# ifdef VBOX_STRICT
2371 /*
2372 * Sanity check.
2373 */
2374 else if ( !pPteSrc->n.u1Dirty
2375 && (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2376 && pPdeDst->n.u1Present)
2377 {
2378 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2379 PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2380 PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2381 if ( pPteDst->n.u1Present
2382 && pPteDst->n.u1Write)
2383 LogFlow(("Writable present page %RGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2384 }
2385# endif /* VBOX_STRICT */
2386 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2387 return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2388 }
2389 AssertRC(rc);
2390 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2391 return rc;
2392
2393
2394l_UpperLevelPageFault:
2395 /*
2396 * Pagefault detected while checking the PML4E, PDPE or PDE.
2397 * Single exit handler to get rid of duplicate code paths.
2398 */
2399 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2400 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2401 Log(("CheckPageFault: real page fault at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2402
2403 if (
2404# if PGM_GST_TYPE == PGM_TYPE_AMD64
2405 pPml4eSrc->n.u1Present &&
2406# endif
2407# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
2408 pPdpeSrc->n.u1Present &&
2409# endif
2410 pPdeSrc->n.u1Present)
2411 {
2412 /* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2413 if (pPdeSrc->b.u1Size && fBigPagesSupported)
2414 {
2415 TRPMSetErrorCode(pVCpu, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2416 }
2417 else
2418 {
2419 /*
2420 * Map the guest page table.
2421 */
2422 PGSTPT pPTSrc;
2423 rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2424 if (RT_SUCCESS(rc))
2425 {
2426 PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2427 const GSTPTE PteSrc = *pPteSrc;
2428 if (pPteSrc->n.u1Present)
2429 TRPMSetErrorCode(pVCpu, uErr | X86_TRAP_PF_P); /* page-level protection violation */
2430 }
2431 AssertRC(rc);
2432 }
2433 }
2434 return VINF_EM_RAW_GUEST_TRAP;
2435}
2436#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2437
2438
2439/**
2440 * Sync a shadow page table.
2441 *
2442 * The shadow page table is not present. This includes the case where
2443 * there is a conflict with a mapping.
2444 *
2445 * @returns VBox status code.
2446 * @param pVCpu The VMCPU handle.
2447 * @param iPD Page directory index.
2448 * @param pPDSrc Source page directory (i.e. Guest OS page directory).
2449 * Assume this is a temporary mapping.
2450 * @param GCPtrPage GC Pointer of the page that caused the fault
2451 */
2452PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2453{
2454 PVM pVM = pVCpu->CTX_SUFF(pVM);
2455 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2456
2457 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2458 STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2459 LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2460
2461 Assert(PGMIsLocked(pVM));
2462
2463#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2464 || PGM_GST_TYPE == PGM_TYPE_PAE \
2465 || PGM_GST_TYPE == PGM_TYPE_AMD64) \
2466 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
2467 && PGM_SHW_TYPE != PGM_TYPE_EPT
2468
2469 int rc = VINF_SUCCESS;
2470
2471 /*
2472 * Validate input a little bit.
2473 */
2474 AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2475# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2476 const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2477 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2478
2479 /* Fetch the pgm pool shadow descriptor. */
2480 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2481 Assert(pShwPde);
2482
2483# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2484 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2485 PPGMPOOLPAGE pShwPde = NULL;
2486 PX86PDPAE pPDDst;
2487 PSHWPDE pPdeDst;
2488
2489 /* Fetch the pgm pool shadow descriptor. */
2490 rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2491 AssertRCSuccessReturn(rc, rc);
2492 Assert(pShwPde);
2493
2494 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2495 pPdeDst = &pPDDst->a[iPDDst];
2496
2497# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2498 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2499 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2500 PX86PDPAE pPDDst;
2501 PX86PDPT pPdptDst;
2502 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2503 AssertRCSuccessReturn(rc, rc);
2504 Assert(pPDDst);
2505 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2506# endif
2507 SHWPDE PdeDst = *pPdeDst;
2508
2509# if PGM_GST_TYPE == PGM_TYPE_AMD64
2510 /* Fetch the pgm pool shadow descriptor. */
2511 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2512 Assert(pShwPde);
2513# endif
2514
2515# ifndef PGM_WITHOUT_MAPPINGS
2516 /*
2517 * Check for conflicts.
2518 * GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2519 * HC: Simply resolve the conflict.
2520 */
2521 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2522 {
2523 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2524# ifndef IN_RING3
2525 Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2526 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2527 return VERR_ADDRESS_CONFLICT;
2528# else
2529 PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2530 Assert(pMapping);
2531# if PGM_GST_TYPE == PGM_TYPE_32BIT
2532 int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2533# elif PGM_GST_TYPE == PGM_TYPE_PAE
2534 int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2535# else
2536 AssertFailed(); /* can't happen for amd64 */
2537# endif
2538 if (RT_FAILURE(rc))
2539 {
2540 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2541 return rc;
2542 }
2543 PdeDst = *pPdeDst;
2544# endif
2545 }
2546# else /* PGM_WITHOUT_MAPPINGS */
2547 Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2548# endif /* PGM_WITHOUT_MAPPINGS */
2549 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2550
2551# if defined(IN_RC)
2552 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2553 PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
2554# endif
2555
2556 /*
2557 * Sync page directory entry.
2558 */
2559 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2560 if (PdeSrc.n.u1Present)
2561 {
2562 /*
2563 * Allocate & map the page table.
2564 */
2565 PSHWPT pPTDst;
2566# if PGM_GST_TYPE == PGM_TYPE_AMD64
2567 const bool fPageTable = !PdeSrc.b.u1Size;
2568# else
2569 const bool fPageTable = !PdeSrc.b.u1Size || !(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2570# endif
2571 PPGMPOOLPAGE pShwPage;
2572 RTGCPHYS GCPhys;
2573 if (fPageTable)
2574 {
2575 GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2576# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2577 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2578 GCPhys |= (iPDDst & 1) * (PAGE_SIZE / 2);
2579# endif
2580 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2581 }
2582 else
2583 {
2584 GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
2585# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2586 /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2587 GCPhys |= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2588# endif
2589 rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, pShwPde->idx, iPDDst, &pShwPage);
2590 }
2591 if (rc == VINF_SUCCESS)
2592 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2593 else if (rc == VINF_PGM_CACHED_PAGE)
2594 {
2595 /*
2596 * The PT was cached, just hook it up.
2597 */
2598 if (fPageTable)
2599 PdeDst.u = pShwPage->Core.Key
2600 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2601 else
2602 {
2603 PdeDst.u = pShwPage->Core.Key
2604 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2605 /* (see explanation and assumptions further down.) */
2606 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2607 {
2608 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2609 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2610 PdeDst.b.u1Write = 0;
2611 }
2612 }
2613 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2614# if defined(IN_RC)
2615 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2616# endif
2617 return VINF_SUCCESS;
2618 }
2619 else if (rc == VERR_PGM_POOL_FLUSHED)
2620 {
2621 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2622# if defined(IN_RC)
2623 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2624# endif
2625 return VINF_PGM_SYNC_CR3;
2626 }
2627 else
2628 AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2629 PdeDst.u &= X86_PDE_AVL_MASK;
2630 PdeDst.u |= pShwPage->Core.Key;
2631
2632 /*
2633 * Page directory has been accessed (this is a fault situation, remember).
2634 */
2635 pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2636 if (fPageTable)
2637 {
2638 /*
2639 * Page table - 4KB.
2640 *
2641 * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2642 */
2643 Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2644 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2645 PGSTPT pPTSrc;
2646 rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2647 if (RT_SUCCESS(rc))
2648 {
2649 /*
2650 * Start by syncing the page directory entry so CSAM's TLB trick works.
2651 */
2652 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
2653 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2654 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2655# if defined(IN_RC)
2656 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2657# endif
2658
2659 /*
2660 * Directory/page user or supervisor privilege: (same goes for read/write)
2661 *
2662 * Directory Page Combined
2663 * U/S U/S U/S
2664 * 0 0 0
2665 * 0 1 0
2666 * 1 0 0
2667 * 1 1 1
2668 *
2669 * Simple AND operation. Table listed for completeness.
2670 *
2671 */
2672 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4K));
2673# ifdef PGM_SYNC_N_PAGES
2674 unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2675 unsigned iPTDst = iPTBase;
2676 const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2677 if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2678 iPTDst = 0;
2679 else
2680 iPTDst -= PGM_SYNC_NR_PAGES / 2;
2681# else /* !PGM_SYNC_N_PAGES */
2682 unsigned iPTDst = 0;
2683 const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2684# endif /* !PGM_SYNC_N_PAGES */
2685# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2686 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2687 const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2688# else
2689 const unsigned offPTSrc = 0;
2690# endif
2691 for (; iPTDst < iPTDstEnd; iPTDst++)
2692 {
2693 const unsigned iPTSrc = iPTDst + offPTSrc;
2694 const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2695
2696 if (PteSrc.n.u1Present) /* we've already cleared it above */
2697 {
2698# ifndef IN_RING0
2699 /*
2700 * Assuming kernel code will be marked as supervisor - and not as user level
2701 * and executed using a conforming code selector - And marked as readonly.
2702 * Also assume that if we're monitoring a page, it's of no interest to CSAM.
2703 */
2704 PPGMPAGE pPage;
2705 if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW | X86_PTE_US))
2706 || !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)))
2707 || ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2708 && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2709 )
2710# endif
2711 PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2712 Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2713 (RTGCPTR)((iPDSrc << GST_PD_SHIFT) | (iPTSrc << PAGE_SHIFT)),
2714 PteSrc.n.u1Present,
2715 PteSrc.n.u1Write & PdeSrc.n.u1Write,
2716 PteSrc.n.u1User & PdeSrc.n.u1User,
2717 (uint64_t)PteSrc.u,
2718 pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2719 (RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2720 }
2721 } /* for PTEs */
2722 }
2723 }
2724 else
2725 {
2726 /*
2727 * Big page - 2/4MB.
2728 *
2729 * We'll walk the ram range list in parallel and optimize lookups.
2730 * We will only sync on shadow page table at a time.
2731 */
2732 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4M));
2733
2734 /**
2735 * @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2736 */
2737
2738 /*
2739 * Start by syncing the page directory entry.
2740 */
2741 PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2742 | (PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PDE_AVL_MASK | X86_PDE_PCD | X86_PDE_PWT | X86_PDE_PS | X86_PDE4M_G | X86_PDE4M_D));
2743
2744 /*
2745 * If the page is not flagged as dirty and is writable, then make it read-only
2746 * at PD level, so we can set the dirty bit when the page is modified.
2747 *
2748 * ASSUMES that page access handlers are implemented on page table entry level.
2749 * Thus we will first catch the dirty access and set PDE.D and restart. If
2750 * there is an access handler, we'll trap again and let it work on the problem.
2751 */
2752 /** @todo move the above stuff to a section in the PGM documentation. */
2753 Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2754 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2755 {
2756 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2757 PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
2758 PdeDst.b.u1Write = 0;
2759 }
2760 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2761# if defined(IN_RC)
2762 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2763# endif
2764
2765 /*
2766 * Fill the shadow page table.
2767 */
2768 /* Get address and flags from the source PDE. */
2769 SHWPTE PteDstBase;
2770 PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT);
2771
2772 /* Loop thru the entries in the shadow PT. */
2773 const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2774 Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2775 GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2776 GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2777 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2778 unsigned iPTDst = 0;
2779 while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2780 && !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2781 {
2782 /* Advance ram range list. */
2783 while (pRam && GCPhys > pRam->GCPhysLast)
2784 pRam = pRam->CTX_SUFF(pNext);
2785 if (pRam && GCPhys >= pRam->GCPhys)
2786 {
2787 unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2788 do
2789 {
2790 /* Make shadow PTE. */
2791 PPGMPAGE pPage = &pRam->aPages[iHCPage];
2792 SHWPTE PteDst;
2793
2794# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2795 /* Try make the page writable if necessary. */
2796 if ( PteDstBase.n.u1Write
2797 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2798 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2799 {
2800 rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
2801 AssertRCReturn(rc, rc);
2802 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2803 break;
2804 }
2805# endif
2806
2807 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2808 {
2809 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2810 {
2811 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2812 PteDst.n.u1Write = 0;
2813 }
2814 else
2815 PteDst.u = 0;
2816 }
2817# ifndef IN_RING0
2818 /*
2819 * Assuming kernel code will be marked as supervisor and not as user level and executed
2820 * using a conforming code selector. Don't check for readonly, as that implies the whole
2821 * 4MB can be code or readonly data. Linux enables write access for its large pages.
2822 */
2823 else if ( !PdeSrc.n.u1User
2824 && CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))))
2825 PteDst.u = 0;
2826# endif
2827 else
2828 PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) | PteDstBase.u;
2829
2830 /* Only map writable pages writable. */
2831 if ( PteDst.n.u1Write
2832 && PteDst.n.u1Present
2833 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2834 {
2835 PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2836 Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
2837 }
2838
2839# ifdef PGMPOOL_WITH_USER_TRACKING
2840 if (PteDst.n.u1Present)
2841 PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2842# endif
2843 /* commit it */
2844 pPTDst->a[iPTDst] = PteDst;
2845 Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2846 (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2847 PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2848
2849 /* advance */
2850 GCPhys += PAGE_SIZE;
2851 iHCPage++;
2852 iPTDst++;
2853 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2854 && GCPhys <= pRam->GCPhysLast);
2855 }
2856 else if (pRam)
2857 {
2858 Log(("Invalid pages at %RGp\n", GCPhys));
2859 do
2860 {
2861 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2862 GCPhys += PAGE_SIZE;
2863 iPTDst++;
2864 } while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2865 && GCPhys < pRam->GCPhys);
2866 }
2867 else
2868 {
2869 Log(("Invalid pages at %RGp (2)\n", GCPhys));
2870 for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2871 pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2872 }
2873 } /* while more PTEs */
2874 } /* 4KB / 4MB */
2875 }
2876 else
2877 AssertRelease(!PdeDst.n.u1Present);
2878
2879 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2880 if (RT_FAILURE(rc))
2881 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPTFailed));
2882 return rc;
2883
2884#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
2885 && PGM_SHW_TYPE != PGM_TYPE_NESTED \
2886 && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
2887 && !defined(IN_RC)
2888
2889 /*
2890 * Validate input a little bit.
2891 */
2892 int rc = VINF_SUCCESS;
2893# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2894 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2895 PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2896
2897 /* Fetch the pgm pool shadow descriptor. */
2898 PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2899 Assert(pShwPde);
2900
2901# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2902 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2903 PPGMPOOLPAGE pShwPde;
2904 PX86PDPAE pPDDst;
2905 PSHWPDE pPdeDst;
2906
2907 /* Fetch the pgm pool shadow descriptor. */
2908 rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2909 AssertRCSuccessReturn(rc, rc);
2910 Assert(pShwPde);
2911
2912 pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2913 pPdeDst = &pPDDst->a[iPDDst];
2914
2915# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2916 const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2917 const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2918 PX86PDPAE pPDDst;
2919 PX86PDPT pPdptDst;
2920 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2921 AssertRCSuccessReturn(rc, rc);
2922 Assert(pPDDst);
2923 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2924
2925 /* Fetch the pgm pool shadow descriptor. */
2926 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2927 Assert(pShwPde);
2928
2929# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2930 const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2931 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2932 PEPTPD pPDDst;
2933 PEPTPDPT pPdptDst;
2934
2935 rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2936 if (rc != VINF_SUCCESS)
2937 {
2938 AssertRC(rc);
2939 return rc;
2940 }
2941 Assert(pPDDst);
2942 PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2943
2944 /* Fetch the pgm pool shadow descriptor. */
2945 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2946 Assert(pShwPde);
2947# endif
2948 SHWPDE PdeDst = *pPdeDst;
2949
2950 Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2951 Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2952
2953 GSTPDE PdeSrc;
2954 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2955 PdeSrc.n.u1Present = 1;
2956 PdeSrc.n.u1Write = 1;
2957 PdeSrc.n.u1Accessed = 1;
2958 PdeSrc.n.u1User = 1;
2959
2960 /*
2961 * Allocate & map the page table.
2962 */
2963 PSHWPT pPTDst;
2964 PPGMPOOLPAGE pShwPage;
2965 RTGCPHYS GCPhys;
2966
2967 /* Virtual address = physical address */
2968 GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
2969 rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2970
2971 if ( rc == VINF_SUCCESS
2972 || rc == VINF_PGM_CACHED_PAGE)
2973 pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2974 else
2975 AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2976
2977 PdeDst.u &= X86_PDE_AVL_MASK;
2978 PdeDst.u |= pShwPage->Core.Key;
2979 PdeDst.n.u1Present = 1;
2980 PdeDst.n.u1Write = 1;
2981# if PGM_SHW_TYPE == PGM_TYPE_EPT
2982 PdeDst.n.u1Execute = 1;
2983# else
2984 PdeDst.n.u1User = 1;
2985 PdeDst.n.u1Accessed = 1;
2986# endif
2987 ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2988
2989 pgmLock(pVM);
2990 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2991 pgmUnlock(pVM);
2992 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2993 return rc;
2994
2995#else
2996 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2997 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2998 return VERR_INTERNAL_ERROR;
2999#endif
3000}
3001
3002
3003
3004/**
3005 * Prefetch a page/set of pages.
3006 *
3007 * Typically used to sync commonly used pages before entering raw mode
3008 * after a CR3 reload.
3009 *
3010 * @returns VBox status code.
3011 * @param pVCpu The VMCPU handle.
3012 * @param GCPtrPage Page to invalidate.
3013 */
3014PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3015{
3016#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \
3017 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3018 /*
3019 * Check that all Guest levels thru the PDE are present, getting the
3020 * PD and PDE in the processes.
3021 */
3022 int rc = VINF_SUCCESS;
3023# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3024# if PGM_GST_TYPE == PGM_TYPE_32BIT
3025 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3026 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3027# elif PGM_GST_TYPE == PGM_TYPE_PAE
3028 unsigned iPDSrc;
3029 X86PDPE PdpeSrc;
3030 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3031 if (!pPDSrc)
3032 return VINF_SUCCESS; /* not present */
3033# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3034 unsigned iPDSrc;
3035 PX86PML4E pPml4eSrc;
3036 X86PDPE PdpeSrc;
3037 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3038 if (!pPDSrc)
3039 return VINF_SUCCESS; /* not present */
3040# endif
3041 const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3042# else
3043 PGSTPD pPDSrc = NULL;
3044 const unsigned iPDSrc = 0;
3045 GSTPDE PdeSrc;
3046
3047 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3048 PdeSrc.n.u1Present = 1;
3049 PdeSrc.n.u1Write = 1;
3050 PdeSrc.n.u1Accessed = 1;
3051 PdeSrc.n.u1User = 1;
3052# endif
3053
3054 if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3055 {
3056 PVM pVM = pVCpu->CTX_SUFF(pVM);
3057 pgmLock(pVM);
3058
3059# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3060 const X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
3061# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3062 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3063 PX86PDPAE pPDDst;
3064 X86PDEPAE PdeDst;
3065# if PGM_GST_TYPE != PGM_TYPE_PAE
3066 X86PDPE PdpeSrc;
3067
3068 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3069 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3070# endif
3071 int rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3072 if (rc != VINF_SUCCESS)
3073 {
3074 pgmUnlock(pVM);
3075 AssertRC(rc);
3076 return rc;
3077 }
3078 Assert(pPDDst);
3079 PdeDst = pPDDst->a[iPDDst];
3080
3081# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3082 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3083 PX86PDPAE pPDDst;
3084 X86PDEPAE PdeDst;
3085
3086# if PGM_GST_TYPE == PGM_TYPE_PROT
3087 /* AMD-V nested paging */
3088 X86PML4E Pml4eSrc;
3089 X86PDPE PdpeSrc;
3090 PX86PML4E pPml4eSrc = &Pml4eSrc;
3091
3092 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3093 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_NX | X86_PML4E_A;
3094 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_NX | X86_PDPE_A;
3095# endif
3096
3097 int rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3098 if (rc != VINF_SUCCESS)
3099 {
3100 pgmUnlock(pVM);
3101 AssertRC(rc);
3102 return rc;
3103 }
3104 Assert(pPDDst);
3105 PdeDst = pPDDst->a[iPDDst];
3106# endif
3107 if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3108 {
3109 if (!PdeDst.n.u1Present)
3110 {
3111 /** r=bird: This guy will set the A bit on the PDE, probably harmless. */
3112 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3113 }
3114 else
3115 {
3116 /** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3117 * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3118 * makes no sense to prefetch more than one page.
3119 */
3120 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3121 if (RT_SUCCESS(rc))
3122 rc = VINF_SUCCESS;
3123 }
3124 }
3125 pgmUnlock(pVM);
3126 }
3127 return rc;
3128
3129#elif PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3130 return VINF_SUCCESS; /* ignore */
3131#endif
3132}
3133
3134
3135
3136
3137/**
3138 * Syncs a page during a PGMVerifyAccess() call.
3139 *
3140 * @returns VBox status code (informational included).
3141 * @param pVCpu The VMCPU handle.
3142 * @param GCPtrPage The address of the page to sync.
3143 * @param fPage The effective guest page flags.
3144 * @param uErr The trap error code.
3145 */
3146PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3147{
3148 PVM pVM = pVCpu->CTX_SUFF(pVM);
3149
3150 LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3151
3152 Assert(!HWACCMIsNestedPagingActive(pVM));
3153#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_TYPE_AMD64) \
3154 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3155
3156# ifndef IN_RING0
3157 if (!(fPage & X86_PTE_US))
3158 {
3159 /*
3160 * Mark this page as safe.
3161 */
3162 /** @todo not correct for pages that contain both code and data!! */
3163 Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3164 CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
3165 }
3166# endif
3167
3168 /*
3169 * Get guest PD and index.
3170 */
3171# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3172# if PGM_GST_TYPE == PGM_TYPE_32BIT
3173 const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3174 PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3175# elif PGM_GST_TYPE == PGM_TYPE_PAE
3176 unsigned iPDSrc = 0;
3177 X86PDPE PdpeSrc;
3178 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3179
3180 if (pPDSrc)
3181 {
3182 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3183 return VINF_EM_RAW_GUEST_TRAP;
3184 }
3185# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3186 unsigned iPDSrc;
3187 PX86PML4E pPml4eSrc;
3188 X86PDPE PdpeSrc;
3189 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3190 if (!pPDSrc)
3191 {
3192 Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3193 return VINF_EM_RAW_GUEST_TRAP;
3194 }
3195# endif
3196# else
3197 PGSTPD pPDSrc = NULL;
3198 const unsigned iPDSrc = 0;
3199# endif
3200 int rc = VINF_SUCCESS;
3201
3202 pgmLock(pVM);
3203
3204 /*
3205 * First check if the shadow pd is present.
3206 */
3207# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3208 PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
3209# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3210 PX86PDEPAE pPdeDst;
3211 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3212 PX86PDPAE pPDDst;
3213# if PGM_GST_TYPE != PGM_TYPE_PAE
3214 X86PDPE PdpeSrc;
3215
3216 /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3217 PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3218# endif
3219 rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3220 if (rc != VINF_SUCCESS)
3221 {
3222 pgmUnlock(pVM);
3223 AssertRC(rc);
3224 return rc;
3225 }
3226 Assert(pPDDst);
3227 pPdeDst = &pPDDst->a[iPDDst];
3228
3229# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3230 const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3231 PX86PDPAE pPDDst;
3232 PX86PDEPAE pPdeDst;
3233
3234# if PGM_GST_TYPE == PGM_TYPE_PROT
3235 /* AMD-V nested paging */
3236 X86PML4E Pml4eSrc;
3237 X86PDPE PdpeSrc;
3238 PX86PML4E pPml4eSrc = &Pml4eSrc;
3239
3240 /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3241 Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_NX | X86_PML4E_A;
3242 PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_NX | X86_PDPE_A;
3243# endif
3244
3245 rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3246 if (rc != VINF_SUCCESS)
3247 {
3248 pgmUnlock(pVM);
3249 AssertRC(rc);
3250 return rc;
3251 }
3252 Assert(pPDDst);
3253 pPdeDst = &pPDDst->a[iPDDst];
3254# endif
3255
3256# if defined(IN_RC)
3257 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3258 PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
3259# endif
3260
3261 if (!pPdeDst->n.u1Present)
3262 {
3263 rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3264 if (rc != VINF_SUCCESS)
3265 {
3266# if defined(IN_RC)
3267 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3268 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3269# endif
3270 pgmUnlock(pVM);
3271 AssertRC(rc);
3272 return rc;
3273 }
3274 }
3275
3276# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3277 /* Check for dirty bit fault */
3278 rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3279 if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3280 Log(("PGMVerifyAccess: success (dirty)\n"));
3281 else
3282 {
3283 GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3284# else
3285 {
3286 GSTPDE PdeSrc;
3287 PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3288 PdeSrc.n.u1Present = 1;
3289 PdeSrc.n.u1Write = 1;
3290 PdeSrc.n.u1Accessed = 1;
3291 PdeSrc.n.u1User = 1;
3292
3293# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
3294 Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3295 if (uErr & X86_TRAP_PF_US)
3296 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
3297 else /* supervisor */
3298 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3299
3300 rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3301 if (RT_SUCCESS(rc))
3302 {
3303 /* Page was successfully synced */
3304 Log2(("PGMVerifyAccess: success (sync)\n"));
3305 rc = VINF_SUCCESS;
3306 }
3307 else
3308 {
3309 Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", GCPtrPage, rc));
3310 rc = VINF_EM_RAW_GUEST_TRAP;
3311 }
3312 }
3313# if defined(IN_RC)
3314 /* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3315 PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3316# endif
3317 pgmUnlock(pVM);
3318 return rc;
3319
3320#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3321
3322 AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3323 return VERR_INTERNAL_ERROR;
3324#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3325}
3326
3327
3328#if PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64
3329# if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
3330/**
3331 * Figures out which kind of shadow page this guest PDE warrants.
3332 *
3333 * @returns Shadow page kind.
3334 * @param pPdeSrc The guest PDE in question.
3335 * @param cr4 The current guest cr4 value.
3336 */
3337DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
3338{
3339# if PMG_GST_TYPE == PGM_TYPE_AMD64
3340 if (!pPdeSrc->n.u1Size)
3341# else
3342 if (!pPdeSrc->n.u1Size || !(cr4 & X86_CR4_PSE))
3343# endif
3344 return BTH_PGMPOOLKIND_PT_FOR_PT;
3345 //switch (pPdeSrc->u & (X86_PDE4M_RW | X86_PDE4M_US /*| X86_PDE4M_PAE_NX*/))
3346 //{
3347 // case 0:
3348 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
3349 // case X86_PDE4M_RW:
3350 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
3351 // case X86_PDE4M_US:
3352 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
3353 // case X86_PDE4M_RW | X86_PDE4M_US:
3354 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
3355# if 0
3356 // case X86_PDE4M_PAE_NX:
3357 // return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
3358 // case X86_PDE4M_RW | X86_PDE4M_PAE_NX:
3359 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
3360 // case X86_PDE4M_US | X86_PDE4M_PAE_NX:
3361 // return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
3362 // case X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PAE_NX:
3363 // return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
3364# endif
3365 return BTH_PGMPOOLKIND_PT_FOR_BIG;
3366 //}
3367}
3368# endif
3369#endif
3370
3371#undef MY_STAM_COUNTER_INC
3372#define MY_STAM_COUNTER_INC(a) do { } while (0)
3373
3374
3375/**
3376 * Syncs the paging hierarchy starting at CR3.
3377 *
3378 * @returns VBox status code, no specials.
3379 * @param pVCpu The VMCPU handle.
3380 * @param cr0 Guest context CR0 register
3381 * @param cr3 Guest context CR3 register
3382 * @param cr4 Guest context CR4 register
3383 * @param fGlobal Including global page directories or not
3384 */
3385PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3386{
3387 PVM pVM = pVCpu->CTX_SUFF(pVM);
3388
3389 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
3390 fGlobal = true; /* Change this CR3 reload to be a global one. */
3391
3392 LogFlow(("SyncCR3 %d\n", fGlobal));
3393
3394#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3395 /*
3396 * Update page access handlers.
3397 * The virtual are always flushed, while the physical are only on demand.
3398 * WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3399 * have to look into that later because it will have a bad influence on the performance.
3400 * @note SvL: There's no need for that. Just invalidate the virtual range(s).
3401 * bird: Yes, but that won't work for aliases.
3402 */
3403 /** @todo this MUST go away. See #1557. */
3404 STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3405 PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3406 STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3407#endif
3408
3409#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3410 /*
3411 * Nested / EPT - almost no work.
3412 */
3413 /** @todo check if this is really necessary; the call does it as well... */
3414 HWACCMFlushTLB(pVCpu);
3415 return VINF_SUCCESS;
3416
3417#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3418 /*
3419 * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3420 * out the shadow parts when the guest modifies its tables.
3421 */
3422 return VINF_SUCCESS;
3423
3424#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3425
3426# ifdef PGM_WITHOUT_MAPPINGS
3427 Assert(pVM->pgm.s.fMappingsFixed);
3428 return VINF_SUCCESS;
3429# else
3430 /* Nothing to do when mappings are fixed. */
3431 if (pVM->pgm.s.fMappingsFixed)
3432 return VINF_SUCCESS;
3433
3434 int rc = PGMMapResolveConflicts(pVM);
3435 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3);
3436 if (rc == VINF_PGM_SYNC_CR3)
3437 {
3438 LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3439 return VINF_PGM_SYNC_CR3;
3440 }
3441# endif
3442 return VINF_SUCCESS;
3443#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3444}
3445
3446
3447
3448
3449#ifdef VBOX_STRICT
3450#ifdef IN_RC
3451# undef AssertMsgFailed
3452# define AssertMsgFailed Log
3453#endif
3454#ifdef IN_RING3
3455# include <VBox/dbgf.h>
3456
3457/**
3458 * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3459 *
3460 * @returns VBox status code (VINF_SUCCESS).
3461 * @param cr3 The root of the hierarchy.
3462 * @param crr The cr4, only PAE and PSE is currently used.
3463 * @param fLongMode Set if long mode, false if not long mode.
3464 * @param cMaxDepth Number of levels to dump.
3465 * @param pHlp Pointer to the output functions.
3466 */
3467__BEGIN_DECLS
3468VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3469__END_DECLS
3470
3471#endif
3472
3473/**
3474 * Checks that the shadow page table is in sync with the guest one.
3475 *
3476 * @returns The number of errors.
3477 * @param pVM The virtual machine.
3478 * @param pVCpu The VMCPU handle.
3479 * @param cr3 Guest context CR3 register
3480 * @param cr4 Guest context CR4 register
3481 * @param GCPtr Where to start. Defaults to 0.
3482 * @param cb How much to check. Defaults to everything.
3483 */
3484PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3485{
3486#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
3487 return 0;
3488#else
3489 unsigned cErrors = 0;
3490 PVM pVM = pVCpu->CTX_SUFF(pVM);
3491 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3492
3493#if PGM_GST_TYPE == PGM_TYPE_PAE
3494 /** @todo currently broken; crashes below somewhere */
3495 AssertFailed();
3496#endif
3497
3498#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3499 || PGM_GST_TYPE == PGM_TYPE_PAE \
3500 || PGM_GST_TYPE == PGM_TYPE_AMD64
3501
3502# if PGM_GST_TYPE == PGM_TYPE_AMD64
3503 bool fBigPagesSupported = true;
3504# else
3505 bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
3506# endif
3507 PPGMCPU pPGM = &pVCpu->pgm.s;
3508 RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3509 RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3510# ifndef IN_RING0
3511 RTHCPHYS HCPhys; /* general usage. */
3512# endif
3513 int rc;
3514
3515 /*
3516 * Check that the Guest CR3 and all its mappings are correct.
3517 */
3518 AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3519 ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3520 false);
3521# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3522# if PGM_GST_TYPE == PGM_TYPE_32BIT
3523 rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3524# else
3525 rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3526# endif
3527 AssertRCReturn(rc, 1);
3528 HCPhys = NIL_RTHCPHYS;
3529 rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3530 AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3531# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3532 pgmGstGet32bitPDPtr(pPGM);
3533 RTGCPHYS GCPhys;
3534 rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3535 AssertRCReturn(rc, 1);
3536 AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3537# endif
3538# endif /* !IN_RING0 */
3539
3540 /*
3541 * Get and check the Shadow CR3.
3542 */
3543# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3544 unsigned cPDEs = X86_PG_ENTRIES;
3545 unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3546# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3547# if PGM_GST_TYPE == PGM_TYPE_32BIT
3548 unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3549# else
3550 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3551# endif
3552 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3553# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3554 unsigned cPDEs = X86_PG_PAE_ENTRIES;
3555 unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3556# endif
3557 if (cb != ~(RTGCPTR)0)
3558 cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3559
3560/** @todo call the other two PGMAssert*() functions. */
3561
3562# if PGM_GST_TYPE == PGM_TYPE_AMD64
3563 unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3564
3565 for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3566 {
3567 PPGMPOOLPAGE pShwPdpt = NULL;
3568 PX86PML4E pPml4eSrc;
3569 PX86PML4E pPml4eDst;
3570 RTGCPHYS GCPhysPdptSrc;
3571
3572 pPml4eSrc = pgmGstGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3573 pPml4eDst = pgmShwGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3574
3575 /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3576 if (!pPml4eDst->n.u1Present)
3577 {
3578 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3579 continue;
3580 }
3581
3582 pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3583 GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3584
3585 if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3586 {
3587 AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3588 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3589 cErrors++;
3590 continue;
3591 }
3592
3593 if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3594 {
3595 AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3596 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3597 cErrors++;
3598 continue;
3599 }
3600
3601 if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3602 || pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3603 || pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3604 {
3605 AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3606 GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3607 cErrors++;
3608 continue;
3609 }
3610# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3611 {
3612# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3613
3614# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
3615 /*
3616 * Check the PDPTEs too.
3617 */
3618 unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3619
3620 for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3621 {
3622 unsigned iPDSrc;
3623 PPGMPOOLPAGE pShwPde = NULL;
3624 PX86PDPE pPdpeDst;
3625 RTGCPHYS GCPhysPdeSrc;
3626# if PGM_GST_TYPE == PGM_TYPE_PAE
3627 X86PDPE PdpeSrc;
3628 PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtr, &iPDSrc, &PdpeSrc);
3629 PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
3630# else
3631 PX86PML4E pPml4eSrc;
3632 X86PDPE PdpeSrc;
3633 PX86PDPT pPdptDst;
3634 PX86PDPAE pPDDst;
3635 PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3636
3637 rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3638 if (rc != VINF_SUCCESS)
3639 {
3640 AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3641 GCPtr += 512 * _2M;
3642 continue; /* next PDPTE */
3643 }
3644 Assert(pPDDst);
3645# endif
3646 Assert(iPDSrc == 0);
3647
3648 pPdpeDst = &pPdptDst->a[iPdpt];
3649
3650 if (!pPdpeDst->n.u1Present)
3651 {
3652 GCPtr += 512 * _2M;
3653 continue; /* next PDPTE */
3654 }
3655
3656 pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3657 GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3658
3659 if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3660 {
3661 AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3662 GCPtr += 512 * _2M;
3663 cErrors++;
3664 continue;
3665 }
3666
3667 if (GCPhysPdeSrc != pShwPde->GCPhys)
3668 {
3669# if PGM_GST_TYPE == PGM_TYPE_AMD64
3670 AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3671# else
3672 AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3673# endif
3674 GCPtr += 512 * _2M;
3675 cErrors++;
3676 continue;
3677 }
3678
3679# if PGM_GST_TYPE == PGM_TYPE_AMD64
3680 if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3681 || pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3682 || pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3683 {
3684 AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3685 GCPtr += 512 * _2M;
3686 cErrors++;
3687 continue;
3688 }
3689# endif
3690
3691# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3692 {
3693# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3694# if PGM_GST_TYPE == PGM_TYPE_32BIT
3695 GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3696# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3697 PCX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
3698# endif
3699# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3700 /*
3701 * Iterate the shadow page directory.
3702 */
3703 GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3704 unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3705
3706 for (;
3707 iPDDst < cPDEs;
3708 iPDDst++, GCPtr += cIncrement)
3709 {
3710# if PGM_SHW_TYPE == PGM_TYPE_PAE
3711 const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pPGM, GCPtr);
3712# else
3713 const SHWPDE PdeDst = pPDDst->a[iPDDst];
3714# endif
3715 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3716 {
3717 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3718 if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3719 {
3720 AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3721 cErrors++;
3722 continue;
3723 }
3724 }
3725 else if ( (PdeDst.u & X86_PDE_P)
3726 || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
3727 )
3728 {
3729 HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3730 PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3731 if (!pPoolPage)
3732 {
3733 AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3734 HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3735 cErrors++;
3736 continue;
3737 }
3738 const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3739
3740 if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
3741 {
3742 AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3743 GCPtr, (uint64_t)PdeDst.u));
3744 cErrors++;
3745 }
3746
3747 if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
3748 {
3749 AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3750 GCPtr, (uint64_t)PdeDst.u));
3751 cErrors++;
3752 }
3753
3754 const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3755 if (!PdeSrc.n.u1Present)
3756 {
3757 AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3758 GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3759 cErrors++;
3760 continue;
3761 }
3762
3763 if ( !PdeSrc.b.u1Size
3764 || !fBigPagesSupported)
3765 {
3766 GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3767# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3768 GCPhysGst |= (iPDDst & 1) * (PAGE_SIZE / 2);
3769# endif
3770 }
3771 else
3772 {
3773# if PGM_GST_TYPE == PGM_TYPE_32BIT
3774 if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3775 {
3776 AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3777 GCPtr, (uint64_t)PdeSrc.u));
3778 cErrors++;
3779 continue;
3780 }
3781# endif
3782 GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3783# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3784 GCPhysGst |= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3785# endif
3786 }
3787
3788 if ( pPoolPage->enmKind
3789 != (!PdeSrc.b.u1Size || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3790 {
3791 AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3792 pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3793 cErrors++;
3794 }
3795
3796 PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3797 if (!pPhysPage)
3798 {
3799 AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3800 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3801 cErrors++;
3802 continue;
3803 }
3804
3805 if (GCPhysGst != pPoolPage->GCPhys)
3806 {
3807 AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3808 GCPhysGst, pPoolPage->GCPhys, GCPtr));
3809 cErrors++;
3810 continue;
3811 }
3812
3813 if ( !PdeSrc.b.u1Size
3814 || !fBigPagesSupported)
3815 {
3816 /*
3817 * Page Table.
3818 */
3819 const GSTPT *pPTSrc;
3820 rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3821 if (RT_FAILURE(rc))
3822 {
3823 AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3824 GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3825 cErrors++;
3826 continue;
3827 }
3828 if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
3829 != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
3830 {
3831 /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3832 // (This problem will go away when/if we shadow multiple CR3s.)
3833 AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3834 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3835 cErrors++;
3836 continue;
3837 }
3838 if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3839 {
3840 AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3841 GCPtr, (uint64_t)PdeDst.u));
3842 cErrors++;
3843 continue;
3844 }
3845
3846 /* iterate the page table. */
3847# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3848 /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3849 const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3850# else
3851 const unsigned offPTSrc = 0;
3852# endif
3853 for (unsigned iPT = 0, off = 0;
3854 iPT < RT_ELEMENTS(pPTDst->a);
3855 iPT++, off += PAGE_SIZE)
3856 {
3857 const SHWPTE PteDst = pPTDst->a[iPT];
3858
3859 /* skip not-present entries. */
3860 if (!(PteDst.u & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3861 continue;
3862 Assert(PteDst.n.u1Present);
3863
3864 const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3865 if (!PteSrc.n.u1Present)
3866 {
3867# ifdef IN_RING3
3868 PGMAssertHandlerAndFlagsInSync(pVM);
3869 PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3870# endif
3871 AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3872 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3873 (PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3874 cErrors++;
3875 continue;
3876 }
3877
3878 uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
3879# if 1 /** @todo sync accessed bit properly... */
3880 fIgnoreFlags |= X86_PTE_A;
3881# endif
3882
3883 /* match the physical addresses */
3884 HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3885 GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3886
3887# ifdef IN_RING3
3888 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3889 if (RT_FAILURE(rc))
3890 {
3891 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3892 {
3893 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3894 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3895 cErrors++;
3896 continue;
3897 }
3898 }
3899 else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3900 {
3901 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3902 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3903 cErrors++;
3904 continue;
3905 }
3906# endif
3907
3908 pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3909 if (!pPhysPage)
3910 {
3911# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3912 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3913 {
3914 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3915 GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3916 cErrors++;
3917 continue;
3918 }
3919# endif
3920 if (PteDst.n.u1Write)
3921 {
3922 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3923 GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3924 cErrors++;
3925 }
3926 fIgnoreFlags |= X86_PTE_RW;
3927 }
3928 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3929 {
3930 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3931 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3932 cErrors++;
3933 continue;
3934 }
3935
3936 /* flags */
3937 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3938 {
3939 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3940 {
3941 if (PteDst.n.u1Write)
3942 {
3943 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3944 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3945 cErrors++;
3946 continue;
3947 }
3948 fIgnoreFlags |= X86_PTE_RW;
3949 }
3950 else
3951 {
3952 if (PteDst.n.u1Present)
3953 {
3954 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3955 GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3956 cErrors++;
3957 continue;
3958 }
3959 fIgnoreFlags |= X86_PTE_P;
3960 }
3961 }
3962 else
3963 {
3964 if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3965 {
3966 if (PteDst.n.u1Write)
3967 {
3968 AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3969 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3970 cErrors++;
3971 continue;
3972 }
3973 if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3974 {
3975 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3976 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3977 cErrors++;
3978 continue;
3979 }
3980 if (PteDst.n.u1Dirty)
3981 {
3982 AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3983 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3984 cErrors++;
3985 }
3986# if 0 /** @todo sync access bit properly... */
3987 if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3988 {
3989 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3990 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3991 cErrors++;
3992 }
3993 fIgnoreFlags |= X86_PTE_RW;
3994# else
3995 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
3996# endif
3997 }
3998 else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3999 {
4000 /* access bit emulation (not implemented). */
4001 if (PteSrc.n.u1Accessed || PteDst.n.u1Present)
4002 {
4003 AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4004 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4005 cErrors++;
4006 continue;
4007 }
4008 if (!PteDst.n.u1Accessed)
4009 {
4010 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4011 GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4012 cErrors++;
4013 }
4014 fIgnoreFlags |= X86_PTE_P;
4015 }
4016# ifdef DEBUG_sandervl
4017 fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
4018# endif
4019 }
4020
4021 if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4022 && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4023 )
4024 {
4025 AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4026 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4027 fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4028 cErrors++;
4029 continue;
4030 }
4031 } /* foreach PTE */
4032 }
4033 else
4034 {
4035 /*
4036 * Big Page.
4037 */
4038 uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
4039 if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4040 {
4041 if (PdeDst.n.u1Write)
4042 {
4043 AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4044 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4045 cErrors++;
4046 continue;
4047 }
4048 if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4049 {
4050 AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4051 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4052 cErrors++;
4053 continue;
4054 }
4055# if 0 /** @todo sync access bit properly... */
4056 if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4057 {
4058 AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4059 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4060 cErrors++;
4061 }
4062 fIgnoreFlags |= X86_PTE_RW;
4063# else
4064 fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
4065# endif
4066 }
4067 else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4068 {
4069 /* access bit emulation (not implemented). */
4070 if (PdeSrc.b.u1Accessed || PdeDst.n.u1Present)
4071 {
4072 AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4073 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4074 cErrors++;
4075 continue;
4076 }
4077 if (!PdeDst.n.u1Accessed)
4078 {
4079 AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4080 GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4081 cErrors++;
4082 }
4083 fIgnoreFlags |= X86_PTE_P;
4084 }
4085
4086 if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4087 {
4088 AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4089 GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4090 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4091 cErrors++;
4092 }
4093
4094 /* iterate the page table. */
4095 for (unsigned iPT = 0, off = 0;
4096 iPT < RT_ELEMENTS(pPTDst->a);
4097 iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4098 {
4099 const SHWPTE PteDst = pPTDst->a[iPT];
4100
4101 if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4102 {
4103 AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4104 GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4105 cErrors++;
4106 }
4107
4108 /* skip not-present entries. */
4109 if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4110 continue;
4111
4112 fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
4113
4114 /* match the physical addresses */
4115 HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4116
4117# ifdef IN_RING3
4118 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4119 if (RT_FAILURE(rc))
4120 {
4121 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4122 {
4123 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4124 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4125 cErrors++;
4126 }
4127 }
4128 else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4129 {
4130 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4131 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4132 cErrors++;
4133 continue;
4134 }
4135# endif
4136 pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4137 if (!pPhysPage)
4138 {
4139# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4140 if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4141 {
4142 AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4143 GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4144 cErrors++;
4145 continue;
4146 }
4147# endif
4148 if (PteDst.n.u1Write)
4149 {
4150 AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4151 GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4152 cErrors++;
4153 }
4154 fIgnoreFlags |= X86_PTE_RW;
4155 }
4156 else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4157 {
4158 AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4159 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4160 cErrors++;
4161 continue;
4162 }
4163
4164 /* flags */
4165 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4166 {
4167 if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4168 {
4169 if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4170 {
4171 if (PteDst.n.u1Write)
4172 {
4173 AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4174 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4175 cErrors++;
4176 continue;
4177 }
4178 fIgnoreFlags |= X86_PTE_RW;
4179 }
4180 }
4181 else
4182 {
4183 if (PteDst.n.u1Present)
4184 {
4185 AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4186 GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4187 cErrors++;
4188 continue;
4189 }
4190 fIgnoreFlags |= X86_PTE_P;
4191 }
4192 }
4193
4194 if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4195 && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4196 )
4197 {
4198 AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4199 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4200 fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4201 cErrors++;
4202 continue;
4203 }
4204 } /* for each PTE */
4205 }
4206 }
4207 /* not present */
4208
4209 } /* for each PDE */
4210
4211 } /* for each PDPTE */
4212
4213 } /* for each PML4E */
4214
4215# ifdef DEBUG
4216 if (cErrors)
4217 LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4218# endif
4219
4220#endif /* GST == 32BIT, PAE or AMD64 */
4221 return cErrors;
4222
4223#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4224}
4225#endif /* VBOX_STRICT */
4226
4227
4228/**
4229 * Sets up the CR3 for shadow paging
4230 *
4231 * @returns Strict VBox status code.
4232 * @retval VINF_SUCCESS.
4233 *
4234 * @param pVCpu The VMCPU handle.
4235 * @param GCPhysCR3 The physical address in the CR3 register.
4236 */
4237PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4238{
4239 PVM pVM = pVCpu->CTX_SUFF(pVM);
4240
4241 /* Update guest paging info. */
4242#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4243 || PGM_GST_TYPE == PGM_TYPE_PAE \
4244 || PGM_GST_TYPE == PGM_TYPE_AMD64
4245
4246 LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4247
4248 /*
4249 * Map the page CR3 points at.
4250 */
4251 RTHCPTR HCPtrGuestCR3;
4252 RTHCPHYS HCPhysGuestCR3;
4253 pgmLock(pVM);
4254 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4255 AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4256 HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPage);
4257 /** @todo this needs some reworking wrt. locking. */
4258# if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4259 HCPtrGuestCR3 = NIL_RTHCPTR;
4260 int rc = VINF_SUCCESS;
4261# else
4262 int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3);
4263# endif
4264 pgmUnlock(pVM);
4265 if (RT_SUCCESS(rc))
4266 {
4267 rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4268 if (RT_SUCCESS(rc))
4269 {
4270# ifdef IN_RC
4271 PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4272# endif
4273# if PGM_GST_TYPE == PGM_TYPE_32BIT
4274 pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4275# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4276 pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4277# endif
4278 pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))pVM->pgm.s.GCPtrCR3Mapping;
4279
4280# elif PGM_GST_TYPE == PGM_TYPE_PAE
4281 unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4282 pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4283# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4284 pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4285# endif
4286 pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RCPTRTYPE(uint8_t *))pVM->pgm.s.GCPtrCR3Mapping + off);
4287 Log(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4288
4289 /*
4290 * Map the 4 PDs too.
4291 */
4292 PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(&pVCpu->pgm.s);
4293 RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4294 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4295 {
4296 if (pGuestPDPT->a[i].n.u1Present)
4297 {
4298 RTHCPTR HCPtr;
4299 RTHCPHYS HCPhys;
4300 RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4301 pgmLock(pVM);
4302 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4303 AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4304 HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4305# if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4306 HCPtr = NIL_RTHCPTR;
4307 int rc2 = VINF_SUCCESS;
4308# else
4309 int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4310# endif
4311 pgmUnlock(pVM);
4312 if (RT_SUCCESS(rc2))
4313 {
4314 rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4315 AssertRCReturn(rc, rc);
4316
4317 pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4318# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4319 pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4320# endif
4321 pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))GCPtr;
4322 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4323# ifdef IN_RC
4324 PGM_INVL_PG(pVCpu, GCPtr);
4325# endif
4326 continue;
4327 }
4328 AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4329 }
4330
4331 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4332# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4333 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4334# endif
4335 pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4336 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4337# ifdef IN_RC
4338 PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4339# endif
4340 }
4341
4342# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4343 pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4344# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4345 pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4346# endif
4347# endif
4348 }
4349 else
4350 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4351 }
4352 else
4353 AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4354
4355#else /* prot/real stub */
4356 int rc = VINF_SUCCESS;
4357#endif
4358
4359 /* Update shadow paging info for guest modes with paging (32, pae, 64). */
4360# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4361 || PGM_SHW_TYPE == PGM_TYPE_PAE \
4362 || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4363 && ( PGM_GST_TYPE != PGM_TYPE_REAL \
4364 && PGM_GST_TYPE != PGM_TYPE_PROT))
4365
4366 Assert(!HWACCMIsNestedPagingActive(pVM));
4367
4368 /*
4369 * Update the shadow root page as well since that's not fixed.
4370 */
4371 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4372 PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4373 uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4374 uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4375 PPGMPOOLPAGE pNewShwPageCR3;
4376
4377 Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4378 rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3);
4379 AssertFatalRC(rc);
4380 rc = VINF_SUCCESS;
4381
4382 /* Mark the page as locked; disallow flushing. */
4383 pgmPoolLockPage(pPool, pNewShwPageCR3);
4384
4385# ifdef IN_RC
4386 /* NOTE: We can't deal with jumps to ring 3 here as we're now in an inconsistent state! */
4387 bool fLog = VMMGCLogDisable(pVM);
4388 pgmLock(pVM);
4389# endif
4390
4391 pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4392 pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4393 pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4394# ifdef IN_RING0
4395 pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4396 pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4397# elif defined(IN_RC)
4398 pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4399 pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4400# else
4401 pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4402 pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4403# endif
4404
4405# ifndef PGM_WITHOUT_MAPPINGS
4406 /*
4407 * Apply all hypervisor mappings to the new CR3.
4408 * Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4409 * make sure we check for conflicts in the new CR3 root.
4410 */
4411# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4412 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) || VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4413# endif
4414 rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4415 AssertRCReturn(rc, rc);
4416# endif
4417
4418 /* Set the current hypervisor CR3. */
4419 CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4420 SELMShadowCR3Changed(pVM, pVCpu);
4421
4422# ifdef IN_RC
4423 pgmUnlock(pVM);
4424 VMMGCLogRestore(pVM, fLog);
4425# endif
4426
4427 /* Clean up the old CR3 root. */
4428 if (pOldShwPageCR3)
4429 {
4430 Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4431# ifndef PGM_WITHOUT_MAPPINGS
4432 /* Remove the hypervisor mappings from the shadow page table. */
4433 pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4434# endif
4435 /* Mark the page as unlocked; allow flushing again. */
4436 pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4437
4438 pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4439 }
4440
4441# endif
4442
4443 return rc;
4444}
4445
4446/**
4447 * Unmaps the shadow CR3.
4448 *
4449 * @returns VBox status, no specials.
4450 * @param pVCpu The VMCPU handle.
4451 */
4452PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4453{
4454 LogFlow(("UnmapCR3\n"));
4455
4456 int rc = VINF_SUCCESS;
4457 PVM pVM = pVCpu->CTX_SUFF(pVM);
4458
4459 /*
4460 * Update guest paging info.
4461 */
4462#if PGM_GST_TYPE == PGM_TYPE_32BIT
4463 pVCpu->pgm.s.pGst32BitPdR3 = 0;
4464# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4465 pVCpu->pgm.s.pGst32BitPdR0 = 0;
4466# endif
4467 pVCpu->pgm.s.pGst32BitPdRC = 0;
4468
4469#elif PGM_GST_TYPE == PGM_TYPE_PAE
4470 pVCpu->pgm.s.pGstPaePdptR3 = 0;
4471# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4472 pVCpu->pgm.s.pGstPaePdptR0 = 0;
4473# endif
4474 pVCpu->pgm.s.pGstPaePdptRC = 0;
4475 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4476 {
4477 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4478# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4479 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4480# endif
4481 pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4482 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4483 }
4484
4485#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4486 pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4487# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4488 pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4489# endif
4490
4491#else /* prot/real mode stub */
4492 /* nothing to do */
4493#endif
4494
4495#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4496 /*
4497 * Update shadow paging info.
4498 */
4499# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4500 || PGM_SHW_TYPE == PGM_TYPE_PAE \
4501 || PGM_SHW_TYPE == PGM_TYPE_AMD64))
4502
4503# if PGM_GST_TYPE != PGM_TYPE_REAL
4504 Assert(!HWACCMIsNestedPagingActive(pVM));
4505# endif
4506
4507# ifndef PGM_WITHOUT_MAPPINGS
4508 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4509 /* Remove the hypervisor mappings from the shadow page table. */
4510 pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4511# endif
4512
4513 if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4514 {
4515 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4516
4517 Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4518
4519 /* Mark the page as unlocked; allow flushing again. */
4520 pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4521
4522 pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4523 pVCpu->pgm.s.pShwPageCR3R3 = 0;
4524 pVCpu->pgm.s.pShwPageCR3R0 = 0;
4525 pVCpu->pgm.s.pShwPageCR3RC = 0;
4526 pVCpu->pgm.s.iShwUser = 0;
4527 pVCpu->pgm.s.iShwUserTable = 0;
4528 }
4529# endif
4530#endif /* !IN_RC*/
4531
4532 return rc;
4533}
4534
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette