VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 22529

Last change on this file since 22529 was 22529, checked in by vboxsync, 15 years ago

Updated assertion

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 179.2 KB
Line 
1/* $Id: PGMAllPool.cpp 22529 2009-08-27 14:06:06Z vboxsync $ */
2/** @file
3 * PGM Shadow Page Pool.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_PGM_POOL
27#include <VBox/pgm.h>
28#include <VBox/mm.h>
29#include <VBox/em.h>
30#include <VBox/cpum.h>
31#ifdef IN_RC
32# include <VBox/patm.h>
33#endif
34#include "PGMInternal.h"
35#include <VBox/vm.h>
36#include <VBox/disopcode.h>
37#include <VBox/hwacc_vmx.h>
38
39#include <VBox/log.h>
40#include <VBox/err.h>
41#include <iprt/asm.h>
42#include <iprt/string.h>
43
44
45/*******************************************************************************
46* Internal Functions *
47*******************************************************************************/
48RT_C_DECLS_BEGIN
49static void pgmPoolFlushAllInt(PPGMPOOL pPool);
50#ifdef PGMPOOL_WITH_USER_TRACKING
51DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
52DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
53static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
54#endif
55#ifdef PGMPOOL_WITH_CACHE
56static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
57#endif
58#ifdef PGMPOOL_WITH_MONITORING
59static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
60#endif
61#ifndef IN_RING3
62DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
63#endif
64#ifdef LOG_ENABLED
65static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
66#endif
67
68void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs);
69void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt);
70int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage);
71PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
72void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
73void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
74
75RT_C_DECLS_END
76
77
78/**
79 * Checks if the specified page pool kind is for a 4MB or 2MB guest page.
80 *
81 * @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
82 * @param enmKind The page kind.
83 */
84DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
85{
86 switch (enmKind)
87 {
88 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
89 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
90 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
91 return true;
92 default:
93 return false;
94 }
95}
96
97/** @def PGMPOOL_PAGE_2_LOCKED_PTR
98 * Maps a pool page pool into the current context and lock it (RC only).
99 *
100 * @returns VBox status code.
101 * @param pVM The VM handle.
102 * @param pPage The pool page.
103 *
104 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
105 * small page window employeed by that function. Be careful.
106 * @remark There is no need to assert on the result.
107 */
108#if defined(IN_RC)
109DECLINLINE(void *) PGMPOOL_PAGE_2_LOCKED_PTR(PVM pVM, PPGMPOOLPAGE pPage)
110{
111 void *pv = pgmPoolMapPageInlined(&pVM->pgm.s, pPage);
112
113 /* Make sure the dynamic mapping will not be reused. */
114 if (pv)
115 PGMDynLockHCPage(pVM, (uint8_t *)pv);
116
117 return pv;
118}
119#else
120# define PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage) PGMPOOL_PAGE_2_PTR(pVM, pPage)
121#endif
122
123/** @def PGMPOOL_UNLOCK_PTR
124 * Unlock a previously locked dynamic caching (RC only).
125 *
126 * @returns VBox status code.
127 * @param pVM The VM handle.
128 * @param pPage The pool page.
129 *
130 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
131 * small page window employeed by that function. Be careful.
132 * @remark There is no need to assert on the result.
133 */
134#if defined(IN_RC)
135DECLINLINE(void) PGMPOOL_UNLOCK_PTR(PVM pVM, void *pvPage)
136{
137 if (pvPage)
138 PGMDynUnlockHCPage(pVM, (uint8_t *)pvPage);
139}
140#else
141# define PGMPOOL_UNLOCK_PTR(pVM, pPage) do {} while (0)
142#endif
143
144
145#ifdef PGMPOOL_WITH_MONITORING
146/**
147 * Determin the size of a write instruction.
148 * @returns number of bytes written.
149 * @param pDis The disassembler state.
150 */
151static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
152{
153 /*
154 * This is very crude and possibly wrong for some opcodes,
155 * but since it's not really supposed to be called we can
156 * probably live with that.
157 */
158 return DISGetParamSize(pDis, &pDis->param1);
159}
160
161
162/**
163 * Flushes a chain of pages sharing the same access monitor.
164 *
165 * @returns VBox status code suitable for scheduling.
166 * @param pPool The pool.
167 * @param pPage A page in the chain.
168 */
169int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
170{
171 LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
172
173 /*
174 * Find the list head.
175 */
176 uint16_t idx = pPage->idx;
177 if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
178 {
179 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
180 {
181 idx = pPage->iMonitoredPrev;
182 Assert(idx != pPage->idx);
183 pPage = &pPool->aPages[idx];
184 }
185 }
186
187 /*
188 * Iterate the list flushing each shadow page.
189 */
190 int rc = VINF_SUCCESS;
191 for (;;)
192 {
193 idx = pPage->iMonitoredNext;
194 Assert(idx != pPage->idx);
195 if (pPage->idx >= PGMPOOL_IDX_FIRST)
196 {
197 int rc2 = pgmPoolFlushPage(pPool, pPage);
198 AssertRC(rc2);
199 }
200 /* next */
201 if (idx == NIL_PGMPOOL_IDX)
202 break;
203 pPage = &pPool->aPages[idx];
204 }
205 return rc;
206}
207
208
209/**
210 * Wrapper for getting the current context pointer to the entry being modified.
211 *
212 * @returns VBox status code suitable for scheduling.
213 * @param pVM VM Handle.
214 * @param pvDst Destination address
215 * @param pvSrc Source guest virtual address.
216 * @param GCPhysSrc The source guest physical address.
217 * @param cb Size of data to read
218 */
219DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVM pVM, void *pvDst, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
220{
221#if defined(IN_RING3)
222 memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
223 return VINF_SUCCESS;
224#else
225 /* @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
226 return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
227#endif
228}
229
230/**
231 * Process shadow entries before they are changed by the guest.
232 *
233 * For PT entries we will clear them. For PD entries, we'll simply check
234 * for mapping conflicts and set the SyncCR3 FF if found.
235 *
236 * @param pVCpu VMCPU handle
237 * @param pPool The pool.
238 * @param pPage The head page.
239 * @param GCPhysFault The guest physical fault address.
240 * @param uAddress In R0 and GC this is the guest context fault address (flat).
241 * In R3 this is the host context 'fault' address.
242 * @param pDis The disassembler state for figuring out the write size.
243 * This need not be specified if the caller knows we won't do cross entry accesses.
244 */
245void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pDis)
246{
247 AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%#x (idx=%#x)\n", pPage->iMonitoredPrev, pPage->idx));
248 const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
249 const unsigned cbWrite = pDis ? pgmPoolDisasWriteSize(pDis) : 0;
250 PVM pVM = pPool->CTX_SUFF(pVM);
251
252 LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n", (RTGCPTR)pvAddress, GCPhysFault, cbWrite));
253
254 for (;;)
255 {
256 union
257 {
258 void *pv;
259 PX86PT pPT;
260 PX86PTPAE pPTPae;
261 PX86PD pPD;
262 PX86PDPAE pPDPae;
263 PX86PDPT pPDPT;
264 PX86PML4 pPML4;
265 } uShw;
266
267 LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s\n", pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
268
269 uShw.pv = NULL;
270 switch (pPage->enmKind)
271 {
272 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
273 {
274 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
275 const unsigned iShw = off / sizeof(X86PTE);
276 LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
277 if (uShw.pPT->a[iShw].n.u1Present)
278 {
279# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
280 X86PTE GstPte;
281
282 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
283 AssertRC(rc);
284 Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
285 pgmPoolTracDerefGCPhysHint(pPool, pPage,
286 uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
287 GstPte.u & X86_PTE_PG_MASK);
288# endif
289 ASMAtomicWriteSize(&uShw.pPT->a[iShw], 0);
290 }
291 break;
292 }
293
294 /* page/2 sized */
295 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
296 {
297 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
298 if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
299 {
300 const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
301 LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
302 if (uShw.pPTPae->a[iShw].n.u1Present)
303 {
304# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
305 X86PTE GstPte;
306 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
307 AssertRC(rc);
308
309 Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
310 pgmPoolTracDerefGCPhysHint(pPool, pPage,
311 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
312 GstPte.u & X86_PTE_PG_MASK);
313# endif
314 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw], 0);
315 }
316 }
317 break;
318 }
319
320 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
321 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
322 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
323 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
324 {
325 unsigned iGst = off / sizeof(X86PDE);
326 unsigned iShwPdpt = iGst / 256;
327 unsigned iShw = (iGst % 256) * 2;
328 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
329
330 LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
331 if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
332 {
333 for (unsigned i = 0; i < 2; i++)
334 {
335# ifndef IN_RING0
336 if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
337 {
338 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
339 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
340 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
341 break;
342 }
343 else
344# endif /* !IN_RING0 */
345 if (uShw.pPDPae->a[iShw+i].n.u1Present)
346 {
347 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
348 pgmPoolFree(pVM,
349 uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
350 pPage->idx,
351 iShw + i);
352 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw+i], 0);
353 }
354
355 /* paranoia / a bit assumptive. */
356 if ( pDis
357 && (off & 3)
358 && (off & 3) + cbWrite > 4)
359 {
360 const unsigned iShw2 = iShw + 2 + i;
361 if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
362 {
363# ifndef IN_RING0
364 if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
365 {
366 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
367 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
368 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
369 break;
370 }
371 else
372# endif /* !IN_RING0 */
373 if (uShw.pPDPae->a[iShw2].n.u1Present)
374 {
375 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
376 pgmPoolFree(pVM,
377 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
378 pPage->idx,
379 iShw2);
380 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
381 }
382 }
383 }
384 }
385 }
386 break;
387 }
388
389 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
390 {
391 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
392 const unsigned iShw = off / sizeof(X86PTEPAE);
393 if (uShw.pPTPae->a[iShw].n.u1Present)
394 {
395# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
396 X86PTEPAE GstPte;
397 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
398 AssertRC(rc);
399
400 Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
401 pgmPoolTracDerefGCPhysHint(pPool, pPage,
402 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
403 GstPte.u & X86_PTE_PAE_PG_MASK);
404# endif
405 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw].u, 0);
406 }
407
408 /* paranoia / a bit assumptive. */
409 if ( pDis
410 && (off & 7)
411 && (off & 7) + cbWrite > sizeof(X86PTEPAE))
412 {
413 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
414 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
415
416 if (uShw.pPTPae->a[iShw2].n.u1Present)
417 {
418# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
419 X86PTEPAE GstPte;
420# ifdef IN_RING3
421 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, (RTHCPTR)((RTHCUINTPTR)pvAddress + sizeof(GstPte)), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
422# else
423 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress + sizeof(GstPte), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
424# endif
425 AssertRC(rc);
426 Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
427 pgmPoolTracDerefGCPhysHint(pPool, pPage,
428 uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK,
429 GstPte.u & X86_PTE_PAE_PG_MASK);
430# endif
431 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw2].u ,0);
432 }
433 }
434 break;
435 }
436
437 case PGMPOOLKIND_32BIT_PD:
438 {
439 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
440 const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
441
442 LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
443# ifndef IN_RING0
444 if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
445 {
446 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
447 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
448 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
449 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
450 break;
451 }
452# endif /* !IN_RING0 */
453# ifndef IN_RING0
454 else
455# endif /* !IN_RING0 */
456 {
457 if (uShw.pPD->a[iShw].n.u1Present)
458 {
459 LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
460 pgmPoolFree(pVM,
461 uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
462 pPage->idx,
463 iShw);
464 ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
465 }
466 }
467 /* paranoia / a bit assumptive. */
468 if ( pDis
469 && (off & 3)
470 && (off & 3) + cbWrite > sizeof(X86PTE))
471 {
472 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
473 if ( iShw2 != iShw
474 && iShw2 < RT_ELEMENTS(uShw.pPD->a))
475 {
476# ifndef IN_RING0
477 if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
478 {
479 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
480 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
481 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
482 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
483 break;
484 }
485# endif /* !IN_RING0 */
486# ifndef IN_RING0
487 else
488# endif /* !IN_RING0 */
489 {
490 if (uShw.pPD->a[iShw2].n.u1Present)
491 {
492 LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
493 pgmPoolFree(pVM,
494 uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
495 pPage->idx,
496 iShw2);
497 ASMAtomicWriteSize(&uShw.pPD->a[iShw2].u, 0);
498 }
499 }
500 }
501 }
502#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
503 if ( uShw.pPD->a[iShw].n.u1Present
504 && !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
505 {
506 LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
507# ifdef IN_RC /* TLB load - we're pushing things a bit... */
508 ASMProbeReadByte(pvAddress);
509# endif
510 pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
511 ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
512 }
513#endif
514 break;
515 }
516
517 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
518 {
519 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
520 const unsigned iShw = off / sizeof(X86PDEPAE);
521#ifndef IN_RING0
522 if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
523 {
524 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
525 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
526 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
527 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
528 break;
529 }
530#endif /* !IN_RING0 */
531 /*
532 * Causes trouble when the guest uses a PDE to refer to the whole page table level
533 * structure. (Invalidate here; faults later on when it tries to change the page
534 * table entries -> recheck; probably only applies to the RC case.)
535 */
536# ifndef IN_RING0
537 else
538# endif /* !IN_RING0 */
539 {
540 if (uShw.pPDPae->a[iShw].n.u1Present)
541 {
542 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
543 pgmPoolFree(pVM,
544 uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
545 pPage->idx,
546 iShw);
547 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
548 }
549 }
550 /* paranoia / a bit assumptive. */
551 if ( pDis
552 && (off & 7)
553 && (off & 7) + cbWrite > sizeof(X86PDEPAE))
554 {
555 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
556 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
557
558#ifndef IN_RING0
559 if ( iShw2 != iShw
560 && uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
561 {
562 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
563 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
564 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
565 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
566 break;
567 }
568#endif /* !IN_RING0 */
569# ifndef IN_RING0
570 else
571# endif /* !IN_RING0 */
572 if (uShw.pPDPae->a[iShw2].n.u1Present)
573 {
574 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
575 pgmPoolFree(pVM,
576 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
577 pPage->idx,
578 iShw2);
579 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
580 }
581 }
582 break;
583 }
584
585 case PGMPOOLKIND_PAE_PDPT:
586 {
587 /*
588 * Hopefully this doesn't happen very often:
589 * - touching unused parts of the page
590 * - messing with the bits of pd pointers without changing the physical address
591 */
592 /* PDPT roots are not page aligned; 32 byte only! */
593 const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
594
595 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
596 const unsigned iShw = offPdpt / sizeof(X86PDPE);
597 if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
598 {
599# ifndef IN_RING0
600 if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
601 {
602 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
603 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
604 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
605 LogFlow(("pgmPoolMonitorChainChanging: Detected pdpt conflict at iShw=%#x!\n", iShw));
606 break;
607 }
608# endif /* !IN_RING0 */
609# ifndef IN_RING0
610 else
611# endif /* !IN_RING0 */
612 if (uShw.pPDPT->a[iShw].n.u1Present)
613 {
614 LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
615 pgmPoolFree(pVM,
616 uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
617 pPage->idx,
618 iShw);
619 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
620 }
621
622 /* paranoia / a bit assumptive. */
623 if ( pDis
624 && (offPdpt & 7)
625 && (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
626 {
627 const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
628 if ( iShw2 != iShw
629 && iShw2 < X86_PG_PAE_PDPE_ENTRIES)
630 {
631# ifndef IN_RING0
632 if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
633 {
634 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
635 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
636 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
637 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
638 break;
639 }
640# endif /* !IN_RING0 */
641# ifndef IN_RING0
642 else
643# endif /* !IN_RING0 */
644 if (uShw.pPDPT->a[iShw2].n.u1Present)
645 {
646 LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
647 pgmPoolFree(pVM,
648 uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
649 pPage->idx,
650 iShw2);
651 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
652 }
653 }
654 }
655 }
656 break;
657 }
658
659#ifndef IN_RC
660 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
661 {
662 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
663 const unsigned iShw = off / sizeof(X86PDEPAE);
664 Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
665 if (uShw.pPDPae->a[iShw].n.u1Present)
666 {
667 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
668 pgmPoolFree(pVM,
669 uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
670 pPage->idx,
671 iShw);
672 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
673 }
674 /* paranoia / a bit assumptive. */
675 if ( pDis
676 && (off & 7)
677 && (off & 7) + cbWrite > sizeof(X86PDEPAE))
678 {
679 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
680 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
681
682 Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
683 if (uShw.pPDPae->a[iShw2].n.u1Present)
684 {
685 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
686 pgmPoolFree(pVM,
687 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
688 pPage->idx,
689 iShw2);
690 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
691 }
692 }
693 break;
694 }
695
696 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
697 {
698 /*
699 * Hopefully this doesn't happen very often:
700 * - messing with the bits of pd pointers without changing the physical address
701 */
702 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
703 {
704 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
705 const unsigned iShw = off / sizeof(X86PDPE);
706 if (uShw.pPDPT->a[iShw].n.u1Present)
707 {
708 LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
709 pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
710 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
711 }
712 /* paranoia / a bit assumptive. */
713 if ( pDis
714 && (off & 7)
715 && (off & 7) + cbWrite > sizeof(X86PDPE))
716 {
717 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
718 if (uShw.pPDPT->a[iShw2].n.u1Present)
719 {
720 LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
721 pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
722 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
723 }
724 }
725 }
726 break;
727 }
728
729 case PGMPOOLKIND_64BIT_PML4:
730 {
731 /*
732 * Hopefully this doesn't happen very often:
733 * - messing with the bits of pd pointers without changing the physical address
734 */
735 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
736 {
737 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
738 const unsigned iShw = off / sizeof(X86PDPE);
739 if (uShw.pPML4->a[iShw].n.u1Present)
740 {
741 LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
742 pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
743 ASMAtomicWriteSize(&uShw.pPML4->a[iShw].u, 0);
744 }
745 /* paranoia / a bit assumptive. */
746 if ( pDis
747 && (off & 7)
748 && (off & 7) + cbWrite > sizeof(X86PDPE))
749 {
750 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
751 if (uShw.pPML4->a[iShw2].n.u1Present)
752 {
753 LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
754 pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
755 ASMAtomicWriteSize(&uShw.pPML4->a[iShw2].u, 0);
756 }
757 }
758 }
759 break;
760 }
761#endif /* IN_RING0 */
762
763 default:
764 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
765 }
766 PGMPOOL_UNLOCK_PTR(pVM, uShw.pv);
767
768 /* next */
769 if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
770 return;
771 pPage = &pPool->aPages[pPage->iMonitoredNext];
772 }
773}
774
775# ifndef IN_RING3
776/**
777 * Checks if a access could be a fork operation in progress.
778 *
779 * Meaning, that the guest is setting up the parent process for Copy-On-Write.
780 *
781 * @returns true if it's likly that we're forking, otherwise false.
782 * @param pPool The pool.
783 * @param pDis The disassembled instruction.
784 * @param offFault The access offset.
785 */
786DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
787{
788 /*
789 * i386 linux is using btr to clear X86_PTE_RW.
790 * The functions involved are (2.6.16 source inspection):
791 * clear_bit
792 * ptep_set_wrprotect
793 * copy_one_pte
794 * copy_pte_range
795 * copy_pmd_range
796 * copy_pud_range
797 * copy_page_range
798 * dup_mmap
799 * dup_mm
800 * copy_mm
801 * copy_process
802 * do_fork
803 */
804 if ( pDis->pCurInstr->opcode == OP_BTR
805 && !(offFault & 4)
806 /** @todo Validate that the bit index is X86_PTE_RW. */
807 )
808 {
809 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,Fork));
810 return true;
811 }
812 return false;
813}
814
815
816/**
817 * Determine whether the page is likely to have been reused.
818 *
819 * @returns true if we consider the page as being reused for a different purpose.
820 * @returns false if we consider it to still be a paging page.
821 * @param pVM VM Handle.
822 * @param pVCpu VMCPU Handle.
823 * @param pRegFrame Trap register frame.
824 * @param pDis The disassembly info for the faulting instruction.
825 * @param pvFault The fault address.
826 *
827 * @remark The REP prefix check is left to the caller because of STOSD/W.
828 */
829DECLINLINE(bool) pgmPoolMonitorIsReused(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault)
830{
831#ifndef IN_RC
832 /** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
833 if ( HWACCMHasPendingIrq(pVM)
834 && (pRegFrame->rsp - pvFault) < 32)
835 {
836 /* Fault caused by stack writes while trying to inject an interrupt event. */
837 Log(("pgmPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
838 return true;
839 }
840#else
841 NOREF(pVM); NOREF(pvFault);
842#endif
843
844 LogFlow(("Reused instr %RGv %d at %RGv param1.flags=%x param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->opcode, pvFault, pDis->param1.flags, pDis->param1.base.reg_gen));
845
846 /* Non-supervisor mode write means it's used for something else. */
847 if (CPUMGetGuestCPL(pVCpu, pRegFrame) != 0)
848 return true;
849
850 switch (pDis->pCurInstr->opcode)
851 {
852 /* call implies the actual push of the return address faulted */
853 case OP_CALL:
854 Log4(("pgmPoolMonitorIsReused: CALL\n"));
855 return true;
856 case OP_PUSH:
857 Log4(("pgmPoolMonitorIsReused: PUSH\n"));
858 return true;
859 case OP_PUSHF:
860 Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
861 return true;
862 case OP_PUSHA:
863 Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
864 return true;
865 case OP_FXSAVE:
866 Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
867 return true;
868 case OP_MOVNTI: /* solaris - block_zero_no_xmm */
869 Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
870 return true;
871 case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
872 Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
873 return true;
874 case OP_MOVSWD:
875 case OP_STOSWD:
876 if ( pDis->prefix == (PREFIX_REP|PREFIX_REX)
877 && pRegFrame->rcx >= 0x40
878 )
879 {
880 Assert(pDis->mode == CPUMODE_64BIT);
881
882 Log(("pgmPoolMonitorIsReused: OP_STOSQ\n"));
883 return true;
884 }
885 return false;
886 }
887 if ( ( (pDis->param1.flags & USE_REG_GEN32)
888 || (pDis->param1.flags & USE_REG_GEN64))
889 && (pDis->param1.base.reg_gen == USE_REG_ESP))
890 {
891 Log4(("pgmPoolMonitorIsReused: ESP\n"));
892 return true;
893 }
894
895 return false;
896}
897
898
899/**
900 * Flushes the page being accessed.
901 *
902 * @returns VBox status code suitable for scheduling.
903 * @param pVM The VM handle.
904 * @param pVCpu The VMCPU handle.
905 * @param pPool The pool.
906 * @param pPage The pool page (head).
907 * @param pDis The disassembly of the write instruction.
908 * @param pRegFrame The trap register frame.
909 * @param GCPhysFault The fault address as guest physical address.
910 * @param pvFault The fault address.
911 */
912static int pgmPoolAccessHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
913 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
914{
915 /*
916 * First, do the flushing.
917 */
918 int rc = pgmPoolMonitorChainFlush(pPool, pPage);
919
920 /*
921 * Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
922 * @todo: why is this necessary? an instruction restart would be sufficient, wouldn't it?
923 */
924 uint32_t cbWritten;
925 int rc2 = EMInterpretInstructionCPU(pVM, pVCpu, pDis, pRegFrame, pvFault, &cbWritten);
926 if (RT_SUCCESS(rc2))
927 pRegFrame->rip += pDis->opsize;
928 else if (rc2 == VERR_EM_INTERPRETER)
929 {
930#ifdef IN_RC
931 if (PATMIsPatchGCAddr(pVM, (RTRCPTR)pRegFrame->eip))
932 {
933 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
934 pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
935 rc = VINF_SUCCESS;
936 STAM_COUNTER_INC(&pPool->StatMonitorRZIntrFailPatch2);
937 }
938 else
939#endif
940 {
941 rc = VINF_EM_RAW_EMULATE_INSTR;
942 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
943 }
944 }
945 else
946 rc = rc2;
947
948 /* See use in pgmPoolAccessHandlerSimple(). */
949 PGM_INVL_VCPU_TLBS(pVCpu);
950
951 LogFlow(("pgmPoolAccessHandlerPT: returns %Rrc (flushed)\n", rc));
952 return rc;
953
954}
955
956
957/**
958 * Handles the STOSD write accesses.
959 *
960 * @returns VBox status code suitable for scheduling.
961 * @param pVM The VM handle.
962 * @param pPool The pool.
963 * @param pPage The pool page (head).
964 * @param pDis The disassembly of the write instruction.
965 * @param pRegFrame The trap register frame.
966 * @param GCPhysFault The fault address as guest physical address.
967 * @param pvFault The fault address.
968 */
969DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
970 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
971{
972 unsigned uIncrement = pDis->param1.size;
973
974 Assert(pDis->mode == CPUMODE_32BIT || pDis->mode == CPUMODE_64BIT);
975 Assert(pRegFrame->rcx <= 0x20);
976
977#ifdef VBOX_STRICT
978 if (pDis->opmode == CPUMODE_32BIT)
979 Assert(uIncrement == 4);
980 else
981 Assert(uIncrement == 8);
982#endif
983
984 Log3(("pgmPoolAccessHandlerSTOSD\n"));
985
986 /*
987 * Increment the modification counter and insert it into the list
988 * of modified pages the first time.
989 */
990 if (!pPage->cModifications++)
991 pgmPoolMonitorModifiedInsert(pPool, pPage);
992
993 /*
994 * Execute REP STOSD.
995 *
996 * This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
997 * write situation, meaning that it's safe to write here.
998 */
999 PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
1000 RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
1001 while (pRegFrame->rcx)
1002 {
1003#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1004 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1005 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1006 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1007#else
1008 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1009#endif
1010#ifdef IN_RC
1011 *(uint32_t *)pu32 = pRegFrame->eax;
1012#else
1013 PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
1014#endif
1015 pu32 += uIncrement;
1016 GCPhysFault += uIncrement;
1017 pRegFrame->rdi += uIncrement;
1018 pRegFrame->rcx--;
1019 }
1020 pRegFrame->rip += pDis->opsize;
1021
1022#ifdef IN_RC
1023 /* See use in pgmPoolAccessHandlerSimple(). */
1024 PGM_INVL_VCPU_TLBS(pVCpu);
1025#endif
1026
1027 LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
1028 return VINF_SUCCESS;
1029}
1030
1031
1032/**
1033 * Handles the simple write accesses.
1034 *
1035 * @returns VBox status code suitable for scheduling.
1036 * @param pVM The VM handle.
1037 * @param pVCpu The VMCPU handle.
1038 * @param pPool The pool.
1039 * @param pPage The pool page (head).
1040 * @param pDis The disassembly of the write instruction.
1041 * @param pRegFrame The trap register frame.
1042 * @param GCPhysFault The fault address as guest physical address.
1043 * @param pvFault The fault address.
1044 */
1045DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
1046 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
1047{
1048 Log3(("pgmPoolAccessHandlerSimple\n"));
1049 /*
1050 * Increment the modification counter and insert it into the list
1051 * of modified pages the first time.
1052 */
1053 if (!pPage->cModifications++)
1054 pgmPoolMonitorModifiedInsert(pPool, pPage);
1055
1056 /*
1057 * Clear all the pages. ASSUMES that pvFault is readable.
1058 */
1059#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1060 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1061 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pDis);
1062 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1063#else
1064 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pDis);
1065#endif
1066
1067 /*
1068 * Interpret the instruction.
1069 */
1070 uint32_t cb;
1071 int rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, pRegFrame, pvFault, &cb);
1072 if (RT_SUCCESS(rc))
1073 pRegFrame->rip += pDis->opsize;
1074 else if (rc == VERR_EM_INTERPRETER)
1075 {
1076 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for %04x:%RGv - opcode=%d\n",
1077 pRegFrame->cs, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->opcode));
1078 rc = VINF_EM_RAW_EMULATE_INSTR;
1079 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
1080 }
1081
1082#ifdef IN_RC
1083 /*
1084 * Quick hack, with logging enabled we're getting stale
1085 * code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
1086 * Flushing here is BAD and expensive, I think EMInterpretDisasOne will
1087 * have to be fixed to support this. But that'll have to wait till next week.
1088 *
1089 * An alternative is to keep track of the changed PTEs together with the
1090 * GCPhys from the guest PT. This may proove expensive though.
1091 *
1092 * At the moment, it's VITAL that it's done AFTER the instruction interpreting
1093 * because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
1094 */
1095 PGM_INVL_VCPU_TLBS(pVCpu);
1096#endif
1097
1098 LogFlow(("pgmPoolAccessHandlerSimple: returns %Rrc cb=%d\n", rc, cb));
1099 return rc;
1100}
1101
1102/**
1103 * \#PF Handler callback for PT write accesses.
1104 *
1105 * @returns VBox status code (appropriate for GC return).
1106 * @param pVM VM Handle.
1107 * @param uErrorCode CPU Error code.
1108 * @param pRegFrame Trap register frame.
1109 * NULL on DMA and other non CPU access.
1110 * @param pvFault The fault address (cr2).
1111 * @param GCPhysFault The GC physical address corresponding to pvFault.
1112 * @param pvUser User argument.
1113 */
1114DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1115{
1116 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1117 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1118 PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1119 PVMCPU pVCpu = VMMGetCpu(pVM);
1120 unsigned cMaxModifications;
1121 bool fForcedFlush = false;
1122
1123 LogFlow(("pgmPoolAccessHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1124
1125 pgmLock(pVM);
1126 if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1127 {
1128 /* Pool page changed while we were waiting for the lock; ignore. */
1129 Log(("CPU%d: pgmPoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1130 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1131 pgmUnlock(pVM);
1132 return VINF_SUCCESS;
1133 }
1134
1135 /*
1136 * Disassemble the faulting instruction.
1137 */
1138 PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
1139 int rc = EMInterpretDisasOne(pVM, pVCpu, pRegFrame, pDis, NULL);
1140 AssertReturnStmt(rc == VINF_SUCCESS, pgmUnlock(pVM), rc);
1141
1142 Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1143
1144 /*
1145 * We should ALWAYS have the list head as user parameter. This
1146 * is because we use that page to record the changes.
1147 */
1148 Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1149#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1150 Assert(!pPage->fDirty);
1151#endif
1152
1153 /* Maximum nr of modifications depends on the guest mode. */
1154 if (pDis->mode == CPUMODE_32BIT)
1155 cMaxModifications = 48;
1156 else
1157 cMaxModifications = 24;
1158
1159 /*
1160 * Incremental page table updates should weight more than random ones.
1161 * (Only applies when started from offset 0)
1162 */
1163 pVCpu->pgm.s.cPoolAccessHandler++;
1164 if ( pPage->pvLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1165 && pPage->pvLastAccessHandlerRip < pRegFrame->rip + 0x40
1166 && pvFault == (pPage->pvLastAccessHandlerFault + pDis->param1.size)
1167 && pVCpu->pgm.s.cPoolAccessHandler == (pPage->cLastAccessHandlerCount + 1))
1168 {
1169 Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1170 pPage->cModifications = pPage->cModifications * 2;
1171 pPage->pvLastAccessHandlerFault = pvFault;
1172 pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1173 if (pPage->cModifications > cMaxModifications)
1174 {
1175 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FlushReinit));
1176 fForcedFlush = true;
1177 }
1178 }
1179
1180 if (pPage->cModifications >= cMaxModifications)
1181 Log(("Mod overflow %VGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1182
1183 /*
1184 * Check if it's worth dealing with.
1185 */
1186 bool fReused = false;
1187 if ( ( pPage->cModifications < cMaxModifications /** @todo #define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */
1188 || pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
1189 )
1190 && !(fReused = pgmPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault))
1191 && !pgmPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1192 {
1193 /*
1194 * Simple instructions, no REP prefix.
1195 */
1196 if (!(pDis->prefix & (PREFIX_REP | PREFIX_REPNE)))
1197 {
1198 rc = pgmPoolAccessHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1199
1200 /* A mov instruction to change the first page table entry will be remembered so we can detect
1201 * full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1202 */
1203 if ( rc == VINF_SUCCESS
1204 && pDis->pCurInstr->opcode == OP_MOV
1205 && (pvFault & PAGE_OFFSET_MASK) == 0)
1206 {
1207 pPage->pvLastAccessHandlerFault = pvFault;
1208 pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1209 pPage->pvLastAccessHandlerRip = pRegFrame->rip;
1210 /* Make sure we don't kick out a page too quickly. */
1211 if (pPage->cModifications > 8)
1212 pPage->cModifications = 2;
1213 }
1214 else
1215 if (pPage->pvLastAccessHandlerFault == pvFault)
1216 {
1217 /* ignore the 2nd write to this page table entry. */
1218 pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1219 }
1220 else
1221 {
1222 pPage->pvLastAccessHandlerFault = 0;
1223 pPage->pvLastAccessHandlerRip = 0;
1224 }
1225
1226 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1227 pgmUnlock(pVM);
1228 return rc;
1229 }
1230
1231 /*
1232 * Windows is frequently doing small memset() operations (netio test 4k+).
1233 * We have to deal with these or we'll kill the cache and performance.
1234 */
1235 if ( pDis->pCurInstr->opcode == OP_STOSWD
1236 && !pRegFrame->eflags.Bits.u1DF
1237 && pDis->opmode == pDis->mode
1238 && pDis->addrmode == pDis->mode)
1239 {
1240 bool fValidStosd = false;
1241
1242 if ( pDis->mode == CPUMODE_32BIT
1243 && pDis->prefix == PREFIX_REP
1244 && pRegFrame->ecx <= 0x20
1245 && pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1246 && !((uintptr_t)pvFault & 3)
1247 && (pRegFrame->eax == 0 || pRegFrame->eax == 0x80) /* the two values observed. */
1248 )
1249 {
1250 fValidStosd = true;
1251 pRegFrame->rcx &= 0xffffffff; /* paranoia */
1252 }
1253 else
1254 if ( pDis->mode == CPUMODE_64BIT
1255 && pDis->prefix == (PREFIX_REP | PREFIX_REX)
1256 && pRegFrame->rcx <= 0x20
1257 && pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1258 && !((uintptr_t)pvFault & 7)
1259 && (pRegFrame->rax == 0 || pRegFrame->rax == 0x80) /* the two values observed. */
1260 )
1261 {
1262 fValidStosd = true;
1263 }
1264
1265 if (fValidStosd)
1266 {
1267 rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1268 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,RepStosd), a);
1269 pgmUnlock(pVM);
1270 return rc;
1271 }
1272 }
1273
1274 /* REP prefix, don't bother. */
1275 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,RepPrefix));
1276 Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1277 pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->opcode, pDis->prefix));
1278 }
1279
1280#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1281 /* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1282 * leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1283 */
1284 if ( !fReused
1285 && !fForcedFlush
1286 && pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1287 && pPage->cModifications >= cMaxModifications)
1288 {
1289 Assert(!pgmPoolIsPageLocked(&pVM->pgm.s, pPage));
1290 Assert(pPage->fDirty == false);
1291
1292 /* Flush any monitored duplicates as we will disable write protection. */
1293 if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1294 || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1295 {
1296 PPGMPOOLPAGE pPageHead = pPage;
1297
1298 /* Find the monitor head. */
1299 while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1300 pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1301
1302 while (pPageHead)
1303 {
1304 unsigned idxNext = pPageHead->iMonitoredNext;
1305
1306 if (pPageHead != pPage)
1307 {
1308 STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1309 Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1310 int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1311 AssertRC(rc2);
1312 }
1313
1314 if (idxNext == NIL_PGMPOOL_IDX)
1315 break;
1316
1317 pPageHead = &pPool->aPages[idxNext];
1318 }
1319 }
1320
1321 /* Temporarily allow write access to the page table again. */
1322 rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys, pPage->GCPhys);
1323 if (rc == VINF_SUCCESS)
1324 {
1325 rc = PGMShwModifyPage(pVCpu, pvFault, 1, X86_PTE_RW, ~(uint64_t)X86_PTE_RW);
1326 AssertMsg(rc == VINF_SUCCESS
1327 /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1328 || rc == VERR_PAGE_TABLE_NOT_PRESENT
1329 || rc == VERR_PAGE_NOT_PRESENT,
1330 ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1331
1332 pgmPoolAddDirtyPage(pVM, pPool, pPage);
1333
1334 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1335 pgmUnlock(pVM);
1336 return rc;
1337 }
1338 }
1339#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1340
1341 /*
1342 * Not worth it, so flush it.
1343 *
1344 * If we considered it to be reused, don't go back to ring-3
1345 * to emulate failed instructions since we usually cannot
1346 * interpret then. This may be a bit risky, in which case
1347 * the reuse detection must be fixed.
1348 */
1349 rc = pgmPoolAccessHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1350 if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
1351 rc = VINF_SUCCESS;
1352 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1353 pgmUnlock(pVM);
1354 return rc;
1355}
1356
1357# endif /* !IN_RING3 */
1358
1359# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1360/**
1361 * Clear references to guest physical memory in a PAE / PAE page table.
1362 *
1363 * @returns nr of changed PTEs
1364 * @param pPool The pool.
1365 * @param pPage The page.
1366 * @param pShwPT The shadow page table (mapping of the page).
1367 * @param pGstPT The guest page table.
1368 * @param pGstPT The old cached guest page table.
1369 */
1370DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT, PCX86PTPAE pOldGstPT)
1371{
1372 unsigned cChanged = 0;
1373
1374 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
1375 {
1376 if (pShwPT->a[i].n.u1Present)
1377 {
1378 /* The the old cached PTE is identical, then there's no need to flush the shadow copy. */
1379 if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1380 {
1381#ifdef VBOX_STRICT
1382 RTHCPHYS HCPhys = -1;
1383 int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1384 AssertMsg(rc == VINF_SUCCESS && (pShwPT->a[i].u & X86_PTE_PAE_PG_MASK) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, pShwPT->a[i].u, HCPhys));
1385#endif
1386 uint64_t uHostAttr = pShwPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
1387 bool fHostRW = !!(pShwPT->a[i].u & X86_PTE_RW);
1388 uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
1389 bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1390
1391 if ( uHostAttr == uGuestAttr
1392 && fHostRW <= fGuestRW)
1393 continue;
1394 }
1395 cChanged++;
1396 /* Something was changed, so flush it. */
1397 Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
1398 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1399 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
1400 ASMAtomicWriteSize(&pShwPT->a[i].u, 0);
1401 }
1402 }
1403 return cChanged;
1404}
1405
1406
1407/**
1408 * Flush a dirty page
1409 *
1410 * @param pVM VM Handle.
1411 * @param pPool The pool.
1412 * @param idxSlot Dirty array slot index
1413 * @param fForceRemoval Force removal from the dirty page list
1414 */
1415static void pgmPoolFlushDirtyPage(PVM pVM, PPGMPOOL pPool, unsigned idxSlot, bool fForceRemoval = false)
1416{
1417 PPGMPOOLPAGE pPage;
1418 unsigned idxPage;
1419
1420 Assert(idxSlot < RT_ELEMENTS(pPool->aIdxDirtyPages));
1421 if (pPool->aIdxDirtyPages[idxSlot] == NIL_PGMPOOL_IDX)
1422 return;
1423
1424 idxPage = pPool->aIdxDirtyPages[idxSlot];
1425 AssertRelease(idxPage != NIL_PGMPOOL_IDX);
1426 pPage = &pPool->aPages[idxPage];
1427 Assert(pPage->idx == idxPage);
1428 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1429
1430 AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1431 Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1432
1433 /* Flush those PTEs that have changed. */
1434 STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1435 void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
1436 void *pvGst;
1437 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1438 unsigned cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst, (PCX86PTPAE)&pPool->aDirtyPages[idxSlot][0]);
1439 STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1440
1441 /** Note: we might want to consider keeping the dirty page active in case there were many changes. */
1442
1443 /* Write protect the page again to catch all write accesses. */
1444 rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys);
1445 Assert(rc == VINF_SUCCESS);
1446 pPage->fDirty = false;
1447 pPage->fZeroed = true;
1448
1449 /* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1450 Assert(pPage->cModifications);
1451 if (cChanges < 4)
1452 pPage->cModifications = 1; /* must use > 0 here */
1453 else
1454 pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1455
1456 STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1457 if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages))
1458 pPool->idxFreeDirtyPage = idxSlot;
1459
1460 pPool->cDirtyPages--;
1461 pPool->aIdxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1462 Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aIdxDirtyPages));
1463 Log(("Removed dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1464}
1465
1466/**
1467 * Add a new dirty page
1468 *
1469 * @param pVM VM Handle.
1470 * @param pPool The pool.
1471 * @param pPage The page.
1472 */
1473void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1474{
1475 unsigned idxFree;
1476
1477 Assert(PGMIsLocked(pVM));
1478 AssertCompile(RT_ELEMENTS(pPool->aIdxDirtyPages) == 8 || RT_ELEMENTS(pPool->aIdxDirtyPages) == 16);
1479
1480 if (pPage->fDirty)
1481 return;
1482
1483 idxFree = pPool->idxFreeDirtyPage;
1484 Assert(idxFree < RT_ELEMENTS(pPool->aIdxDirtyPages));
1485 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1486
1487 if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aIdxDirtyPages))
1488 pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* force removal */);
1489 Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aIdxDirtyPages));
1490 AssertMsg(pPool->aIdxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1491
1492 /* Make a copy of the guest page table as we require valid GCPhys addresses when removing
1493 * references to physical pages. (the HCPhys linear lookup is *extremely* expensive!)
1494 */
1495 void *pvGst;
1496 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1497 memcpy(&pPool->aDirtyPages[idxFree][0], pvGst, PAGE_SIZE);
1498
1499 STAM_COUNTER_INC(&pPool->StatDirtyPage);
1500 Log(("Mark dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1501 pPage->fDirty = true;
1502 pPage->idxDirty = idxFree;
1503 pPool->aIdxDirtyPages[idxFree] = pPage->idx;
1504 pPool->cDirtyPages++;
1505
1506 pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aIdxDirtyPages) - 1);
1507 if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aIdxDirtyPages)
1508 && pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1509 {
1510 unsigned i;
1511 for (i = 1; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1512 {
1513 idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aIdxDirtyPages) - 1);
1514 if (pPool->aIdxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1515 {
1516 pPool->idxFreeDirtyPage = idxFree;
1517 break;
1518 }
1519 }
1520 Assert(i != RT_ELEMENTS(pPool->aIdxDirtyPages));
1521 }
1522
1523 Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages) || pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1524 return;
1525}
1526
1527
1528/**
1529 * Reset all dirty pages by reinstating page monitoring.
1530 *
1531 * @param pVM VM Handle.
1532 * @param fForceRemoval Force removal of all dirty pages
1533 */
1534void pgmPoolResetDirtyPages(PVM pVM, bool fForceRemoval)
1535{
1536 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1537 Assert(PGMIsLocked(pVM));
1538 Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aIdxDirtyPages));
1539
1540 if (!pPool->cDirtyPages)
1541 return;
1542
1543 Log(("pgmPoolResetDirtyPages\n"));
1544 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1545 pgmPoolFlushDirtyPage(pVM, pPool, i, fForceRemoval);
1546
1547 pPool->idxFreeDirtyPage = 0;
1548 if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aIdxDirtyPages)
1549 && pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1550 {
1551 unsigned i;
1552 for (i = 1; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1553 {
1554 if (pPool->aIdxDirtyPages[i] == NIL_PGMPOOL_IDX)
1555 {
1556 pPool->idxFreeDirtyPage = i;
1557 break;
1558 }
1559 }
1560 AssertMsg(i != RT_ELEMENTS(pPool->aIdxDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
1561 }
1562
1563 Assert(pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX || pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages));
1564 return;
1565}
1566# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1567#endif /* PGMPOOL_WITH_MONITORING */
1568
1569#ifdef PGMPOOL_WITH_CACHE
1570
1571/**
1572 * Inserts a page into the GCPhys hash table.
1573 *
1574 * @param pPool The pool.
1575 * @param pPage The page.
1576 */
1577DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1578{
1579 Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1580 Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1581 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1582 pPage->iNext = pPool->aiHash[iHash];
1583 pPool->aiHash[iHash] = pPage->idx;
1584}
1585
1586
1587/**
1588 * Removes a page from the GCPhys hash table.
1589 *
1590 * @param pPool The pool.
1591 * @param pPage The page.
1592 */
1593DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1594{
1595 Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1596 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1597 if (pPool->aiHash[iHash] == pPage->idx)
1598 pPool->aiHash[iHash] = pPage->iNext;
1599 else
1600 {
1601 uint16_t iPrev = pPool->aiHash[iHash];
1602 for (;;)
1603 {
1604 const int16_t i = pPool->aPages[iPrev].iNext;
1605 if (i == pPage->idx)
1606 {
1607 pPool->aPages[iPrev].iNext = pPage->iNext;
1608 break;
1609 }
1610 if (i == NIL_PGMPOOL_IDX)
1611 {
1612 AssertReleaseMsgFailed(("GCPhys=%RGp idx=%#x\n", pPage->GCPhys, pPage->idx));
1613 break;
1614 }
1615 iPrev = i;
1616 }
1617 }
1618 pPage->iNext = NIL_PGMPOOL_IDX;
1619}
1620
1621
1622/**
1623 * Frees up one cache page.
1624 *
1625 * @returns VBox status code.
1626 * @retval VINF_SUCCESS on success.
1627 * @param pPool The pool.
1628 * @param iUser The user index.
1629 */
1630static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1631{
1632#ifndef IN_RC
1633 const PVM pVM = pPool->CTX_SUFF(pVM);
1634#endif
1635 Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1636 STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1637
1638 /*
1639 * Select one page from the tail of the age list.
1640 */
1641 PPGMPOOLPAGE pPage;
1642 for (unsigned iLoop = 0; ; iLoop++)
1643 {
1644 uint16_t iToFree = pPool->iAgeTail;
1645 if (iToFree == iUser)
1646 iToFree = pPool->aPages[iToFree].iAgePrev;
1647/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
1648 if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
1649 {
1650 uint16_t i = pPool->aPages[iToFree].iAgePrev;
1651 for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
1652 {
1653 if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
1654 continue;
1655 iToFree = i;
1656 break;
1657 }
1658 }
1659*/
1660 Assert(iToFree != iUser);
1661 AssertRelease(iToFree != NIL_PGMPOOL_IDX);
1662 pPage = &pPool->aPages[iToFree];
1663
1664 /*
1665 * Reject any attempts at flushing the currently active shadow CR3 mapping.
1666 * Call pgmPoolCacheUsed to move the page to the head of the age list.
1667 */
1668 if (!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage))
1669 break;
1670 LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
1671 pgmPoolCacheUsed(pPool, pPage);
1672 AssertLogRelReturn(iLoop < 8192, VERR_INTERNAL_ERROR);
1673 }
1674
1675 /*
1676 * Found a usable page, flush it and return.
1677 */
1678 int rc = pgmPoolFlushPage(pPool, pPage);
1679 /* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
1680 if (rc == VINF_SUCCESS)
1681 PGM_INVL_ALL_VCPU_TLBS(pVM);
1682 return rc;
1683}
1684
1685
1686/**
1687 * Checks if a kind mismatch is really a page being reused
1688 * or if it's just normal remappings.
1689 *
1690 * @returns true if reused and the cached page (enmKind1) should be flushed
1691 * @returns false if not reused.
1692 * @param enmKind1 The kind of the cached page.
1693 * @param enmKind2 The kind of the requested page.
1694 */
1695static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
1696{
1697 switch (enmKind1)
1698 {
1699 /*
1700 * Never reuse them. There is no remapping in non-paging mode.
1701 */
1702 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1703 case PGMPOOLKIND_32BIT_PD_PHYS:
1704 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1705 case PGMPOOLKIND_PAE_PD_PHYS:
1706 case PGMPOOLKIND_PAE_PDPT_PHYS:
1707 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1708 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1709 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1710 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1711 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1712 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
1713 return false;
1714
1715 /*
1716 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1717 */
1718 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1719 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1720 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1721 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1722 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1723 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1724 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1725 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1726 case PGMPOOLKIND_32BIT_PD:
1727 case PGMPOOLKIND_PAE_PDPT:
1728 switch (enmKind2)
1729 {
1730 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1731 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1732 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1733 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1734 case PGMPOOLKIND_64BIT_PML4:
1735 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1736 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1737 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1738 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1739 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1740 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1741 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1742 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1743 return true;
1744 default:
1745 return false;
1746 }
1747
1748 /*
1749 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1750 */
1751 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1752 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1753 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1754 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1755 case PGMPOOLKIND_64BIT_PML4:
1756 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1757 switch (enmKind2)
1758 {
1759 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1760 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1761 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1762 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1763 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1764 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1765 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1766 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1767 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1768 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1769 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1770 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1771 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1772 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1773 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1774 return true;
1775 default:
1776 return false;
1777 }
1778
1779 /*
1780 * These cannot be flushed, and it's common to reuse the PDs as PTs.
1781 */
1782 case PGMPOOLKIND_ROOT_NESTED:
1783 return false;
1784
1785 default:
1786 AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1787 }
1788}
1789
1790
1791/**
1792 * Attempts to satisfy a pgmPoolAlloc request from the cache.
1793 *
1794 * @returns VBox status code.
1795 * @retval VINF_PGM_CACHED_PAGE on success.
1796 * @retval VERR_FILE_NOT_FOUND if not found.
1797 * @param pPool The pool.
1798 * @param GCPhys The GC physical address of the page we're gonna shadow.
1799 * @param enmKind The kind of mapping.
1800 * @param enmAccess Access type for the mapping (only relevant for big pages)
1801 * @param iUser The shadow page pool index of the user table.
1802 * @param iUserTable The index into the user table (shadowed).
1803 * @param ppPage Where to store the pointer to the page.
1804 */
1805static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
1806{
1807#ifndef IN_RC
1808 const PVM pVM = pPool->CTX_SUFF(pVM);
1809#endif
1810 /*
1811 * Look up the GCPhys in the hash.
1812 */
1813 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1814 Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%x iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
1815 if (i != NIL_PGMPOOL_IDX)
1816 {
1817 do
1818 {
1819 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1820 Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
1821 if (pPage->GCPhys == GCPhys)
1822 {
1823 if ( (PGMPOOLKIND)pPage->enmKind == enmKind
1824 && (PGMPOOLACCESS)pPage->enmAccess == enmAccess)
1825 {
1826 /* Put it at the start of the use list to make sure pgmPoolTrackAddUser
1827 * doesn't flush it in case there are no more free use records.
1828 */
1829 pgmPoolCacheUsed(pPool, pPage);
1830
1831 int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1832 if (RT_SUCCESS(rc))
1833 {
1834 Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
1835 *ppPage = pPage;
1836 if (pPage->cModifications)
1837 pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
1838 STAM_COUNTER_INC(&pPool->StatCacheHits);
1839 return VINF_PGM_CACHED_PAGE;
1840 }
1841 return rc;
1842 }
1843
1844 if ((PGMPOOLKIND)pPage->enmKind != enmKind)
1845 {
1846 /*
1847 * The kind is different. In some cases we should now flush the page
1848 * as it has been reused, but in most cases this is normal remapping
1849 * of PDs as PT or big pages using the GCPhys field in a slightly
1850 * different way than the other kinds.
1851 */
1852 if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1853 {
1854 STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1855 pgmPoolFlushPage(pPool, pPage);
1856 PGM_INVL_VCPU_TLBS(VMMGetCpu(pVM)); /* see PT handler. */
1857 break;
1858 }
1859 }
1860 }
1861
1862 /* next */
1863 i = pPage->iNext;
1864 } while (i != NIL_PGMPOOL_IDX);
1865 }
1866
1867 Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
1868 STAM_COUNTER_INC(&pPool->StatCacheMisses);
1869 return VERR_FILE_NOT_FOUND;
1870}
1871
1872
1873/**
1874 * Inserts a page into the cache.
1875 *
1876 * @param pPool The pool.
1877 * @param pPage The cached page.
1878 * @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1879 */
1880static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1881{
1882 /*
1883 * Insert into the GCPhys hash if the page is fit for that.
1884 */
1885 Assert(!pPage->fCached);
1886 if (fCanBeCached)
1887 {
1888 pPage->fCached = true;
1889 pgmPoolHashInsert(pPool, pPage);
1890 Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1891 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1892 STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1893 }
1894 else
1895 {
1896 Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1897 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1898 STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1899 }
1900
1901 /*
1902 * Insert at the head of the age list.
1903 */
1904 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1905 pPage->iAgeNext = pPool->iAgeHead;
1906 if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1907 pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1908 else
1909 pPool->iAgeTail = pPage->idx;
1910 pPool->iAgeHead = pPage->idx;
1911}
1912
1913
1914/**
1915 * Flushes a cached page.
1916 *
1917 * @param pPool The pool.
1918 * @param pPage The cached page.
1919 */
1920static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1921{
1922 Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
1923
1924 /*
1925 * Remove the page from the hash.
1926 */
1927 if (pPage->fCached)
1928 {
1929 pPage->fCached = false;
1930 pgmPoolHashRemove(pPool, pPage);
1931 }
1932 else
1933 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1934
1935 /*
1936 * Remove it from the age list.
1937 */
1938 if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1939 pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1940 else
1941 pPool->iAgeTail = pPage->iAgePrev;
1942 if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1943 pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1944 else
1945 pPool->iAgeHead = pPage->iAgeNext;
1946 pPage->iAgeNext = NIL_PGMPOOL_IDX;
1947 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1948}
1949
1950#endif /* PGMPOOL_WITH_CACHE */
1951#ifdef PGMPOOL_WITH_MONITORING
1952
1953/**
1954 * Looks for pages sharing the monitor.
1955 *
1956 * @returns Pointer to the head page.
1957 * @returns NULL if not found.
1958 * @param pPool The Pool
1959 * @param pNewPage The page which is going to be monitored.
1960 */
1961static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1962{
1963#ifdef PGMPOOL_WITH_CACHE
1964 /*
1965 * Look up the GCPhys in the hash.
1966 */
1967 RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1968 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1969 if (i == NIL_PGMPOOL_IDX)
1970 return NULL;
1971 do
1972 {
1973 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1974 if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1975 && pPage != pNewPage)
1976 {
1977 switch (pPage->enmKind)
1978 {
1979 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1980 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1981 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1982 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1983 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1984 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1985 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1986 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1987 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1988 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1989 case PGMPOOLKIND_64BIT_PML4:
1990 case PGMPOOLKIND_32BIT_PD:
1991 case PGMPOOLKIND_PAE_PDPT:
1992 {
1993 /* find the head */
1994 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1995 {
1996 Assert(pPage->iMonitoredPrev != pPage->idx);
1997 pPage = &pPool->aPages[pPage->iMonitoredPrev];
1998 }
1999 return pPage;
2000 }
2001
2002 /* ignore, no monitoring. */
2003 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2004 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2005 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2006 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2007 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2008 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2009 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2010 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2011 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2012 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2013 case PGMPOOLKIND_ROOT_NESTED:
2014 case PGMPOOLKIND_PAE_PD_PHYS:
2015 case PGMPOOLKIND_PAE_PDPT_PHYS:
2016 case PGMPOOLKIND_32BIT_PD_PHYS:
2017 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2018 break;
2019 default:
2020 AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2021 }
2022 }
2023
2024 /* next */
2025 i = pPage->iNext;
2026 } while (i != NIL_PGMPOOL_IDX);
2027#endif
2028 return NULL;
2029}
2030
2031
2032/**
2033 * Enabled write monitoring of a guest page.
2034 *
2035 * @returns VBox status code.
2036 * @retval VINF_SUCCESS on success.
2037 * @param pPool The pool.
2038 * @param pPage The cached page.
2039 */
2040static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2041{
2042 LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
2043
2044 /*
2045 * Filter out the relevant kinds.
2046 */
2047 switch (pPage->enmKind)
2048 {
2049 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2050 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2051 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2052 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2053 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2054 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2055 case PGMPOOLKIND_64BIT_PML4:
2056 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2057 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2058 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2059 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2060 case PGMPOOLKIND_32BIT_PD:
2061 case PGMPOOLKIND_PAE_PDPT:
2062 break;
2063
2064 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2065 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2066 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2067 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2068 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2069 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2070 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2071 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2072 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2073 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2074 case PGMPOOLKIND_ROOT_NESTED:
2075 /* Nothing to monitor here. */
2076 return VINF_SUCCESS;
2077
2078 case PGMPOOLKIND_32BIT_PD_PHYS:
2079 case PGMPOOLKIND_PAE_PDPT_PHYS:
2080 case PGMPOOLKIND_PAE_PD_PHYS:
2081 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2082 /* Nothing to monitor here. */
2083 return VINF_SUCCESS;
2084#ifdef PGMPOOL_WITH_MIXED_PT_CR3
2085 break;
2086#else
2087 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2088#endif
2089 default:
2090 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2091 }
2092
2093 /*
2094 * Install handler.
2095 */
2096 int rc;
2097 PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2098 if (pPageHead)
2099 {
2100 Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2101 Assert(pPageHead->iMonitoredPrev != pPage->idx);
2102
2103#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2104 if (pPageHead->fDirty)
2105 pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirty, true /* force removal */);
2106#endif
2107
2108 pPage->iMonitoredPrev = pPageHead->idx;
2109 pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2110 if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2111 pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2112 pPageHead->iMonitoredNext = pPage->idx;
2113 rc = VINF_SUCCESS;
2114 }
2115 else
2116 {
2117 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2118 PVM pVM = pPool->CTX_SUFF(pVM);
2119 const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
2120 rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2121 GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
2122 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
2123 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
2124 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
2125 pPool->pszAccessHandler);
2126 /** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2127 * the heap size should suffice. */
2128 AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2129 Assert(!(VMMGetCpu(pVM)->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_ISSET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3));
2130 }
2131 pPage->fMonitored = true;
2132 return rc;
2133}
2134
2135
2136/**
2137 * Disables write monitoring of a guest page.
2138 *
2139 * @returns VBox status code.
2140 * @retval VINF_SUCCESS on success.
2141 * @param pPool The pool.
2142 * @param pPage The cached page.
2143 */
2144static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2145{
2146 /*
2147 * Filter out the relevant kinds.
2148 */
2149 switch (pPage->enmKind)
2150 {
2151 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2152 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2153 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2154 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2155 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2156 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2157 case PGMPOOLKIND_64BIT_PML4:
2158 case PGMPOOLKIND_32BIT_PD:
2159 case PGMPOOLKIND_PAE_PDPT:
2160 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2161 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2162 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2163 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2164 break;
2165
2166 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2167 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2168 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2169 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2170 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2171 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2172 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2173 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2174 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2175 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2176 case PGMPOOLKIND_ROOT_NESTED:
2177 case PGMPOOLKIND_PAE_PD_PHYS:
2178 case PGMPOOLKIND_PAE_PDPT_PHYS:
2179 case PGMPOOLKIND_32BIT_PD_PHYS:
2180 /* Nothing to monitor here. */
2181 return VINF_SUCCESS;
2182
2183#ifdef PGMPOOL_WITH_MIXED_PT_CR3
2184 break;
2185#endif
2186 default:
2187 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2188 }
2189
2190 /*
2191 * Remove the page from the monitored list or uninstall it if last.
2192 */
2193 const PVM pVM = pPool->CTX_SUFF(pVM);
2194 int rc;
2195 if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2196 || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2197 {
2198 if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2199 {
2200 PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2201 pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2202 rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
2203 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
2204 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
2205 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pNewHead),
2206 pPool->pszAccessHandler);
2207 AssertFatalRCSuccess(rc);
2208 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2209 }
2210 else
2211 {
2212 pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2213 if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2214 {
2215 pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2216 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2217 }
2218 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2219 rc = VINF_SUCCESS;
2220 }
2221 }
2222 else
2223 {
2224 rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
2225 AssertFatalRC(rc);
2226#ifdef VBOX_STRICT
2227 PVMCPU pVCpu = VMMGetCpu(pVM);
2228#endif
2229 AssertMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2230 ("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2231 }
2232 pPage->fMonitored = false;
2233
2234 /*
2235 * Remove it from the list of modified pages (if in it).
2236 */
2237 pgmPoolMonitorModifiedRemove(pPool, pPage);
2238
2239 return rc;
2240}
2241
2242
2243/**
2244 * Inserts the page into the list of modified pages.
2245 *
2246 * @param pPool The pool.
2247 * @param pPage The page.
2248 */
2249void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2250{
2251 Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2252 AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2253 && pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2254 && pPool->iModifiedHead != pPage->idx,
2255 ("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2256 pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2257 pPool->iModifiedHead, pPool->cModifiedPages));
2258
2259 pPage->iModifiedNext = pPool->iModifiedHead;
2260 if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2261 pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2262 pPool->iModifiedHead = pPage->idx;
2263 pPool->cModifiedPages++;
2264#ifdef VBOX_WITH_STATISTICS
2265 if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2266 pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2267#endif
2268}
2269
2270
2271/**
2272 * Removes the page from the list of modified pages and resets the
2273 * moficiation counter.
2274 *
2275 * @param pPool The pool.
2276 * @param pPage The page which is believed to be in the list of modified pages.
2277 */
2278static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2279{
2280 Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2281 if (pPool->iModifiedHead == pPage->idx)
2282 {
2283 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2284 pPool->iModifiedHead = pPage->iModifiedNext;
2285 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2286 {
2287 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2288 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2289 }
2290 pPool->cModifiedPages--;
2291 }
2292 else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2293 {
2294 pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2295 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2296 {
2297 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2298 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2299 }
2300 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2301 pPool->cModifiedPages--;
2302 }
2303 else
2304 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2305 pPage->cModifications = 0;
2306}
2307
2308
2309/**
2310 * Zaps the list of modified pages, resetting their modification counters in the process.
2311 *
2312 * @param pVM The VM handle.
2313 */
2314static void pgmPoolMonitorModifiedClearAll(PVM pVM)
2315{
2316 pgmLock(pVM);
2317 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2318 LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2319
2320 unsigned cPages = 0; NOREF(cPages);
2321
2322#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2323 pgmPoolResetDirtyPages(pVM, true /* force removal. */);
2324#endif
2325
2326 uint16_t idx = pPool->iModifiedHead;
2327 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2328 while (idx != NIL_PGMPOOL_IDX)
2329 {
2330 PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2331 idx = pPage->iModifiedNext;
2332 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2333 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2334 pPage->cModifications = 0;
2335 Assert(++cPages);
2336 }
2337 AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2338 pPool->cModifiedPages = 0;
2339 pgmUnlock(pVM);
2340}
2341
2342
2343#ifdef IN_RING3
2344/**
2345 * Callback to clear all shadow pages and clear all modification counters.
2346 *
2347 * @returns VBox status code.
2348 * @param pVM The VM handle.
2349 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
2350 * @param pvUser Unused parameter.
2351 *
2352 * @remark Should only be used when monitoring is available, thus placed in
2353 * the PGMPOOL_WITH_MONITORING \#ifdef.
2354 */
2355DECLCALLBACK(int) pgmPoolClearAll(PVM pVM, PVMCPU pVCpu, void *pvUser)
2356{
2357 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2358 STAM_PROFILE_START(&pPool->StatClearAll, c);
2359 LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
2360 NOREF(pvUser); NOREF(pVCpu);
2361
2362 pgmLock(pVM);
2363
2364 /*
2365 * Iterate all the pages until we've encountered all that in use.
2366 * This is simple but not quite optimal solution.
2367 */
2368 unsigned cModifiedPages = 0; NOREF(cModifiedPages);
2369 unsigned cLeft = pPool->cUsedPages;
2370 unsigned iPage = pPool->cCurPages;
2371 while (--iPage >= PGMPOOL_IDX_FIRST)
2372 {
2373 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2374 if (pPage->GCPhys != NIL_RTGCPHYS)
2375 {
2376 switch (pPage->enmKind)
2377 {
2378 /*
2379 * We only care about shadow page tables.
2380 */
2381 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2382 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2383 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2384 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2385 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2386 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2387 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2388 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2389 {
2390#ifdef PGMPOOL_WITH_USER_TRACKING
2391 if (pPage->cPresent)
2392#endif
2393 {
2394 void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
2395 STAM_PROFILE_START(&pPool->StatZeroPage, z);
2396 ASMMemZeroPage(pvShw);
2397 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
2398#ifdef PGMPOOL_WITH_USER_TRACKING
2399 pPage->cPresent = 0;
2400 pPage->iFirstPresent = ~0;
2401#endif
2402 }
2403 }
2404 /* fall thru */
2405
2406 default:
2407 Assert(!pPage->cModifications || ++cModifiedPages);
2408 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
2409 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
2410 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2411 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2412 pPage->cModifications = 0;
2413 break;
2414
2415 }
2416 if (!--cLeft)
2417 break;
2418 }
2419 }
2420
2421 /* swipe the special pages too. */
2422 for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
2423 {
2424 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2425 if (pPage->GCPhys != NIL_RTGCPHYS)
2426 {
2427 Assert(!pPage->cModifications || ++cModifiedPages);
2428 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
2429 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
2430 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2431 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2432 pPage->cModifications = 0;
2433 }
2434 }
2435
2436#ifndef DEBUG_michael
2437 AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
2438#endif
2439 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2440 pPool->cModifiedPages = 0;
2441
2442#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2443 /*
2444 * Clear all the GCPhys links and rebuild the phys ext free list.
2445 */
2446 for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
2447 pRam;
2448 pRam = pRam->CTX_SUFF(pNext))
2449 {
2450 unsigned iPage = pRam->cb >> PAGE_SHIFT;
2451 while (iPage-- > 0)
2452 PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
2453 }
2454
2455 pPool->iPhysExtFreeHead = 0;
2456 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
2457 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
2458 for (unsigned i = 0; i < cMaxPhysExts; i++)
2459 {
2460 paPhysExts[i].iNext = i + 1;
2461 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
2462 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
2463 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
2464 }
2465 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2466#endif
2467
2468#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2469 /* Clear all dirty pages. */
2470 pPool->idxFreeDirtyPage = 0;
2471 pPool->cDirtyPages = 0;
2472 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
2473 pPool->aIdxDirtyPages[i] = NIL_PGMPOOL_IDX;
2474#endif
2475
2476 /* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
2477 for (unsigned idCpu = 0; idCpu < pVM->cCPUs; idCpu++)
2478 {
2479 PVMCPU pVCpu = &pVM->aCpus[idCpu];
2480
2481 pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2482 }
2483
2484 pPool->cPresent = 0;
2485 pgmUnlock(pVM);
2486 PGM_INVL_ALL_VCPU_TLBS(pVM);
2487 STAM_PROFILE_STOP(&pPool->StatClearAll, c);
2488 return VINF_SUCCESS;
2489}
2490#endif /* IN_RING3 */
2491
2492
2493/**
2494 * Handle SyncCR3 pool tasks
2495 *
2496 * @returns VBox status code.
2497 * @retval VINF_SUCCESS if successfully added.
2498 * @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2499 * @param pVCpu The VMCPU handle.
2500 * @remark Should only be used when monitoring is available, thus placed in
2501 * the PGMPOOL_WITH_MONITORING #ifdef.
2502 */
2503int pgmPoolSyncCR3(PVMCPU pVCpu)
2504{
2505 PVM pVM = pVCpu->CTX_SUFF(pVM);
2506 LogFlow(("pgmPoolSyncCR3\n"));
2507
2508 /*
2509 * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2510 * Occasionally we will have to clear all the shadow page tables because we wanted
2511 * to monitor a page which was mapped by too many shadowed page tables. This operation
2512 * sometimes refered to as a 'lightweight flush'.
2513 */
2514# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2515 if (ASMBitTestAndClear(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_CLEAR_PGM_POOL_BIT))
2516 {
2517 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmPoolClearAll, NULL);
2518 AssertRC(rc);
2519 }
2520# else /* !IN_RING3 */
2521 if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2522 {
2523 LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2524 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2525 return VINF_PGM_SYNC_CR3;
2526 }
2527# endif /* !IN_RING3 */
2528 else
2529 pgmPoolMonitorModifiedClearAll(pVM);
2530
2531 return VINF_SUCCESS;
2532}
2533
2534#endif /* PGMPOOL_WITH_MONITORING */
2535#ifdef PGMPOOL_WITH_USER_TRACKING
2536
2537/**
2538 * Frees up at least one user entry.
2539 *
2540 * @returns VBox status code.
2541 * @retval VINF_SUCCESS if successfully added.
2542 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2543 * @param pPool The pool.
2544 * @param iUser The user index.
2545 */
2546static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2547{
2548 STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2549#ifdef PGMPOOL_WITH_CACHE
2550 /*
2551 * Just free cached pages in a braindead fashion.
2552 */
2553 /** @todo walk the age list backwards and free the first with usage. */
2554 int rc = VINF_SUCCESS;
2555 do
2556 {
2557 int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2558 if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2559 rc = rc2;
2560 } while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2561 return rc;
2562#else
2563 /*
2564 * Lazy approach.
2565 */
2566 /* @todo This path no longer works (CR3 root pages will be flushed)!! */
2567 AssertCompileFailed();
2568 Assert(!CPUMIsGuestInLongMode(pVM));
2569 pgmPoolFlushAllInt(pPool);
2570 return VERR_PGM_POOL_FLUSHED;
2571#endif
2572}
2573
2574
2575/**
2576 * Inserts a page into the cache.
2577 *
2578 * This will create user node for the page, insert it into the GCPhys
2579 * hash, and insert it into the age list.
2580 *
2581 * @returns VBox status code.
2582 * @retval VINF_SUCCESS if successfully added.
2583 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2584 * @param pPool The pool.
2585 * @param pPage The cached page.
2586 * @param GCPhys The GC physical address of the page we're gonna shadow.
2587 * @param iUser The user index.
2588 * @param iUserTable The user table index.
2589 */
2590DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2591{
2592 int rc = VINF_SUCCESS;
2593 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2594
2595 LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser %x iUserTable %x\n", GCPhys, iUser, iUserTable));
2596
2597#ifdef VBOX_STRICT
2598 /*
2599 * Check that the entry doesn't already exists.
2600 */
2601 if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2602 {
2603 uint16_t i = pPage->iUserHead;
2604 do
2605 {
2606 Assert(i < pPool->cMaxUsers);
2607 AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2608 i = paUsers[i].iNext;
2609 } while (i != NIL_PGMPOOL_USER_INDEX);
2610 }
2611#endif
2612
2613 /*
2614 * Find free a user node.
2615 */
2616 uint16_t i = pPool->iUserFreeHead;
2617 if (i == NIL_PGMPOOL_USER_INDEX)
2618 {
2619 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2620 if (RT_FAILURE(rc))
2621 return rc;
2622 i = pPool->iUserFreeHead;
2623 }
2624
2625 /*
2626 * Unlink the user node from the free list,
2627 * initialize and insert it into the user list.
2628 */
2629 pPool->iUserFreeHead = paUsers[i].iNext;
2630 paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2631 paUsers[i].iUser = iUser;
2632 paUsers[i].iUserTable = iUserTable;
2633 pPage->iUserHead = i;
2634
2635 /*
2636 * Insert into cache and enable monitoring of the guest page if enabled.
2637 *
2638 * Until we implement caching of all levels, including the CR3 one, we'll
2639 * have to make sure we don't try monitor & cache any recursive reuse of
2640 * a monitored CR3 page. Because all windows versions are doing this we'll
2641 * have to be able to do combined access monitoring, CR3 + PT and
2642 * PD + PT (guest PAE).
2643 *
2644 * Update:
2645 * We're now cooperating with the CR3 monitor if an uncachable page is found.
2646 */
2647#if defined(PGMPOOL_WITH_MONITORING) || defined(PGMPOOL_WITH_CACHE)
2648# ifdef PGMPOOL_WITH_MIXED_PT_CR3
2649 const bool fCanBeMonitored = true;
2650# else
2651 bool fCanBeMonitored = pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
2652 || (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
2653 || pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
2654# endif
2655# ifdef PGMPOOL_WITH_CACHE
2656 pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2657# endif
2658 if (fCanBeMonitored)
2659 {
2660# ifdef PGMPOOL_WITH_MONITORING
2661 rc = pgmPoolMonitorInsert(pPool, pPage);
2662 AssertRC(rc);
2663 }
2664# endif
2665#endif /* PGMPOOL_WITH_MONITORING */
2666 return rc;
2667}
2668
2669
2670# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
2671/**
2672 * Adds a user reference to a page.
2673 *
2674 * This will move the page to the head of the
2675 *
2676 * @returns VBox status code.
2677 * @retval VINF_SUCCESS if successfully added.
2678 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2679 * @param pPool The pool.
2680 * @param pPage The cached page.
2681 * @param iUser The user index.
2682 * @param iUserTable The user table.
2683 */
2684static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2685{
2686 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2687
2688 Log3(("pgmPoolTrackAddUser GCPhys = %RGp iUser %x iUserTable %x\n", pPage->GCPhys, iUser, iUserTable));
2689
2690# ifdef VBOX_STRICT
2691 /*
2692 * Check that the entry doesn't already exists. We only allow multiple users of top-level paging structures (SHW_POOL_ROOT_IDX).
2693 */
2694 if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2695 {
2696 uint16_t i = pPage->iUserHead;
2697 do
2698 {
2699 Assert(i < pPool->cMaxUsers);
2700 AssertMsg(iUser != PGMPOOL_IDX_PD || iUser != PGMPOOL_IDX_PDPT || iUser != PGMPOOL_IDX_NESTED_ROOT || iUser != PGMPOOL_IDX_AMD64_CR3 ||
2701 paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2702 i = paUsers[i].iNext;
2703 } while (i != NIL_PGMPOOL_USER_INDEX);
2704 }
2705# endif
2706
2707 /*
2708 * Allocate a user node.
2709 */
2710 uint16_t i = pPool->iUserFreeHead;
2711 if (i == NIL_PGMPOOL_USER_INDEX)
2712 {
2713 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2714 if (RT_FAILURE(rc))
2715 return rc;
2716 i = pPool->iUserFreeHead;
2717 }
2718 pPool->iUserFreeHead = paUsers[i].iNext;
2719
2720 /*
2721 * Initialize the user node and insert it.
2722 */
2723 paUsers[i].iNext = pPage->iUserHead;
2724 paUsers[i].iUser = iUser;
2725 paUsers[i].iUserTable = iUserTable;
2726 pPage->iUserHead = i;
2727
2728# ifdef PGMPOOL_WITH_CACHE
2729 /*
2730 * Tell the cache to update its replacement stats for this page.
2731 */
2732 pgmPoolCacheUsed(pPool, pPage);
2733# endif
2734 return VINF_SUCCESS;
2735}
2736# endif /* PGMPOOL_WITH_CACHE */
2737
2738
2739/**
2740 * Frees a user record associated with a page.
2741 *
2742 * This does not clear the entry in the user table, it simply replaces the
2743 * user record to the chain of free records.
2744 *
2745 * @param pPool The pool.
2746 * @param HCPhys The HC physical address of the shadow page.
2747 * @param iUser The shadow page pool index of the user table.
2748 * @param iUserTable The index into the user table (shadowed).
2749 */
2750static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2751{
2752 /*
2753 * Unlink and free the specified user entry.
2754 */
2755 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2756
2757 Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
2758 /* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2759 uint16_t i = pPage->iUserHead;
2760 if ( i != NIL_PGMPOOL_USER_INDEX
2761 && paUsers[i].iUser == iUser
2762 && paUsers[i].iUserTable == iUserTable)
2763 {
2764 pPage->iUserHead = paUsers[i].iNext;
2765
2766 paUsers[i].iUser = NIL_PGMPOOL_IDX;
2767 paUsers[i].iNext = pPool->iUserFreeHead;
2768 pPool->iUserFreeHead = i;
2769 return;
2770 }
2771
2772 /* General: Linear search. */
2773 uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2774 while (i != NIL_PGMPOOL_USER_INDEX)
2775 {
2776 if ( paUsers[i].iUser == iUser
2777 && paUsers[i].iUserTable == iUserTable)
2778 {
2779 if (iPrev != NIL_PGMPOOL_USER_INDEX)
2780 paUsers[iPrev].iNext = paUsers[i].iNext;
2781 else
2782 pPage->iUserHead = paUsers[i].iNext;
2783
2784 paUsers[i].iUser = NIL_PGMPOOL_IDX;
2785 paUsers[i].iNext = pPool->iUserFreeHead;
2786 pPool->iUserFreeHead = i;
2787 return;
2788 }
2789 iPrev = i;
2790 i = paUsers[i].iNext;
2791 }
2792
2793 /* Fatal: didn't find it */
2794 AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%RGp\n",
2795 iUser, iUserTable, pPage->GCPhys));
2796}
2797
2798
2799/**
2800 * Gets the entry size of a shadow table.
2801 *
2802 * @param enmKind The kind of page.
2803 *
2804 * @returns The size of the entry in bytes. That is, 4 or 8.
2805 * @returns If the kind is not for a table, an assertion is raised and 0 is
2806 * returned.
2807 */
2808DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2809{
2810 switch (enmKind)
2811 {
2812 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2813 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2814 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2815 case PGMPOOLKIND_32BIT_PD:
2816 case PGMPOOLKIND_32BIT_PD_PHYS:
2817 return 4;
2818
2819 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2820 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2821 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2822 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2823 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2824 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2825 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2826 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2827 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2828 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2829 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2830 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2831 case PGMPOOLKIND_64BIT_PML4:
2832 case PGMPOOLKIND_PAE_PDPT:
2833 case PGMPOOLKIND_ROOT_NESTED:
2834 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2835 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2836 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2837 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2838 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2839 case PGMPOOLKIND_PAE_PD_PHYS:
2840 case PGMPOOLKIND_PAE_PDPT_PHYS:
2841 return 8;
2842
2843 default:
2844 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2845 }
2846}
2847
2848
2849/**
2850 * Gets the entry size of a guest table.
2851 *
2852 * @param enmKind The kind of page.
2853 *
2854 * @returns The size of the entry in bytes. That is, 0, 4 or 8.
2855 * @returns If the kind is not for a table, an assertion is raised and 0 is
2856 * returned.
2857 */
2858DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2859{
2860 switch (enmKind)
2861 {
2862 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2863 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2864 case PGMPOOLKIND_32BIT_PD:
2865 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2866 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2867 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2868 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2869 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2870 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2871 return 4;
2872
2873 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2874 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2875 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2876 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2877 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2878 case PGMPOOLKIND_64BIT_PML4:
2879 case PGMPOOLKIND_PAE_PDPT:
2880 return 8;
2881
2882 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2883 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2884 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2885 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2886 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2887 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2888 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2889 case PGMPOOLKIND_ROOT_NESTED:
2890 case PGMPOOLKIND_PAE_PD_PHYS:
2891 case PGMPOOLKIND_PAE_PDPT_PHYS:
2892 case PGMPOOLKIND_32BIT_PD_PHYS:
2893 /** @todo can we return 0? (nobody is calling this...) */
2894 AssertFailed();
2895 return 0;
2896
2897 default:
2898 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2899 }
2900}
2901
2902#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2903
2904/**
2905 * Scans one shadow page table for mappings of a physical page.
2906 *
2907 * @param pVM The VM handle.
2908 * @param pPhysPage The guest page in question.
2909 * @param iShw The shadow page table.
2910 * @param cRefs The number of references made in that PT.
2911 */
2912static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2913{
2914 LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
2915 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2916
2917 /*
2918 * Assert sanity.
2919 */
2920 Assert(cRefs == 1);
2921 AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2922 PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2923
2924 /*
2925 * Then, clear the actual mappings to the page in the shadow PT.
2926 */
2927 switch (pPage->enmKind)
2928 {
2929 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2930 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2931 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2932 {
2933 const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2934 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2935 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2936 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2937 {
2938 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2939 pPT->a[i].u = 0;
2940 cRefs--;
2941 if (!cRefs)
2942 return;
2943 }
2944#ifdef LOG_ENABLED
2945 Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2946 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2947 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2948 {
2949 Log(("i=%d cRefs=%d\n", i, cRefs--));
2950 }
2951#endif
2952 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2953 break;
2954 }
2955
2956 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2957 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2958 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2959 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2960 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2961 {
2962 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2963 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2964 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2965 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2966 {
2967 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2968 pPT->a[i].u = 0;
2969 cRefs--;
2970 if (!cRefs)
2971 return;
2972 }
2973#ifdef LOG_ENABLED
2974 Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2975 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2976 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2977 {
2978 Log(("i=%d cRefs=%d\n", i, cRefs--));
2979 }
2980#endif
2981 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d u64=%RX64\n", cRefs, pPage->iFirstPresent, pPage->cPresent, u64));
2982 break;
2983 }
2984
2985 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2986 {
2987 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2988 PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2989 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2990 if ((pPT->a[i].u & (EPT_PTE_PG_MASK | X86_PTE_P)) == u64)
2991 {
2992 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2993 pPT->a[i].u = 0;
2994 cRefs--;
2995 if (!cRefs)
2996 return;
2997 }
2998#ifdef LOG_ENABLED
2999 Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3000 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3001 if ((pPT->a[i].u & (EPT_PTE_PG_MASK | X86_PTE_P)) == u64)
3002 {
3003 Log(("i=%d cRefs=%d\n", i, cRefs--));
3004 }
3005#endif
3006 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3007 break;
3008 }
3009
3010 default:
3011 AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3012 }
3013}
3014
3015
3016/**
3017 * Scans one shadow page table for mappings of a physical page.
3018 *
3019 * @param pVM The VM handle.
3020 * @param pPhysPage The guest page in question.
3021 * @param iShw The shadow page table.
3022 * @param cRefs The number of references made in that PT.
3023 */
3024void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
3025{
3026 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3027 LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
3028 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3029 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
3030 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3031 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3032}
3033
3034
3035/**
3036 * Flushes a list of shadow page tables mapping the same physical page.
3037 *
3038 * @param pVM The VM handle.
3039 * @param pPhysPage The guest page in question.
3040 * @param iPhysExt The physical cross reference extent list to flush.
3041 */
3042void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
3043{
3044 Assert(PGMIsLockOwner(pVM));
3045 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3046 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3047 LogFlow(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%R[pgmpage] iPhysExt\n", pPhysPage, iPhysExt));
3048
3049 const uint16_t iPhysExtStart = iPhysExt;
3050 PPGMPOOLPHYSEXT pPhysExt;
3051 do
3052 {
3053 Assert(iPhysExt < pPool->cMaxPhysExts);
3054 pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3055 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3056 if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3057 {
3058 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
3059 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3060 }
3061
3062 /* next */
3063 iPhysExt = pPhysExt->iNext;
3064 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3065
3066 /* insert the list into the free list and clear the ram range entry. */
3067 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3068 pPool->iPhysExtFreeHead = iPhysExtStart;
3069 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3070
3071 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3072}
3073
3074#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3075
3076/**
3077 * Flushes all shadow page table mappings of the given guest page.
3078 *
3079 * This is typically called when the host page backing the guest one has been
3080 * replaced or when the page protection was changed due to an access handler.
3081 *
3082 * @returns VBox status code.
3083 * @retval VINF_SUCCESS if all references has been successfully cleared.
3084 * @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3085 * pool cleaning. FF and sync flags are set.
3086 *
3087 * @param pVM The VM handle.
3088 * @param pPhysPage The guest page in question.
3089 * @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3090 * flushed, it is NOT touched if this isn't necessary.
3091 * The caller MUST initialized this to @a false.
3092 */
3093int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs)
3094{
3095 PVMCPU pVCpu = VMMGetCpu(pVM);
3096 pgmLock(pVM);
3097 int rc = VINF_SUCCESS;
3098#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3099 const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3100 if (u16)
3101 {
3102 /*
3103 * The zero page is currently screwing up the tracking and we'll
3104 * have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3105 * is defined, zero pages won't normally be mapped. Some kind of solution
3106 * will be needed for this problem of course, but it will have to wait...
3107 */
3108 if (PGM_PAGE_IS_ZERO(pPhysPage))
3109 rc = VINF_PGM_GCPHYS_ALIASED;
3110 else
3111 {
3112# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3113 /* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
3114 pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
3115 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3116# endif
3117
3118 if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3119 pgmPoolTrackFlushGCPhysPT(pVM,
3120 pPhysPage,
3121 PGMPOOL_TD_GET_IDX(u16),
3122 PGMPOOL_TD_GET_CREFS(u16));
3123 else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3124 pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, PGMPOOL_TD_GET_IDX(u16));
3125 else
3126 rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3127 *pfFlushTLBs = true;
3128
3129# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3130 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3131# endif
3132 }
3133 }
3134
3135#elif defined(PGMPOOL_WITH_CACHE)
3136 if (PGM_PAGE_IS_ZERO(pPhysPage))
3137 rc = VINF_PGM_GCPHYS_ALIASED;
3138 else
3139 {
3140# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3141 /* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow kill the pool otherwise. */
3142 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3143# endif
3144 rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3145 if (rc == VINF_SUCCESS)
3146 *pfFlushTLBs = true;
3147 }
3148
3149# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3150 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3151# endif
3152
3153#else
3154 rc = VINF_PGM_GCPHYS_ALIASED;
3155#endif
3156
3157 if (rc == VINF_PGM_GCPHYS_ALIASED)
3158 {
3159 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
3160 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3161 rc = VINF_PGM_SYNC_CR3;
3162 }
3163 pgmUnlock(pVM);
3164 return rc;
3165}
3166
3167
3168/**
3169 * Scans all shadow page tables for mappings of a physical page.
3170 *
3171 * This may be slow, but it's most likely more efficient than cleaning
3172 * out the entire page pool / cache.
3173 *
3174 * @returns VBox status code.
3175 * @retval VINF_SUCCESS if all references has been successfully cleared.
3176 * @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3177 * a page pool cleaning.
3178 *
3179 * @param pVM The VM handle.
3180 * @param pPhysPage The guest page in question.
3181 */
3182int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
3183{
3184 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3185 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3186 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3187 pPool->cUsedPages, pPool->cPresent, pPhysPage));
3188
3189#if 1
3190 /*
3191 * There is a limit to what makes sense.
3192 */
3193 if (pPool->cPresent > 1024)
3194 {
3195 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3196 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3197 return VINF_PGM_GCPHYS_ALIASED;
3198 }
3199#endif
3200
3201 /*
3202 * Iterate all the pages until we've encountered all that in use.
3203 * This is simple but not quite optimal solution.
3204 */
3205 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
3206 const uint32_t u32 = u64;
3207 unsigned cLeft = pPool->cUsedPages;
3208 unsigned iPage = pPool->cCurPages;
3209 while (--iPage >= PGMPOOL_IDX_FIRST)
3210 {
3211 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3212 if (pPage->GCPhys != NIL_RTGCPHYS)
3213 {
3214 switch (pPage->enmKind)
3215 {
3216 /*
3217 * We only care about shadow page tables.
3218 */
3219 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3220 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3221 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3222 {
3223 unsigned cPresent = pPage->cPresent;
3224 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3225 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3226 if (pPT->a[i].n.u1Present)
3227 {
3228 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
3229 {
3230 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3231 pPT->a[i].u = 0;
3232 }
3233 if (!--cPresent)
3234 break;
3235 }
3236 break;
3237 }
3238
3239 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3240 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3241 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3242 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3243 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3244 {
3245 unsigned cPresent = pPage->cPresent;
3246 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3247 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3248 if (pPT->a[i].n.u1Present)
3249 {
3250 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
3251 {
3252 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3253 pPT->a[i].u = 0;
3254 }
3255 if (!--cPresent)
3256 break;
3257 }
3258 break;
3259 }
3260 }
3261 if (!--cLeft)
3262 break;
3263 }
3264 }
3265
3266 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3267 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3268 return VINF_SUCCESS;
3269}
3270
3271
3272/**
3273 * Clears the user entry in a user table.
3274 *
3275 * This is used to remove all references to a page when flushing it.
3276 */
3277static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3278{
3279 Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3280 Assert(pUser->iUser < pPool->cCurPages);
3281 uint32_t iUserTable = pUser->iUserTable;
3282
3283 /*
3284 * Map the user page.
3285 */
3286 PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3287 union
3288 {
3289 uint64_t *pau64;
3290 uint32_t *pau32;
3291 } u;
3292 u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3293
3294 LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3295
3296 /* Safety precaution in case we change the paging for other modes too in the future. */
3297 Assert(!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage));
3298
3299#ifdef VBOX_STRICT
3300 /*
3301 * Some sanity checks.
3302 */
3303 switch (pUserPage->enmKind)
3304 {
3305 case PGMPOOLKIND_32BIT_PD:
3306 case PGMPOOLKIND_32BIT_PD_PHYS:
3307 Assert(iUserTable < X86_PG_ENTRIES);
3308 break;
3309 case PGMPOOLKIND_PAE_PDPT:
3310 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3311 case PGMPOOLKIND_PAE_PDPT_PHYS:
3312 Assert(iUserTable < 4);
3313 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3314 break;
3315 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3316 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3317 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3318 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3319 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3320 case PGMPOOLKIND_PAE_PD_PHYS:
3321 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3322 break;
3323 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3324 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3325 Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
3326 break;
3327 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3328 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3329 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3330 break;
3331 case PGMPOOLKIND_64BIT_PML4:
3332 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3333 /* GCPhys >> PAGE_SHIFT is the index here */
3334 break;
3335 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3336 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3337 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3338 break;
3339
3340 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3341 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3342 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3343 break;
3344
3345 case PGMPOOLKIND_ROOT_NESTED:
3346 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3347 break;
3348
3349 default:
3350 AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3351 break;
3352 }
3353#endif /* VBOX_STRICT */
3354
3355 /*
3356 * Clear the entry in the user page.
3357 */
3358 switch (pUserPage->enmKind)
3359 {
3360 /* 32-bit entries */
3361 case PGMPOOLKIND_32BIT_PD:
3362 case PGMPOOLKIND_32BIT_PD_PHYS:
3363 u.pau32[iUserTable] = 0;
3364 break;
3365
3366 /* 64-bit entries */
3367 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3368 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3369 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3370 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3371 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3372#if defined(IN_RC)
3373 /* In 32 bits PAE mode we *must* invalidate the TLB when changing a PDPT entry; the CPU fetches them only during cr3 load, so any
3374 * non-present PDPT will continue to cause page faults.
3375 */
3376 ASMReloadCR3();
3377#endif
3378 /* no break */
3379 case PGMPOOLKIND_PAE_PD_PHYS:
3380 case PGMPOOLKIND_PAE_PDPT_PHYS:
3381 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3382 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3383 case PGMPOOLKIND_64BIT_PML4:
3384 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3385 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3386 case PGMPOOLKIND_PAE_PDPT:
3387 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3388 case PGMPOOLKIND_ROOT_NESTED:
3389 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3390 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3391 u.pau64[iUserTable] = 0;
3392 break;
3393
3394 default:
3395 AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
3396 }
3397}
3398
3399
3400/**
3401 * Clears all users of a page.
3402 */
3403static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3404{
3405 /*
3406 * Free all the user records.
3407 */
3408 LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
3409
3410 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3411 uint16_t i = pPage->iUserHead;
3412 while (i != NIL_PGMPOOL_USER_INDEX)
3413 {
3414 /* Clear enter in user table. */
3415 pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
3416
3417 /* Free it. */
3418 const uint16_t iNext = paUsers[i].iNext;
3419 paUsers[i].iUser = NIL_PGMPOOL_IDX;
3420 paUsers[i].iNext = pPool->iUserFreeHead;
3421 pPool->iUserFreeHead = i;
3422
3423 /* Next. */
3424 i = iNext;
3425 }
3426 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3427}
3428
3429#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3430
3431/**
3432 * Allocates a new physical cross reference extent.
3433 *
3434 * @returns Pointer to the allocated extent on success. NULL if we're out of them.
3435 * @param pVM The VM handle.
3436 * @param piPhysExt Where to store the phys ext index.
3437 */
3438PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
3439{
3440 Assert(PGMIsLockOwner(pVM));
3441 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3442 uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3443 if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3444 {
3445 STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3446 return NULL;
3447 }
3448 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3449 pPool->iPhysExtFreeHead = pPhysExt->iNext;
3450 pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3451 *piPhysExt = iPhysExt;
3452 return pPhysExt;
3453}
3454
3455
3456/**
3457 * Frees a physical cross reference extent.
3458 *
3459 * @param pVM The VM handle.
3460 * @param iPhysExt The extent to free.
3461 */
3462void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
3463{
3464 Assert(PGMIsLockOwner(pVM));
3465 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3466 Assert(iPhysExt < pPool->cMaxPhysExts);
3467 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3468 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3469 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3470 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3471 pPool->iPhysExtFreeHead = iPhysExt;
3472}
3473
3474
3475/**
3476 * Frees a physical cross reference extent.
3477 *
3478 * @param pVM The VM handle.
3479 * @param iPhysExt The extent to free.
3480 */
3481void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
3482{
3483 Assert(PGMIsLockOwner(pVM));
3484 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3485
3486 const uint16_t iPhysExtStart = iPhysExt;
3487 PPGMPOOLPHYSEXT pPhysExt;
3488 do
3489 {
3490 Assert(iPhysExt < pPool->cMaxPhysExts);
3491 pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3492 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3493 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3494
3495 /* next */
3496 iPhysExt = pPhysExt->iNext;
3497 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3498
3499 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3500 pPool->iPhysExtFreeHead = iPhysExtStart;
3501}
3502
3503
3504/**
3505 * Insert a reference into a list of physical cross reference extents.
3506 *
3507 * @returns The new tracking data for PGMPAGE.
3508 *
3509 * @param pVM The VM handle.
3510 * @param iPhysExt The physical extent index of the list head.
3511 * @param iShwPT The shadow page table index.
3512 *
3513 */
3514static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
3515{
3516 Assert(PGMIsLockOwner(pVM));
3517 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3518 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3519
3520 /* special common case. */
3521 if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3522 {
3523 paPhysExts[iPhysExt].aidx[2] = iShwPT;
3524 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3525 LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d}\n", iPhysExt, iShwPT));
3526 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3527 }
3528
3529 /* general treatment. */
3530 const uint16_t iPhysExtStart = iPhysExt;
3531 unsigned cMax = 15;
3532 for (;;)
3533 {
3534 Assert(iPhysExt < pPool->cMaxPhysExts);
3535 for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3536 if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3537 {
3538 paPhysExts[iPhysExt].aidx[i] = iShwPT;
3539 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3540 LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
3541 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
3542 }
3543 if (!--cMax)
3544 {
3545 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3546 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3547 LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
3548 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3549 }
3550 }
3551
3552 /* add another extent to the list. */
3553 PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3554 if (!pNew)
3555 {
3556 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3557 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3558 LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
3559 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3560 }
3561 pNew->iNext = iPhysExtStart;
3562 pNew->aidx[0] = iShwPT;
3563 LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
3564 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3565}
3566
3567
3568/**
3569 * Add a reference to guest physical page where extents are in use.
3570 *
3571 * @returns The new tracking data for PGMPAGE.
3572 *
3573 * @param pVM The VM handle.
3574 * @param u16 The ram range flags (top 16-bits).
3575 * @param iShwPT The shadow page table index.
3576 */
3577uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
3578{
3579 pgmLock(pVM);
3580 if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3581 {
3582 /*
3583 * Convert to extent list.
3584 */
3585 Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3586 uint16_t iPhysExt;
3587 PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3588 if (pPhysExt)
3589 {
3590 LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
3591 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
3592 pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
3593 pPhysExt->aidx[1] = iShwPT;
3594 u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3595 }
3596 else
3597 u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3598 }
3599 else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3600 {
3601 /*
3602 * Insert into the extent list.
3603 */
3604 u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT);
3605 }
3606 else
3607 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
3608 pgmUnlock(pVM);
3609 return u16;
3610}
3611
3612
3613/**
3614 * Clear references to guest physical memory.
3615 *
3616 * @param pPool The pool.
3617 * @param pPage The page.
3618 * @param pPhysPage Pointer to the aPages entry in the ram range.
3619 */
3620void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
3621{
3622 const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
3623 AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3624
3625 uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
3626 if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
3627 {
3628 PVM pVM = pPool->CTX_SUFF(pVM);
3629 pgmLock(pVM);
3630
3631 uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
3632 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3633 do
3634 {
3635 Assert(iPhysExt < pPool->cMaxPhysExts);
3636
3637 /*
3638 * Look for the shadow page and check if it's all freed.
3639 */
3640 for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3641 {
3642 if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
3643 {
3644 paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
3645
3646 for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3647 if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
3648 {
3649 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3650 pgmUnlock(pVM);
3651 return;
3652 }
3653
3654 /* we can free the node. */
3655 const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
3656 if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
3657 && iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
3658 {
3659 /* lonely node */
3660 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3661 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
3662 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3663 }
3664 else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
3665 {
3666 /* head */
3667 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
3668 PGM_PAGE_SET_TRACKING(pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
3669 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3670 }
3671 else
3672 {
3673 /* in list */
3674 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3675 paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
3676 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3677 }
3678 iPhysExt = iPhysExtNext;
3679 pgmUnlock(pVM);
3680 return;
3681 }
3682 }
3683
3684 /* next */
3685 iPhysExtPrev = iPhysExt;
3686 iPhysExt = paPhysExts[iPhysExt].iNext;
3687 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3688
3689 pgmUnlock(pVM);
3690 AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3691 }
3692 else /* nothing to do */
3693 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
3694}
3695
3696
3697/**
3698 * Clear references to guest physical memory.
3699 *
3700 * This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
3701 * is assumed to be correct, so the linear search can be skipped and we can assert
3702 * at an earlier point.
3703 *
3704 * @param pPool The pool.
3705 * @param pPage The page.
3706 * @param HCPhys The host physical address corresponding to the guest page.
3707 * @param GCPhys The guest physical address corresponding to HCPhys.
3708 */
3709static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
3710{
3711 /*
3712 * Walk range list.
3713 */
3714 PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3715 while (pRam)
3716 {
3717 RTGCPHYS off = GCPhys - pRam->GCPhys;
3718 if (off < pRam->cb)
3719 {
3720 /* does it match? */
3721 const unsigned iPage = off >> PAGE_SHIFT;
3722 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3723#ifdef LOG_ENABLED
3724RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
3725Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
3726#endif
3727 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3728 {
3729 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3730 return;
3731 }
3732 break;
3733 }
3734 pRam = pRam->CTX_SUFF(pNext);
3735 }
3736 AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
3737}
3738
3739
3740/**
3741 * Clear references to guest physical memory.
3742 *
3743 * @param pPool The pool.
3744 * @param pPage The page.
3745 * @param HCPhys The host physical address corresponding to the guest page.
3746 * @param GCPhysHint The guest physical address which may corresponding to HCPhys.
3747 */
3748void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
3749{
3750 Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
3751
3752 /*
3753 * Walk range list.
3754 */
3755 PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3756 while (pRam)
3757 {
3758 RTGCPHYS off = GCPhysHint - pRam->GCPhys;
3759 if (off < pRam->cb)
3760 {
3761 /* does it match? */
3762 const unsigned iPage = off >> PAGE_SHIFT;
3763 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3764 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3765 {
3766 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3767 return;
3768 }
3769 break;
3770 }
3771 pRam = pRam->CTX_SUFF(pNext);
3772 }
3773
3774 /*
3775 * Damn, the hint didn't work. We'll have to do an expensive linear search.
3776 */
3777 STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
3778 pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3779 while (pRam)
3780 {
3781 unsigned iPage = pRam->cb >> PAGE_SHIFT;
3782 while (iPage-- > 0)
3783 {
3784 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3785 {
3786 Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
3787 HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
3788 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3789 return;
3790 }
3791 }
3792 pRam = pRam->CTX_SUFF(pNext);
3793 }
3794
3795 AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp\n", HCPhys, GCPhysHint));
3796}
3797
3798
3799/**
3800 * Clear references to guest physical memory in a 32-bit / 32-bit page table.
3801 *
3802 * @param pPool The pool.
3803 * @param pPage The page.
3804 * @param pShwPT The shadow page table (mapping of the page).
3805 * @param pGstPT The guest page table.
3806 */
3807DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
3808{
3809 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
3810 if (pShwPT->a[i].n.u1Present)
3811 {
3812 Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
3813 i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3814 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3815 if (!--pPage->cPresent)
3816 break;
3817 }
3818}
3819
3820
3821/**
3822 * Clear references to guest physical memory in a PAE / 32-bit page table.
3823 *
3824 * @param pPool The pool.
3825 * @param pPage The page.
3826 * @param pShwPT The shadow page table (mapping of the page).
3827 * @param pGstPT The guest page table (just a half one).
3828 */
3829DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
3830{
3831 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3832 if (pShwPT->a[i].n.u1Present)
3833 {
3834 Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
3835 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3836 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3837 }
3838}
3839
3840
3841/**
3842 * Clear references to guest physical memory in a PAE / PAE page table.
3843 *
3844 * @param pPool The pool.
3845 * @param pPage The page.
3846 * @param pShwPT The shadow page table (mapping of the page).
3847 * @param pGstPT The guest page table.
3848 */
3849DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
3850{
3851 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3852 if (pShwPT->a[i].n.u1Present)
3853 {
3854 Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
3855 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
3856 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
3857 }
3858}
3859
3860
3861/**
3862 * Clear references to guest physical memory in a 32-bit / 4MB page table.
3863 *
3864 * @param pPool The pool.
3865 * @param pPage The page.
3866 * @param pShwPT The shadow page table (mapping of the page).
3867 */
3868DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
3869{
3870 RTGCPHYS GCPhys = pPage->GCPhys;
3871 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3872 if (pShwPT->a[i].n.u1Present)
3873 {
3874 Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
3875 i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
3876 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
3877 }
3878}
3879
3880
3881/**
3882 * Clear references to guest physical memory in a PAE / 2/4MB page table.
3883 *
3884 * @param pPool The pool.
3885 * @param pPage The page.
3886 * @param pShwPT The shadow page table (mapping of the page).
3887 */
3888DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
3889{
3890 RTGCPHYS GCPhys = pPage->GCPhys;
3891 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3892 if (pShwPT->a[i].n.u1Present)
3893 {
3894 Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
3895 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
3896 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
3897 }
3898}
3899
3900#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3901
3902
3903/**
3904 * Clear references to shadowed pages in a 32 bits page directory.
3905 *
3906 * @param pPool The pool.
3907 * @param pPage The page.
3908 * @param pShwPD The shadow page directory (mapping of the page).
3909 */
3910DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
3911{
3912 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3913 {
3914 if ( pShwPD->a[i].n.u1Present
3915 && !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3916 )
3917 {
3918 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
3919 if (pSubPage)
3920 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3921 else
3922 AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
3923 }
3924 }
3925}
3926
3927/**
3928 * Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
3929 *
3930 * @param pPool The pool.
3931 * @param pPage The page.
3932 * @param pShwPD The shadow page directory (mapping of the page).
3933 */
3934DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
3935{
3936 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3937 {
3938 if ( pShwPD->a[i].n.u1Present
3939 && !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3940 )
3941 {
3942 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
3943 if (pSubPage)
3944 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3945 else
3946 AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
3947 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3948 }
3949 }
3950}
3951
3952/**
3953 * Clear references to shadowed pages in a PAE page directory pointer table.
3954 *
3955 * @param pPool The pool.
3956 * @param pPage The page.
3957 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3958 */
3959DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3960{
3961 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
3962 {
3963 if ( pShwPDPT->a[i].n.u1Present
3964 && !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
3965 )
3966 {
3967 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3968 if (pSubPage)
3969 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3970 else
3971 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3972 }
3973 }
3974}
3975
3976
3977/**
3978 * Clear references to shadowed pages in a 64-bit page directory pointer table.
3979 *
3980 * @param pPool The pool.
3981 * @param pPage The page.
3982 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3983 */
3984DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3985{
3986 for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3987 {
3988 Assert(!(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING));
3989 if (pShwPDPT->a[i].n.u1Present)
3990 {
3991 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3992 if (pSubPage)
3993 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3994 else
3995 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3996 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3997 }
3998 }
3999}
4000
4001
4002/**
4003 * Clear references to shadowed pages in a 64-bit level 4 page table.
4004 *
4005 * @param pPool The pool.
4006 * @param pPage The page.
4007 * @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4008 */
4009DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4010{
4011 for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4012 {
4013 if (pShwPML4->a[i].n.u1Present)
4014 {
4015 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
4016 if (pSubPage)
4017 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4018 else
4019 AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
4020 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4021 }
4022 }
4023}
4024
4025
4026/**
4027 * Clear references to shadowed pages in an EPT page table.
4028 *
4029 * @param pPool The pool.
4030 * @param pPage The page.
4031 * @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4032 */
4033DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4034{
4035 RTGCPHYS GCPhys = pPage->GCPhys;
4036 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4037 if (pShwPT->a[i].n.u1Present)
4038 {
4039 Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4040 i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
4041 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys);
4042 }
4043}
4044
4045
4046/**
4047 * Clear references to shadowed pages in an EPT page directory.
4048 *
4049 * @param pPool The pool.
4050 * @param pPage The page.
4051 * @param pShwPD The shadow page directory (mapping of the page).
4052 */
4053DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4054{
4055 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4056 {
4057 if (pShwPD->a[i].n.u1Present)
4058 {
4059 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
4060 if (pSubPage)
4061 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4062 else
4063 AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4064 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4065 }
4066 }
4067}
4068
4069
4070/**
4071 * Clear references to shadowed pages in an EPT page directory pointer table.
4072 *
4073 * @param pPool The pool.
4074 * @param pPage The page.
4075 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4076 */
4077DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4078{
4079 for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4080 {
4081 if (pShwPDPT->a[i].n.u1Present)
4082 {
4083 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
4084 if (pSubPage)
4085 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4086 else
4087 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
4088 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4089 }
4090 }
4091}
4092
4093
4094/**
4095 * Clears all references made by this page.
4096 *
4097 * This includes other shadow pages and GC physical addresses.
4098 *
4099 * @param pPool The pool.
4100 * @param pPage The page.
4101 */
4102static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4103{
4104 /*
4105 * Map the shadow page and take action according to the page kind.
4106 */
4107 void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
4108 switch (pPage->enmKind)
4109 {
4110#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4111 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4112 {
4113 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4114 void *pvGst;
4115 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4116 pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4117 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4118 break;
4119 }
4120
4121 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4122 {
4123 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4124 void *pvGst;
4125 int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4126 pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
4127 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4128 break;
4129 }
4130
4131 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4132 {
4133 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4134 void *pvGst;
4135 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4136 pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
4137 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4138 break;
4139 }
4140
4141 case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4142 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4143 {
4144 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4145 pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4146 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4147 break;
4148 }
4149
4150 case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
4151 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4152 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4153 {
4154 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4155 pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
4156 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4157 break;
4158 }
4159
4160#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
4161 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4162 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4163 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4164 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4165 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4166 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4167 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4168 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4169 break;
4170#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
4171
4172 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4173 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4174 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4175 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4176 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4177 case PGMPOOLKIND_PAE_PD_PHYS:
4178 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4179 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4180 pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
4181 break;
4182
4183 case PGMPOOLKIND_32BIT_PD_PHYS:
4184 case PGMPOOLKIND_32BIT_PD:
4185 pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
4186 break;
4187
4188 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4189 case PGMPOOLKIND_PAE_PDPT:
4190 case PGMPOOLKIND_PAE_PDPT_PHYS:
4191 pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
4192 break;
4193
4194 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4195 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4196 pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
4197 break;
4198
4199 case PGMPOOLKIND_64BIT_PML4:
4200 pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
4201 break;
4202
4203 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4204 pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
4205 break;
4206
4207 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4208 pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
4209 break;
4210
4211 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4212 pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
4213 break;
4214
4215 default:
4216 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
4217 }
4218
4219 /* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
4220 STAM_PROFILE_START(&pPool->StatZeroPage, z);
4221 ASMMemZeroPage(pvShw);
4222 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4223 pPage->fZeroed = true;
4224 PGMPOOL_UNLOCK_PTR(pPool->CTX_SUFF(pVM), pvShw);
4225}
4226#endif /* PGMPOOL_WITH_USER_TRACKING */
4227
4228/**
4229 * Flushes a pool page.
4230 *
4231 * This moves the page to the free list after removing all user references to it.
4232 *
4233 * @returns VBox status code.
4234 * @retval VINF_SUCCESS on success.
4235 * @param pPool The pool.
4236 * @param HCPhys The HC physical address of the shadow page.
4237 */
4238int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4239{
4240 PVM pVM = pPool->CTX_SUFF(pVM);
4241
4242 int rc = VINF_SUCCESS;
4243 STAM_PROFILE_START(&pPool->StatFlushPage, f);
4244 LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4245 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4246
4247 /*
4248 * Quietly reject any attempts at flushing any of the special root pages.
4249 */
4250 if (pPage->idx < PGMPOOL_IDX_FIRST)
4251 {
4252 AssertFailed(); /* can no longer happen */
4253 Log(("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4254 return VINF_SUCCESS;
4255 }
4256
4257 pgmLock(pVM);
4258
4259 /*
4260 * Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4261 */
4262 if (pgmPoolIsPageLocked(&pVM->pgm.s, pPage))
4263 {
4264 AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4265 || pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4266 || pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4267 || pPage->enmKind == PGMPOOLKIND_32BIT_PD
4268 || pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4269 || pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4270 || pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4271 || pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4272 || pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
4273 ("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
4274 Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4275 pgmUnlock(pVM);
4276 return VINF_SUCCESS;
4277 }
4278
4279#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4280 /* Start a subset so we won't run out of mapping space. */
4281 PVMCPU pVCpu = VMMGetCpu(pVM);
4282 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
4283#endif
4284
4285 /*
4286 * Mark the page as being in need of a ASMMemZeroPage().
4287 */
4288 pPage->fZeroed = false;
4289
4290#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4291 if (pPage->fDirty)
4292 pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirty, true /* force removal */);
4293#endif
4294
4295#ifdef PGMPOOL_WITH_USER_TRACKING
4296 /*
4297 * Clear the page.
4298 */
4299 pgmPoolTrackClearPageUsers(pPool, pPage);
4300 STAM_PROFILE_START(&pPool->StatTrackDeref,a);
4301 pgmPoolTrackDeref(pPool, pPage);
4302 STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
4303#endif
4304
4305#ifdef PGMPOOL_WITH_CACHE
4306 /*
4307 * Flush it from the cache.
4308 */
4309 pgmPoolCacheFlushPage(pPool, pPage);
4310#endif /* PGMPOOL_WITH_CACHE */
4311
4312#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4313 /* Heavy stuff done. */
4314 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
4315#endif
4316
4317#ifdef PGMPOOL_WITH_MONITORING
4318 /*
4319 * Deregistering the monitoring.
4320 */
4321 if (pPage->fMonitored)
4322 rc = pgmPoolMonitorFlush(pPool, pPage);
4323#endif
4324
4325 /*
4326 * Free the page.
4327 */
4328 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
4329 pPage->iNext = pPool->iFreeHead;
4330 pPool->iFreeHead = pPage->idx;
4331 pPage->enmKind = PGMPOOLKIND_FREE;
4332 pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4333 pPage->GCPhys = NIL_RTGCPHYS;
4334 pPage->fReusedFlushPending = false;
4335
4336 pPool->cUsedPages--;
4337 pgmUnlock(pVM);
4338 STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
4339 return rc;
4340}
4341
4342
4343/**
4344 * Frees a usage of a pool page.
4345 *
4346 * The caller is responsible to updating the user table so that it no longer
4347 * references the shadow page.
4348 *
4349 * @param pPool The pool.
4350 * @param HCPhys The HC physical address of the shadow page.
4351 * @param iUser The shadow page pool index of the user table.
4352 * @param iUserTable The index into the user table (shadowed).
4353 */
4354void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
4355{
4356 PVM pVM = pPool->CTX_SUFF(pVM);
4357
4358 STAM_PROFILE_START(&pPool->StatFree, a);
4359 LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%#x iUserTable=%#x\n",
4360 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
4361 Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
4362 pgmLock(pVM);
4363#ifdef PGMPOOL_WITH_USER_TRACKING
4364 pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
4365#endif
4366#ifdef PGMPOOL_WITH_CACHE
4367 if (!pPage->fCached)
4368#endif
4369 pgmPoolFlushPage(pPool, pPage);
4370 pgmUnlock(pVM);
4371 STAM_PROFILE_STOP(&pPool->StatFree, a);
4372}
4373
4374
4375/**
4376 * Makes one or more free page free.
4377 *
4378 * @returns VBox status code.
4379 * @retval VINF_SUCCESS on success.
4380 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4381 *
4382 * @param pPool The pool.
4383 * @param enmKind Page table kind
4384 * @param iUser The user of the page.
4385 */
4386static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
4387{
4388 PVM pVM = pPool->CTX_SUFF(pVM);
4389
4390 LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
4391
4392 /*
4393 * If the pool isn't full grown yet, expand it.
4394 */
4395 if ( pPool->cCurPages < pPool->cMaxPages
4396#if defined(IN_RC)
4397 /* Hack alert: we can't deal with jumps to ring 3 when called from MapCR3 and allocating pages for PAE PDs. */
4398 && enmKind != PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4399 && (enmKind < PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD || enmKind > PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD)
4400#endif
4401 )
4402 {
4403 STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
4404#ifdef IN_RING3
4405 int rc = PGMR3PoolGrow(pVM);
4406#else
4407 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
4408#endif
4409 if (RT_FAILURE(rc))
4410 return rc;
4411 STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
4412 if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
4413 return VINF_SUCCESS;
4414 }
4415
4416#ifdef PGMPOOL_WITH_CACHE
4417 /*
4418 * Free one cached page.
4419 */
4420 return pgmPoolCacheFreeOne(pPool, iUser);
4421#else
4422 /*
4423 * Flush the pool.
4424 *
4425 * If we have tracking enabled, it should be possible to come up with
4426 * a cheap replacement strategy...
4427 */
4428 /* @todo This path no longer works (CR3 root pages will be flushed)!! */
4429 AssertCompileFailed();
4430 Assert(!CPUMIsGuestInLongMode(pVM));
4431 pgmPoolFlushAllInt(pPool);
4432 return VERR_PGM_POOL_FLUSHED;
4433#endif
4434}
4435
4436/**
4437 * Allocates a page from the pool.
4438 *
4439 * This page may actually be a cached page and not in need of any processing
4440 * on the callers part.
4441 *
4442 * @returns VBox status code.
4443 * @retval VINF_SUCCESS if a NEW page was allocated.
4444 * @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4445 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4446 * @param pVM The VM handle.
4447 * @param GCPhys The GC physical address of the page we're gonna shadow.
4448 * For 4MB and 2MB PD entries, it's the first address the
4449 * shadow PT is covering.
4450 * @param enmKind The kind of mapping.
4451 * @param enmAccess Access type for the mapping (only relevant for big pages)
4452 * @param iUser The shadow page pool index of the user table.
4453 * @param iUserTable The index into the user table (shadowed).
4454 * @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4455 * @param fLockPage Lock the page
4456 */
4457int pgmPoolAllocEx(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage, bool fLockPage)
4458{
4459 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4460 STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4461 LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%#x iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
4462 *ppPage = NULL;
4463 /** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4464 * (TRPMR3SyncIDT) because of FF priority. Try fix that?
4465 * Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
4466
4467 pgmLock(pVM);
4468
4469#ifdef PGMPOOL_WITH_CACHE
4470 if (pPool->fCacheEnabled)
4471 {
4472 int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, iUser, iUserTable, ppPage);
4473 if (RT_SUCCESS(rc2))
4474 {
4475 if (fLockPage)
4476 pgmPoolLockPage(pPool, *ppPage);
4477 pgmUnlock(pVM);
4478 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4479 LogFlow(("pgmPoolAlloc: cached returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, *ppPage, (*ppPage)->Core.Key, (*ppPage)->idx));
4480 return rc2;
4481 }
4482 }
4483#endif
4484
4485 /*
4486 * Allocate a new one.
4487 */
4488 int rc = VINF_SUCCESS;
4489 uint16_t iNew = pPool->iFreeHead;
4490 if (iNew == NIL_PGMPOOL_IDX)
4491 {
4492 rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
4493 if (RT_FAILURE(rc))
4494 {
4495 pgmUnlock(pVM);
4496 Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
4497 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4498 return rc;
4499 }
4500 iNew = pPool->iFreeHead;
4501 AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
4502 }
4503
4504 /* unlink the free head */
4505 PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
4506 pPool->iFreeHead = pPage->iNext;
4507 pPage->iNext = NIL_PGMPOOL_IDX;
4508
4509 /*
4510 * Initialize it.
4511 */
4512 pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
4513 pPage->enmKind = enmKind;
4514 pPage->enmAccess = enmAccess;
4515 pPage->GCPhys = GCPhys;
4516 pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
4517 pPage->fMonitored = false;
4518 pPage->fCached = false;
4519#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4520 pPage->fDirty = false;
4521#endif
4522 pPage->fReusedFlushPending = false;
4523#ifdef PGMPOOL_WITH_MONITORING
4524 pPage->cModifications = 0;
4525 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4526 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4527#else
4528 pPage->fCR3Mix = false;
4529#endif
4530#ifdef PGMPOOL_WITH_USER_TRACKING
4531 pPage->cPresent = 0;
4532 pPage->iFirstPresent = ~0;
4533 pPage->pvLastAccessHandlerFault = 0;
4534 pPage->cLastAccessHandlerCount = 0;
4535 pPage->pvLastAccessHandlerRip = 0;
4536
4537 /*
4538 * Insert into the tracking and cache. If this fails, free the page.
4539 */
4540 int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
4541 if (RT_FAILURE(rc3))
4542 {
4543 pPool->cUsedPages--;
4544 pPage->enmKind = PGMPOOLKIND_FREE;
4545 pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4546 pPage->GCPhys = NIL_RTGCPHYS;
4547 pPage->iNext = pPool->iFreeHead;
4548 pPool->iFreeHead = pPage->idx;
4549 pgmUnlock(pVM);
4550 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4551 Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
4552 return rc3;
4553 }
4554#endif /* PGMPOOL_WITH_USER_TRACKING */
4555
4556 /*
4557 * Commit the allocation, clear the page and return.
4558 */
4559#ifdef VBOX_WITH_STATISTICS
4560 if (pPool->cUsedPages > pPool->cUsedPagesHigh)
4561 pPool->cUsedPagesHigh = pPool->cUsedPages;
4562#endif
4563
4564 if (!pPage->fZeroed)
4565 {
4566 STAM_PROFILE_START(&pPool->StatZeroPage, z);
4567 void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4568 ASMMemZeroPage(pv);
4569 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4570 }
4571
4572 *ppPage = pPage;
4573 if (fLockPage)
4574 pgmPoolLockPage(pPool, pPage);
4575 pgmUnlock(pVM);
4576 LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
4577 rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
4578 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4579 return rc;
4580}
4581
4582
4583/**
4584 * Frees a usage of a pool page.
4585 *
4586 * @param pVM The VM handle.
4587 * @param HCPhys The HC physical address of the shadow page.
4588 * @param iUser The shadow page pool index of the user table.
4589 * @param iUserTable The index into the user table (shadowed).
4590 */
4591void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
4592{
4593 LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
4594 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4595 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
4596}
4597
4598/**
4599 * Internal worker for finding a 'in-use' shadow page give by it's physical address.
4600 *
4601 * @returns Pointer to the shadow page structure.
4602 * @param pPool The pool.
4603 * @param HCPhys The HC physical address of the shadow page.
4604 */
4605PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
4606{
4607 PVM pVM = pPool->CTX_SUFF(pVM);
4608
4609 Assert(PGMIsLockOwner(pVM));
4610
4611 /*
4612 * Look up the page.
4613 */
4614 pgmLock(pVM);
4615 PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
4616 pgmUnlock(pVM);
4617
4618 AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
4619 return pPage;
4620}
4621
4622
4623#ifdef IN_RING3
4624/**
4625 * Flushes the entire cache.
4626 *
4627 * It will assert a global CR3 flush (FF) and assumes the caller is aware of this
4628 * and execute this CR3 flush.
4629 *
4630 * @param pPool The pool.
4631 */
4632void pgmR3PoolReset(PVM pVM)
4633{
4634 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4635
4636 Assert(PGMIsLockOwner(pVM));
4637 STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
4638 LogFlow(("pgmPoolFlushAllInt:\n"));
4639
4640 /*
4641 * If there are no pages in the pool, there is nothing to do.
4642 */
4643 if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
4644 {
4645 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4646 return;
4647 }
4648
4649 /*
4650 * Exit the shadow mode since we're going to clear everything,
4651 * including the root page.
4652 */
4653 for (unsigned i=0;i<pVM->cCPUs;i++)
4654 {
4655 PVMCPU pVCpu = &pVM->aCpus[i];
4656 pgmR3ExitShadowModeBeforePoolFlush(pVM, pVCpu);
4657 }
4658
4659 /*
4660 * Nuke the free list and reinsert all pages into it.
4661 */
4662 for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
4663 {
4664 PPGMPOOLPAGE pPage = &pPool->aPages[i];
4665
4666 Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
4667#ifdef PGMPOOL_WITH_MONITORING
4668 if (pPage->fMonitored)
4669 pgmPoolMonitorFlush(pPool, pPage);
4670 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4671 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4672 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4673 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4674 pPage->cModifications = 0;
4675#endif
4676 pPage->GCPhys = NIL_RTGCPHYS;
4677 pPage->enmKind = PGMPOOLKIND_FREE;
4678 pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4679 Assert(pPage->idx == i);
4680 pPage->iNext = i + 1;
4681 pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
4682 pPage->fSeenNonGlobal = false;
4683 pPage->fMonitored = false;
4684#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4685 pPage->fDirty = false;
4686#endif
4687 pPage->fCached = false;
4688 pPage->fReusedFlushPending = false;
4689#ifdef PGMPOOL_WITH_USER_TRACKING
4690 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4691#else
4692 pPage->fCR3Mix = false;
4693#endif
4694#ifdef PGMPOOL_WITH_CACHE
4695 pPage->iAgeNext = NIL_PGMPOOL_IDX;
4696 pPage->iAgePrev = NIL_PGMPOOL_IDX;
4697#endif
4698 pPage->cLocked = 0;
4699 }
4700 pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
4701 pPool->iFreeHead = PGMPOOL_IDX_FIRST;
4702 pPool->cUsedPages = 0;
4703
4704#ifdef PGMPOOL_WITH_USER_TRACKING
4705 /*
4706 * Zap and reinitialize the user records.
4707 */
4708 pPool->cPresent = 0;
4709 pPool->iUserFreeHead = 0;
4710 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4711 const unsigned cMaxUsers = pPool->cMaxUsers;
4712 for (unsigned i = 0; i < cMaxUsers; i++)
4713 {
4714 paUsers[i].iNext = i + 1;
4715 paUsers[i].iUser = NIL_PGMPOOL_IDX;
4716 paUsers[i].iUserTable = 0xfffffffe;
4717 }
4718 paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
4719#endif
4720
4721#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4722 /*
4723 * Clear all the GCPhys links and rebuild the phys ext free list.
4724 */
4725 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
4726 pRam;
4727 pRam = pRam->CTX_SUFF(pNext))
4728 {
4729 unsigned iPage = pRam->cb >> PAGE_SHIFT;
4730 while (iPage-- > 0)
4731 PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
4732 }
4733
4734 pPool->iPhysExtFreeHead = 0;
4735 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4736 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
4737 for (unsigned i = 0; i < cMaxPhysExts; i++)
4738 {
4739 paPhysExts[i].iNext = i + 1;
4740 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
4741 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
4742 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
4743 }
4744 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4745#endif
4746
4747#ifdef PGMPOOL_WITH_MONITORING
4748 /*
4749 * Just zap the modified list.
4750 */
4751 pPool->cModifiedPages = 0;
4752 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
4753#endif
4754
4755#ifdef PGMPOOL_WITH_CACHE
4756 /*
4757 * Clear the GCPhys hash and the age list.
4758 */
4759 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
4760 pPool->aiHash[i] = NIL_PGMPOOL_IDX;
4761 pPool->iAgeHead = NIL_PGMPOOL_IDX;
4762 pPool->iAgeTail = NIL_PGMPOOL_IDX;
4763#endif
4764
4765#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4766 /* Clear all dirty pages. */
4767 pPool->idxFreeDirtyPage = 0;
4768 pPool->cDirtyPages = 0;
4769 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
4770 pPool->aIdxDirtyPages[i] = NIL_PGMPOOL_IDX;
4771#endif
4772
4773 /*
4774 * Reinsert active pages into the hash and ensure monitoring chains are correct.
4775 */
4776 for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
4777 {
4778 PPGMPOOLPAGE pPage = &pPool->aPages[i];
4779 pPage->iNext = NIL_PGMPOOL_IDX;
4780#ifdef PGMPOOL_WITH_MONITORING
4781 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4782 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4783 pPage->cModifications = 0;
4784 /* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
4785 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4786 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4787 if (pPage->fMonitored)
4788 {
4789 int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
4790 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
4791 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
4792 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
4793 pPool->pszAccessHandler);
4794 AssertFatalRCSuccess(rc);
4795# ifdef PGMPOOL_WITH_CACHE
4796 pgmPoolHashInsert(pPool, pPage);
4797# endif
4798 }
4799#endif
4800#ifdef PGMPOOL_WITH_USER_TRACKING
4801 Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
4802#endif
4803#ifdef PGMPOOL_WITH_CACHE
4804 Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
4805 Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
4806#endif
4807 }
4808
4809 for (unsigned i=0;i<pVM->cCPUs;i++)
4810 {
4811 PVMCPU pVCpu = &pVM->aCpus[i];
4812 /*
4813 * Re-enter the shadowing mode and assert Sync CR3 FF.
4814 */
4815 pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
4816 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4817 }
4818
4819 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4820}
4821#endif /* IN_RING3 */
4822
4823#ifdef LOG_ENABLED
4824static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
4825{
4826 switch(enmKind)
4827 {
4828 case PGMPOOLKIND_INVALID:
4829 return "PGMPOOLKIND_INVALID";
4830 case PGMPOOLKIND_FREE:
4831 return "PGMPOOLKIND_FREE";
4832 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4833 return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
4834 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4835 return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
4836 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4837 return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
4838 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4839 return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
4840 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4841 return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
4842 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4843 return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
4844 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4845 return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
4846 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4847 return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
4848 case PGMPOOLKIND_32BIT_PD:
4849 return "PGMPOOLKIND_32BIT_PD";
4850 case PGMPOOLKIND_32BIT_PD_PHYS:
4851 return "PGMPOOLKIND_32BIT_PD_PHYS";
4852 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4853 return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
4854 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4855 return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
4856 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4857 return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
4858 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4859 return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
4860 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4861 return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
4862 case PGMPOOLKIND_PAE_PD_PHYS:
4863 return "PGMPOOLKIND_PAE_PD_PHYS";
4864 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4865 return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
4866 case PGMPOOLKIND_PAE_PDPT:
4867 return "PGMPOOLKIND_PAE_PDPT";
4868 case PGMPOOLKIND_PAE_PDPT_PHYS:
4869 return "PGMPOOLKIND_PAE_PDPT_PHYS";
4870 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4871 return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
4872 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4873 return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
4874 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4875 return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
4876 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4877 return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
4878 case PGMPOOLKIND_64BIT_PML4:
4879 return "PGMPOOLKIND_64BIT_PML4";
4880 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4881 return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
4882 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4883 return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
4884 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4885 return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
4886 case PGMPOOLKIND_ROOT_NESTED:
4887 return "PGMPOOLKIND_ROOT_NESTED";
4888 }
4889 return "Unknown kind!";
4890}
4891#endif /* LOG_ENABLED*/
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette