VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 22537

Last change on this file since 22537 was 22537, checked in by vboxsync, 15 years ago

Another reason to flush.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 179.3 KB
Line 
1/* $Id: PGMAllPool.cpp 22537 2009-08-27 16:23:10Z vboxsync $ */
2/** @file
3 * PGM Shadow Page Pool.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_PGM_POOL
27#include <VBox/pgm.h>
28#include <VBox/mm.h>
29#include <VBox/em.h>
30#include <VBox/cpum.h>
31#ifdef IN_RC
32# include <VBox/patm.h>
33#endif
34#include "PGMInternal.h"
35#include <VBox/vm.h>
36#include <VBox/disopcode.h>
37#include <VBox/hwacc_vmx.h>
38
39#include <VBox/log.h>
40#include <VBox/err.h>
41#include <iprt/asm.h>
42#include <iprt/string.h>
43
44
45/*******************************************************************************
46* Internal Functions *
47*******************************************************************************/
48RT_C_DECLS_BEGIN
49static void pgmPoolFlushAllInt(PPGMPOOL pPool);
50#ifdef PGMPOOL_WITH_USER_TRACKING
51DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
52DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
53static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
54#endif
55#ifdef PGMPOOL_WITH_CACHE
56static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
57#endif
58#ifdef PGMPOOL_WITH_MONITORING
59static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
60#endif
61#ifndef IN_RING3
62DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
63#endif
64#ifdef LOG_ENABLED
65static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
66#endif
67
68void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs);
69void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt);
70int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage);
71PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
72void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
73void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
74
75RT_C_DECLS_END
76
77
78/**
79 * Checks if the specified page pool kind is for a 4MB or 2MB guest page.
80 *
81 * @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
82 * @param enmKind The page kind.
83 */
84DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
85{
86 switch (enmKind)
87 {
88 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
89 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
90 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
91 return true;
92 default:
93 return false;
94 }
95}
96
97/** @def PGMPOOL_PAGE_2_LOCKED_PTR
98 * Maps a pool page pool into the current context and lock it (RC only).
99 *
100 * @returns VBox status code.
101 * @param pVM The VM handle.
102 * @param pPage The pool page.
103 *
104 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
105 * small page window employeed by that function. Be careful.
106 * @remark There is no need to assert on the result.
107 */
108#if defined(IN_RC)
109DECLINLINE(void *) PGMPOOL_PAGE_2_LOCKED_PTR(PVM pVM, PPGMPOOLPAGE pPage)
110{
111 void *pv = pgmPoolMapPageInlined(&pVM->pgm.s, pPage);
112
113 /* Make sure the dynamic mapping will not be reused. */
114 if (pv)
115 PGMDynLockHCPage(pVM, (uint8_t *)pv);
116
117 return pv;
118}
119#else
120# define PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage) PGMPOOL_PAGE_2_PTR(pVM, pPage)
121#endif
122
123/** @def PGMPOOL_UNLOCK_PTR
124 * Unlock a previously locked dynamic caching (RC only).
125 *
126 * @returns VBox status code.
127 * @param pVM The VM handle.
128 * @param pPage The pool page.
129 *
130 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
131 * small page window employeed by that function. Be careful.
132 * @remark There is no need to assert on the result.
133 */
134#if defined(IN_RC)
135DECLINLINE(void) PGMPOOL_UNLOCK_PTR(PVM pVM, void *pvPage)
136{
137 if (pvPage)
138 PGMDynUnlockHCPage(pVM, (uint8_t *)pvPage);
139}
140#else
141# define PGMPOOL_UNLOCK_PTR(pVM, pPage) do {} while (0)
142#endif
143
144
145#ifdef PGMPOOL_WITH_MONITORING
146/**
147 * Determin the size of a write instruction.
148 * @returns number of bytes written.
149 * @param pDis The disassembler state.
150 */
151static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
152{
153 /*
154 * This is very crude and possibly wrong for some opcodes,
155 * but since it's not really supposed to be called we can
156 * probably live with that.
157 */
158 return DISGetParamSize(pDis, &pDis->param1);
159}
160
161
162/**
163 * Flushes a chain of pages sharing the same access monitor.
164 *
165 * @returns VBox status code suitable for scheduling.
166 * @param pPool The pool.
167 * @param pPage A page in the chain.
168 */
169int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
170{
171 LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
172
173 /*
174 * Find the list head.
175 */
176 uint16_t idx = pPage->idx;
177 if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
178 {
179 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
180 {
181 idx = pPage->iMonitoredPrev;
182 Assert(idx != pPage->idx);
183 pPage = &pPool->aPages[idx];
184 }
185 }
186
187 /*
188 * Iterate the list flushing each shadow page.
189 */
190 int rc = VINF_SUCCESS;
191 for (;;)
192 {
193 idx = pPage->iMonitoredNext;
194 Assert(idx != pPage->idx);
195 if (pPage->idx >= PGMPOOL_IDX_FIRST)
196 {
197 int rc2 = pgmPoolFlushPage(pPool, pPage);
198 AssertRC(rc2);
199 }
200 /* next */
201 if (idx == NIL_PGMPOOL_IDX)
202 break;
203 pPage = &pPool->aPages[idx];
204 }
205 return rc;
206}
207
208
209/**
210 * Wrapper for getting the current context pointer to the entry being modified.
211 *
212 * @returns VBox status code suitable for scheduling.
213 * @param pVM VM Handle.
214 * @param pvDst Destination address
215 * @param pvSrc Source guest virtual address.
216 * @param GCPhysSrc The source guest physical address.
217 * @param cb Size of data to read
218 */
219DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVM pVM, void *pvDst, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
220{
221#if defined(IN_RING3)
222 memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
223 return VINF_SUCCESS;
224#else
225 /* @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
226 return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
227#endif
228}
229
230/**
231 * Process shadow entries before they are changed by the guest.
232 *
233 * For PT entries we will clear them. For PD entries, we'll simply check
234 * for mapping conflicts and set the SyncCR3 FF if found.
235 *
236 * @param pVCpu VMCPU handle
237 * @param pPool The pool.
238 * @param pPage The head page.
239 * @param GCPhysFault The guest physical fault address.
240 * @param uAddress In R0 and GC this is the guest context fault address (flat).
241 * In R3 this is the host context 'fault' address.
242 * @param pDis The disassembler state for figuring out the write size.
243 * This need not be specified if the caller knows we won't do cross entry accesses.
244 */
245void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pDis)
246{
247 AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%#x (idx=%#x)\n", pPage->iMonitoredPrev, pPage->idx));
248 const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
249 const unsigned cbWrite = pDis ? pgmPoolDisasWriteSize(pDis) : 0;
250 PVM pVM = pPool->CTX_SUFF(pVM);
251
252 LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n", (RTGCPTR)pvAddress, GCPhysFault, cbWrite));
253
254 for (;;)
255 {
256 union
257 {
258 void *pv;
259 PX86PT pPT;
260 PX86PTPAE pPTPae;
261 PX86PD pPD;
262 PX86PDPAE pPDPae;
263 PX86PDPT pPDPT;
264 PX86PML4 pPML4;
265 } uShw;
266
267 LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s\n", pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
268
269 uShw.pv = NULL;
270 switch (pPage->enmKind)
271 {
272 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
273 {
274 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
275 const unsigned iShw = off / sizeof(X86PTE);
276 LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
277 if (uShw.pPT->a[iShw].n.u1Present)
278 {
279# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
280 X86PTE GstPte;
281
282 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
283 AssertRC(rc);
284 Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
285 pgmPoolTracDerefGCPhysHint(pPool, pPage,
286 uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
287 GstPte.u & X86_PTE_PG_MASK);
288# endif
289 ASMAtomicWriteSize(&uShw.pPT->a[iShw], 0);
290 }
291 break;
292 }
293
294 /* page/2 sized */
295 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
296 {
297 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
298 if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
299 {
300 const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
301 LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
302 if (uShw.pPTPae->a[iShw].n.u1Present)
303 {
304# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
305 X86PTE GstPte;
306 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
307 AssertRC(rc);
308
309 Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
310 pgmPoolTracDerefGCPhysHint(pPool, pPage,
311 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
312 GstPte.u & X86_PTE_PG_MASK);
313# endif
314 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw], 0);
315 }
316 }
317 break;
318 }
319
320 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
321 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
322 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
323 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
324 {
325 unsigned iGst = off / sizeof(X86PDE);
326 unsigned iShwPdpt = iGst / 256;
327 unsigned iShw = (iGst % 256) * 2;
328 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
329
330 LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
331 if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
332 {
333 for (unsigned i = 0; i < 2; i++)
334 {
335# ifndef IN_RING0
336 if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
337 {
338 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
339 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
340 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
341 break;
342 }
343 else
344# endif /* !IN_RING0 */
345 if (uShw.pPDPae->a[iShw+i].n.u1Present)
346 {
347 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
348 pgmPoolFree(pVM,
349 uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
350 pPage->idx,
351 iShw + i);
352 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw+i], 0);
353 }
354
355 /* paranoia / a bit assumptive. */
356 if ( pDis
357 && (off & 3)
358 && (off & 3) + cbWrite > 4)
359 {
360 const unsigned iShw2 = iShw + 2 + i;
361 if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
362 {
363# ifndef IN_RING0
364 if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
365 {
366 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
367 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
368 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
369 break;
370 }
371 else
372# endif /* !IN_RING0 */
373 if (uShw.pPDPae->a[iShw2].n.u1Present)
374 {
375 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
376 pgmPoolFree(pVM,
377 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
378 pPage->idx,
379 iShw2);
380 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
381 }
382 }
383 }
384 }
385 }
386 break;
387 }
388
389 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
390 {
391 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
392 const unsigned iShw = off / sizeof(X86PTEPAE);
393 if (uShw.pPTPae->a[iShw].n.u1Present)
394 {
395# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
396 X86PTEPAE GstPte;
397 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
398 AssertRC(rc);
399
400 Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
401 pgmPoolTracDerefGCPhysHint(pPool, pPage,
402 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
403 GstPte.u & X86_PTE_PAE_PG_MASK);
404# endif
405 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw].u, 0);
406 }
407
408 /* paranoia / a bit assumptive. */
409 if ( pDis
410 && (off & 7)
411 && (off & 7) + cbWrite > sizeof(X86PTEPAE))
412 {
413 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
414 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
415
416 if (uShw.pPTPae->a[iShw2].n.u1Present)
417 {
418# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
419 X86PTEPAE GstPte;
420# ifdef IN_RING3
421 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, (RTHCPTR)((RTHCUINTPTR)pvAddress + sizeof(GstPte)), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
422# else
423 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress + sizeof(GstPte), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
424# endif
425 AssertRC(rc);
426 Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
427 pgmPoolTracDerefGCPhysHint(pPool, pPage,
428 uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK,
429 GstPte.u & X86_PTE_PAE_PG_MASK);
430# endif
431 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw2].u ,0);
432 }
433 }
434 break;
435 }
436
437 case PGMPOOLKIND_32BIT_PD:
438 {
439 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
440 const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
441
442 LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
443# ifndef IN_RING0
444 if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
445 {
446 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
447 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
448 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
449 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
450 break;
451 }
452# endif /* !IN_RING0 */
453# ifndef IN_RING0
454 else
455# endif /* !IN_RING0 */
456 {
457 if (uShw.pPD->a[iShw].n.u1Present)
458 {
459 LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
460 pgmPoolFree(pVM,
461 uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
462 pPage->idx,
463 iShw);
464 ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
465 }
466 }
467 /* paranoia / a bit assumptive. */
468 if ( pDis
469 && (off & 3)
470 && (off & 3) + cbWrite > sizeof(X86PTE))
471 {
472 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
473 if ( iShw2 != iShw
474 && iShw2 < RT_ELEMENTS(uShw.pPD->a))
475 {
476# ifndef IN_RING0
477 if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
478 {
479 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
480 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
481 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
482 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
483 break;
484 }
485# endif /* !IN_RING0 */
486# ifndef IN_RING0
487 else
488# endif /* !IN_RING0 */
489 {
490 if (uShw.pPD->a[iShw2].n.u1Present)
491 {
492 LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
493 pgmPoolFree(pVM,
494 uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
495 pPage->idx,
496 iShw2);
497 ASMAtomicWriteSize(&uShw.pPD->a[iShw2].u, 0);
498 }
499 }
500 }
501 }
502#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
503 if ( uShw.pPD->a[iShw].n.u1Present
504 && !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
505 {
506 LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
507# ifdef IN_RC /* TLB load - we're pushing things a bit... */
508 ASMProbeReadByte(pvAddress);
509# endif
510 pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
511 ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
512 }
513#endif
514 break;
515 }
516
517 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
518 {
519 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
520 const unsigned iShw = off / sizeof(X86PDEPAE);
521#ifndef IN_RING0
522 if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
523 {
524 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
525 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
526 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
527 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
528 break;
529 }
530#endif /* !IN_RING0 */
531 /*
532 * Causes trouble when the guest uses a PDE to refer to the whole page table level
533 * structure. (Invalidate here; faults later on when it tries to change the page
534 * table entries -> recheck; probably only applies to the RC case.)
535 */
536# ifndef IN_RING0
537 else
538# endif /* !IN_RING0 */
539 {
540 if (uShw.pPDPae->a[iShw].n.u1Present)
541 {
542 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
543 pgmPoolFree(pVM,
544 uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
545 pPage->idx,
546 iShw);
547 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
548 }
549 }
550 /* paranoia / a bit assumptive. */
551 if ( pDis
552 && (off & 7)
553 && (off & 7) + cbWrite > sizeof(X86PDEPAE))
554 {
555 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
556 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
557
558#ifndef IN_RING0
559 if ( iShw2 != iShw
560 && uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
561 {
562 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
563 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
564 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
565 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
566 break;
567 }
568#endif /* !IN_RING0 */
569# ifndef IN_RING0
570 else
571# endif /* !IN_RING0 */
572 if (uShw.pPDPae->a[iShw2].n.u1Present)
573 {
574 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
575 pgmPoolFree(pVM,
576 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
577 pPage->idx,
578 iShw2);
579 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
580 }
581 }
582 break;
583 }
584
585 case PGMPOOLKIND_PAE_PDPT:
586 {
587 /*
588 * Hopefully this doesn't happen very often:
589 * - touching unused parts of the page
590 * - messing with the bits of pd pointers without changing the physical address
591 */
592 /* PDPT roots are not page aligned; 32 byte only! */
593 const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
594
595 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
596 const unsigned iShw = offPdpt / sizeof(X86PDPE);
597 if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
598 {
599# ifndef IN_RING0
600 if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
601 {
602 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
603 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
604 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
605 LogFlow(("pgmPoolMonitorChainChanging: Detected pdpt conflict at iShw=%#x!\n", iShw));
606 break;
607 }
608# endif /* !IN_RING0 */
609# ifndef IN_RING0
610 else
611# endif /* !IN_RING0 */
612 if (uShw.pPDPT->a[iShw].n.u1Present)
613 {
614 LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
615 pgmPoolFree(pVM,
616 uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
617 pPage->idx,
618 iShw);
619 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
620 }
621
622 /* paranoia / a bit assumptive. */
623 if ( pDis
624 && (offPdpt & 7)
625 && (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
626 {
627 const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
628 if ( iShw2 != iShw
629 && iShw2 < X86_PG_PAE_PDPE_ENTRIES)
630 {
631# ifndef IN_RING0
632 if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
633 {
634 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
635 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
636 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
637 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
638 break;
639 }
640# endif /* !IN_RING0 */
641# ifndef IN_RING0
642 else
643# endif /* !IN_RING0 */
644 if (uShw.pPDPT->a[iShw2].n.u1Present)
645 {
646 LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
647 pgmPoolFree(pVM,
648 uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
649 pPage->idx,
650 iShw2);
651 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
652 }
653 }
654 }
655 }
656 break;
657 }
658
659#ifndef IN_RC
660 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
661 {
662 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
663 const unsigned iShw = off / sizeof(X86PDEPAE);
664 Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
665 if (uShw.pPDPae->a[iShw].n.u1Present)
666 {
667 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
668 pgmPoolFree(pVM,
669 uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
670 pPage->idx,
671 iShw);
672 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
673 }
674 /* paranoia / a bit assumptive. */
675 if ( pDis
676 && (off & 7)
677 && (off & 7) + cbWrite > sizeof(X86PDEPAE))
678 {
679 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
680 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
681
682 Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
683 if (uShw.pPDPae->a[iShw2].n.u1Present)
684 {
685 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
686 pgmPoolFree(pVM,
687 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
688 pPage->idx,
689 iShw2);
690 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
691 }
692 }
693 break;
694 }
695
696 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
697 {
698 /*
699 * Hopefully this doesn't happen very often:
700 * - messing with the bits of pd pointers without changing the physical address
701 */
702 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
703 {
704 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
705 const unsigned iShw = off / sizeof(X86PDPE);
706 if (uShw.pPDPT->a[iShw].n.u1Present)
707 {
708 LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
709 pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
710 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
711 }
712 /* paranoia / a bit assumptive. */
713 if ( pDis
714 && (off & 7)
715 && (off & 7) + cbWrite > sizeof(X86PDPE))
716 {
717 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
718 if (uShw.pPDPT->a[iShw2].n.u1Present)
719 {
720 LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
721 pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
722 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
723 }
724 }
725 }
726 break;
727 }
728
729 case PGMPOOLKIND_64BIT_PML4:
730 {
731 /*
732 * Hopefully this doesn't happen very often:
733 * - messing with the bits of pd pointers without changing the physical address
734 */
735 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
736 {
737 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
738 const unsigned iShw = off / sizeof(X86PDPE);
739 if (uShw.pPML4->a[iShw].n.u1Present)
740 {
741 LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
742 pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
743 ASMAtomicWriteSize(&uShw.pPML4->a[iShw].u, 0);
744 }
745 /* paranoia / a bit assumptive. */
746 if ( pDis
747 && (off & 7)
748 && (off & 7) + cbWrite > sizeof(X86PDPE))
749 {
750 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
751 if (uShw.pPML4->a[iShw2].n.u1Present)
752 {
753 LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
754 pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
755 ASMAtomicWriteSize(&uShw.pPML4->a[iShw2].u, 0);
756 }
757 }
758 }
759 break;
760 }
761#endif /* IN_RING0 */
762
763 default:
764 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
765 }
766 PGMPOOL_UNLOCK_PTR(pVM, uShw.pv);
767
768 /* next */
769 if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
770 return;
771 pPage = &pPool->aPages[pPage->iMonitoredNext];
772 }
773}
774
775# ifndef IN_RING3
776/**
777 * Checks if a access could be a fork operation in progress.
778 *
779 * Meaning, that the guest is setting up the parent process for Copy-On-Write.
780 *
781 * @returns true if it's likly that we're forking, otherwise false.
782 * @param pPool The pool.
783 * @param pDis The disassembled instruction.
784 * @param offFault The access offset.
785 */
786DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
787{
788 /*
789 * i386 linux is using btr to clear X86_PTE_RW.
790 * The functions involved are (2.6.16 source inspection):
791 * clear_bit
792 * ptep_set_wrprotect
793 * copy_one_pte
794 * copy_pte_range
795 * copy_pmd_range
796 * copy_pud_range
797 * copy_page_range
798 * dup_mmap
799 * dup_mm
800 * copy_mm
801 * copy_process
802 * do_fork
803 */
804 if ( pDis->pCurInstr->opcode == OP_BTR
805 && !(offFault & 4)
806 /** @todo Validate that the bit index is X86_PTE_RW. */
807 )
808 {
809 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,Fork));
810 return true;
811 }
812 return false;
813}
814
815
816/**
817 * Determine whether the page is likely to have been reused.
818 *
819 * @returns true if we consider the page as being reused for a different purpose.
820 * @returns false if we consider it to still be a paging page.
821 * @param pVM VM Handle.
822 * @param pVCpu VMCPU Handle.
823 * @param pRegFrame Trap register frame.
824 * @param pDis The disassembly info for the faulting instruction.
825 * @param pvFault The fault address.
826 *
827 * @remark The REP prefix check is left to the caller because of STOSD/W.
828 */
829DECLINLINE(bool) pgmPoolMonitorIsReused(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault)
830{
831#ifndef IN_RC
832 /** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
833 if ( HWACCMHasPendingIrq(pVM)
834 && (pRegFrame->rsp - pvFault) < 32)
835 {
836 /* Fault caused by stack writes while trying to inject an interrupt event. */
837 Log(("pgmPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
838 return true;
839 }
840#else
841 NOREF(pVM); NOREF(pvFault);
842#endif
843
844 LogFlow(("Reused instr %RGv %d at %RGv param1.flags=%x param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->opcode, pvFault, pDis->param1.flags, pDis->param1.base.reg_gen));
845
846 /* Non-supervisor mode write means it's used for something else. */
847 if (CPUMGetGuestCPL(pVCpu, pRegFrame) != 0)
848 return true;
849
850 switch (pDis->pCurInstr->opcode)
851 {
852 /* call implies the actual push of the return address faulted */
853 case OP_CALL:
854 Log4(("pgmPoolMonitorIsReused: CALL\n"));
855 return true;
856 case OP_PUSH:
857 Log4(("pgmPoolMonitorIsReused: PUSH\n"));
858 return true;
859 case OP_PUSHF:
860 Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
861 return true;
862 case OP_PUSHA:
863 Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
864 return true;
865 case OP_FXSAVE:
866 Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
867 return true;
868 case OP_MOVNTI: /* solaris - block_zero_no_xmm */
869 Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
870 return true;
871 case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
872 Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
873 return true;
874 case OP_MOVSWD:
875 case OP_STOSWD:
876 if ( pDis->prefix == (PREFIX_REP|PREFIX_REX)
877 && pRegFrame->rcx >= 0x40
878 )
879 {
880 Assert(pDis->mode == CPUMODE_64BIT);
881
882 Log(("pgmPoolMonitorIsReused: OP_STOSQ\n"));
883 return true;
884 }
885 return false;
886 }
887 if ( ( (pDis->param1.flags & USE_REG_GEN32)
888 || (pDis->param1.flags & USE_REG_GEN64))
889 && (pDis->param1.base.reg_gen == USE_REG_ESP))
890 {
891 Log4(("pgmPoolMonitorIsReused: ESP\n"));
892 return true;
893 }
894
895 return false;
896}
897
898
899/**
900 * Flushes the page being accessed.
901 *
902 * @returns VBox status code suitable for scheduling.
903 * @param pVM The VM handle.
904 * @param pVCpu The VMCPU handle.
905 * @param pPool The pool.
906 * @param pPage The pool page (head).
907 * @param pDis The disassembly of the write instruction.
908 * @param pRegFrame The trap register frame.
909 * @param GCPhysFault The fault address as guest physical address.
910 * @param pvFault The fault address.
911 */
912static int pgmPoolAccessHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
913 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
914{
915 /*
916 * First, do the flushing.
917 */
918 int rc = pgmPoolMonitorChainFlush(pPool, pPage);
919
920 /*
921 * Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
922 * @todo: why is this necessary? an instruction restart would be sufficient, wouldn't it?
923 */
924 uint32_t cbWritten;
925 int rc2 = EMInterpretInstructionCPU(pVM, pVCpu, pDis, pRegFrame, pvFault, &cbWritten);
926 if (RT_SUCCESS(rc2))
927 pRegFrame->rip += pDis->opsize;
928 else if (rc2 == VERR_EM_INTERPRETER)
929 {
930#ifdef IN_RC
931 if (PATMIsPatchGCAddr(pVM, (RTRCPTR)pRegFrame->eip))
932 {
933 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
934 pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
935 rc = VINF_SUCCESS;
936 STAM_COUNTER_INC(&pPool->StatMonitorRZIntrFailPatch2);
937 }
938 else
939#endif
940 {
941 rc = VINF_EM_RAW_EMULATE_INSTR;
942 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
943 }
944 }
945 else
946 rc = rc2;
947
948 /* See use in pgmPoolAccessHandlerSimple(). */
949 PGM_INVL_VCPU_TLBS(pVCpu);
950
951 LogFlow(("pgmPoolAccessHandlerPT: returns %Rrc (flushed)\n", rc));
952 return rc;
953
954}
955
956
957/**
958 * Handles the STOSD write accesses.
959 *
960 * @returns VBox status code suitable for scheduling.
961 * @param pVM The VM handle.
962 * @param pPool The pool.
963 * @param pPage The pool page (head).
964 * @param pDis The disassembly of the write instruction.
965 * @param pRegFrame The trap register frame.
966 * @param GCPhysFault The fault address as guest physical address.
967 * @param pvFault The fault address.
968 */
969DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
970 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
971{
972 unsigned uIncrement = pDis->param1.size;
973
974 Assert(pDis->mode == CPUMODE_32BIT || pDis->mode == CPUMODE_64BIT);
975 Assert(pRegFrame->rcx <= 0x20);
976
977#ifdef VBOX_STRICT
978 if (pDis->opmode == CPUMODE_32BIT)
979 Assert(uIncrement == 4);
980 else
981 Assert(uIncrement == 8);
982#endif
983
984 Log3(("pgmPoolAccessHandlerSTOSD\n"));
985
986 /*
987 * Increment the modification counter and insert it into the list
988 * of modified pages the first time.
989 */
990 if (!pPage->cModifications++)
991 pgmPoolMonitorModifiedInsert(pPool, pPage);
992
993 /*
994 * Execute REP STOSD.
995 *
996 * This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
997 * write situation, meaning that it's safe to write here.
998 */
999 PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
1000 RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
1001 while (pRegFrame->rcx)
1002 {
1003#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1004 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1005 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1006 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1007#else
1008 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1009#endif
1010#ifdef IN_RC
1011 *(uint32_t *)pu32 = pRegFrame->eax;
1012#else
1013 PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
1014#endif
1015 pu32 += uIncrement;
1016 GCPhysFault += uIncrement;
1017 pRegFrame->rdi += uIncrement;
1018 pRegFrame->rcx--;
1019 }
1020 pRegFrame->rip += pDis->opsize;
1021
1022#ifdef IN_RC
1023 /* See use in pgmPoolAccessHandlerSimple(). */
1024 PGM_INVL_VCPU_TLBS(pVCpu);
1025#endif
1026
1027 LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
1028 return VINF_SUCCESS;
1029}
1030
1031
1032/**
1033 * Handles the simple write accesses.
1034 *
1035 * @returns VBox status code suitable for scheduling.
1036 * @param pVM The VM handle.
1037 * @param pVCpu The VMCPU handle.
1038 * @param pPool The pool.
1039 * @param pPage The pool page (head).
1040 * @param pDis The disassembly of the write instruction.
1041 * @param pRegFrame The trap register frame.
1042 * @param GCPhysFault The fault address as guest physical address.
1043 * @param pvFault The fault address.
1044 */
1045DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
1046 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
1047{
1048 Log3(("pgmPoolAccessHandlerSimple\n"));
1049 /*
1050 * Increment the modification counter and insert it into the list
1051 * of modified pages the first time.
1052 */
1053 if (!pPage->cModifications++)
1054 pgmPoolMonitorModifiedInsert(pPool, pPage);
1055
1056 /*
1057 * Clear all the pages. ASSUMES that pvFault is readable.
1058 */
1059#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1060 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1061 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pDis);
1062 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1063#else
1064 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pDis);
1065#endif
1066
1067 /*
1068 * Interpret the instruction.
1069 */
1070 uint32_t cb;
1071 int rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, pRegFrame, pvFault, &cb);
1072 if (RT_SUCCESS(rc))
1073 pRegFrame->rip += pDis->opsize;
1074 else if (rc == VERR_EM_INTERPRETER)
1075 {
1076 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for %04x:%RGv - opcode=%d\n",
1077 pRegFrame->cs, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->opcode));
1078 rc = VINF_EM_RAW_EMULATE_INSTR;
1079 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
1080 }
1081
1082#ifdef IN_RC
1083 /*
1084 * Quick hack, with logging enabled we're getting stale
1085 * code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
1086 * Flushing here is BAD and expensive, I think EMInterpretDisasOne will
1087 * have to be fixed to support this. But that'll have to wait till next week.
1088 *
1089 * An alternative is to keep track of the changed PTEs together with the
1090 * GCPhys from the guest PT. This may proove expensive though.
1091 *
1092 * At the moment, it's VITAL that it's done AFTER the instruction interpreting
1093 * because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
1094 */
1095 PGM_INVL_VCPU_TLBS(pVCpu);
1096#endif
1097
1098 LogFlow(("pgmPoolAccessHandlerSimple: returns %Rrc cb=%d\n", rc, cb));
1099 return rc;
1100}
1101
1102/**
1103 * \#PF Handler callback for PT write accesses.
1104 *
1105 * @returns VBox status code (appropriate for GC return).
1106 * @param pVM VM Handle.
1107 * @param uErrorCode CPU Error code.
1108 * @param pRegFrame Trap register frame.
1109 * NULL on DMA and other non CPU access.
1110 * @param pvFault The fault address (cr2).
1111 * @param GCPhysFault The GC physical address corresponding to pvFault.
1112 * @param pvUser User argument.
1113 */
1114DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1115{
1116 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1117 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1118 PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1119 PVMCPU pVCpu = VMMGetCpu(pVM);
1120 unsigned cMaxModifications;
1121 bool fForcedFlush = false;
1122
1123 LogFlow(("pgmPoolAccessHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1124
1125 pgmLock(pVM);
1126 if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1127 {
1128 /* Pool page changed while we were waiting for the lock; ignore. */
1129 Log(("CPU%d: pgmPoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1130 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1131 pgmUnlock(pVM);
1132 return VINF_SUCCESS;
1133 }
1134
1135 /*
1136 * Disassemble the faulting instruction.
1137 */
1138 PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
1139 int rc = EMInterpretDisasOne(pVM, pVCpu, pRegFrame, pDis, NULL);
1140 AssertReturnStmt(rc == VINF_SUCCESS, pgmUnlock(pVM), rc);
1141
1142 Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1143
1144 /*
1145 * We should ALWAYS have the list head as user parameter. This
1146 * is because we use that page to record the changes.
1147 */
1148 Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1149#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1150 Assert(!pPage->fDirty);
1151#endif
1152
1153 /* Maximum nr of modifications depends on the guest mode. */
1154 if (pDis->mode == CPUMODE_32BIT)
1155 cMaxModifications = 48;
1156 else
1157 cMaxModifications = 24;
1158
1159 /*
1160 * Incremental page table updates should weight more than random ones.
1161 * (Only applies when started from offset 0)
1162 */
1163 pVCpu->pgm.s.cPoolAccessHandler++;
1164 if ( pPage->pvLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1165 && pPage->pvLastAccessHandlerRip < pRegFrame->rip + 0x40
1166 && pvFault == (pPage->pvLastAccessHandlerFault + pDis->param1.size)
1167 && pVCpu->pgm.s.cPoolAccessHandler == (pPage->cLastAccessHandlerCount + 1))
1168 {
1169 Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1170 pPage->cModifications = pPage->cModifications * 2;
1171 pPage->pvLastAccessHandlerFault = pvFault;
1172 pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1173 if (pPage->cModifications > cMaxModifications)
1174 {
1175 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FlushReinit));
1176 fForcedFlush = true;
1177 }
1178 }
1179
1180 if (pPage->cModifications >= cMaxModifications)
1181 Log(("Mod overflow %VGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1182
1183 /*
1184 * Check if it's worth dealing with.
1185 */
1186 bool fReused = false;
1187 if ( ( pPage->cModifications < cMaxModifications /** @todo #define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */
1188 || pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
1189 )
1190 && !(fReused = pgmPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault))
1191 && !pgmPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1192 {
1193 /*
1194 * Simple instructions, no REP prefix.
1195 */
1196 if (!(pDis->prefix & (PREFIX_REP | PREFIX_REPNE)))
1197 {
1198 rc = pgmPoolAccessHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1199
1200 /* A mov instruction to change the first page table entry will be remembered so we can detect
1201 * full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1202 */
1203 if ( rc == VINF_SUCCESS
1204 && pDis->pCurInstr->opcode == OP_MOV
1205 && (pvFault & PAGE_OFFSET_MASK) == 0)
1206 {
1207 pPage->pvLastAccessHandlerFault = pvFault;
1208 pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1209 pPage->pvLastAccessHandlerRip = pRegFrame->rip;
1210 /* Make sure we don't kick out a page too quickly. */
1211 if (pPage->cModifications > 8)
1212 pPage->cModifications = 2;
1213 }
1214 else
1215 if (pPage->pvLastAccessHandlerFault == pvFault)
1216 {
1217 /* ignore the 2nd write to this page table entry. */
1218 pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1219 }
1220 else
1221 {
1222 pPage->pvLastAccessHandlerFault = 0;
1223 pPage->pvLastAccessHandlerRip = 0;
1224 }
1225
1226 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1227 pgmUnlock(pVM);
1228 return rc;
1229 }
1230
1231 /*
1232 * Windows is frequently doing small memset() operations (netio test 4k+).
1233 * We have to deal with these or we'll kill the cache and performance.
1234 */
1235 if ( pDis->pCurInstr->opcode == OP_STOSWD
1236 && !pRegFrame->eflags.Bits.u1DF
1237 && pDis->opmode == pDis->mode
1238 && pDis->addrmode == pDis->mode)
1239 {
1240 bool fValidStosd = false;
1241
1242 if ( pDis->mode == CPUMODE_32BIT
1243 && pDis->prefix == PREFIX_REP
1244 && pRegFrame->ecx <= 0x20
1245 && pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1246 && !((uintptr_t)pvFault & 3)
1247 && (pRegFrame->eax == 0 || pRegFrame->eax == 0x80) /* the two values observed. */
1248 )
1249 {
1250 fValidStosd = true;
1251 pRegFrame->rcx &= 0xffffffff; /* paranoia */
1252 }
1253 else
1254 if ( pDis->mode == CPUMODE_64BIT
1255 && pDis->prefix == (PREFIX_REP | PREFIX_REX)
1256 && pRegFrame->rcx <= 0x20
1257 && pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1258 && !((uintptr_t)pvFault & 7)
1259 && (pRegFrame->rax == 0 || pRegFrame->rax == 0x80) /* the two values observed. */
1260 )
1261 {
1262 fValidStosd = true;
1263 }
1264
1265 if (fValidStosd)
1266 {
1267 rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1268 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,RepStosd), a);
1269 pgmUnlock(pVM);
1270 return rc;
1271 }
1272 }
1273
1274 /* REP prefix, don't bother. */
1275 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,RepPrefix));
1276 Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1277 pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->opcode, pDis->prefix));
1278 }
1279
1280#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1281 /* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1282 * leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1283 */
1284 if ( !fReused
1285 && !fForcedFlush
1286 && pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1287 && pPage->cModifications >= cMaxModifications)
1288 {
1289 Assert(!pgmPoolIsPageLocked(&pVM->pgm.s, pPage));
1290 Assert(pPage->fDirty == false);
1291
1292 /* Flush any monitored duplicates as we will disable write protection. */
1293 if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1294 || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1295 {
1296 PPGMPOOLPAGE pPageHead = pPage;
1297
1298 /* Find the monitor head. */
1299 while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1300 pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1301
1302 while (pPageHead)
1303 {
1304 unsigned idxNext = pPageHead->iMonitoredNext;
1305
1306 if (pPageHead != pPage)
1307 {
1308 STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1309 Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1310 int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1311 AssertRC(rc2);
1312 }
1313
1314 if (idxNext == NIL_PGMPOOL_IDX)
1315 break;
1316
1317 pPageHead = &pPool->aPages[idxNext];
1318 }
1319 }
1320
1321 /* Temporarily allow write access to the page table again. */
1322 rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys, pPage->GCPhys);
1323 if (rc == VINF_SUCCESS)
1324 {
1325 rc = PGMShwModifyPage(pVCpu, pvFault, 1, X86_PTE_RW, ~(uint64_t)X86_PTE_RW);
1326 AssertMsg(rc == VINF_SUCCESS
1327 /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1328 || rc == VERR_PAGE_TABLE_NOT_PRESENT
1329 || rc == VERR_PAGE_NOT_PRESENT,
1330 ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1331
1332 pgmPoolAddDirtyPage(pVM, pPool, pPage);
1333
1334 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1335 pgmUnlock(pVM);
1336 return rc;
1337 }
1338 }
1339#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1340
1341 /*
1342 * Not worth it, so flush it.
1343 *
1344 * If we considered it to be reused, don't go back to ring-3
1345 * to emulate failed instructions since we usually cannot
1346 * interpret then. This may be a bit risky, in which case
1347 * the reuse detection must be fixed.
1348 */
1349 rc = pgmPoolAccessHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1350 if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
1351 rc = VINF_SUCCESS;
1352 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1353 pgmUnlock(pVM);
1354 return rc;
1355}
1356
1357# endif /* !IN_RING3 */
1358
1359# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1360/**
1361 * Clear references to guest physical memory in a PAE / PAE page table.
1362 *
1363 * @returns nr of changed PTEs
1364 * @param pPool The pool.
1365 * @param pPage The page.
1366 * @param pShwPT The shadow page table (mapping of the page).
1367 * @param pGstPT The guest page table.
1368 * @param pGstPT The old cached guest page table.
1369 */
1370DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT, PCX86PTPAE pOldGstPT)
1371{
1372 unsigned cChanged = 0;
1373
1374 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
1375 {
1376 if (pShwPT->a[i].n.u1Present)
1377 {
1378 /* The the old cached PTE is identical, then there's no need to flush the shadow copy. */
1379 if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1380 {
1381#ifdef VBOX_STRICT
1382 RTHCPHYS HCPhys = -1;
1383 int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1384 AssertMsg(rc == VINF_SUCCESS && (pShwPT->a[i].u & X86_PTE_PAE_PG_MASK) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, pShwPT->a[i].u, HCPhys));
1385#endif
1386 uint64_t uHostAttr = pShwPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
1387 bool fHostRW = !!(pShwPT->a[i].u & X86_PTE_RW);
1388 uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
1389 bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1390
1391 if ( uHostAttr == uGuestAttr
1392 && fHostRW <= fGuestRW)
1393 continue;
1394 }
1395 cChanged++;
1396 /* Something was changed, so flush it. */
1397 Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
1398 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1399 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
1400 ASMAtomicWriteSize(&pShwPT->a[i].u, 0);
1401 }
1402 }
1403 return cChanged;
1404}
1405
1406
1407/**
1408 * Flush a dirty page
1409 *
1410 * @param pVM VM Handle.
1411 * @param pPool The pool.
1412 * @param idxSlot Dirty array slot index
1413 * @param fForceRemoval Force removal from the dirty page list
1414 */
1415static void pgmPoolFlushDirtyPage(PVM pVM, PPGMPOOL pPool, unsigned idxSlot, bool fForceRemoval = false)
1416{
1417 PPGMPOOLPAGE pPage;
1418 unsigned idxPage;
1419
1420 Assert(idxSlot < RT_ELEMENTS(pPool->aIdxDirtyPages));
1421 if (pPool->aIdxDirtyPages[idxSlot] == NIL_PGMPOOL_IDX)
1422 return;
1423
1424 idxPage = pPool->aIdxDirtyPages[idxSlot];
1425 AssertRelease(idxPage != NIL_PGMPOOL_IDX);
1426 pPage = &pPool->aPages[idxPage];
1427 Assert(pPage->idx == idxPage);
1428 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1429
1430 AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1431 Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1432
1433 /* Flush those PTEs that have changed. */
1434 STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1435 void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
1436 void *pvGst;
1437 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1438 unsigned cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst, (PCX86PTPAE)&pPool->aDirtyPages[idxSlot][0]);
1439 STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1440
1441 /** Note: we might want to consider keeping the dirty page active in case there were many changes. */
1442
1443 /* Write protect the page again to catch all write accesses. */
1444 rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys);
1445 Assert(rc == VINF_SUCCESS);
1446 pPage->fDirty = false;
1447
1448 /* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1449 Assert(pPage->cModifications);
1450 if (cChanges < 4)
1451 pPage->cModifications = 1; /* must use > 0 here */
1452 else
1453 pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1454
1455 STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1456 if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages))
1457 pPool->idxFreeDirtyPage = idxSlot;
1458
1459 pPool->cDirtyPages--;
1460 pPool->aIdxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1461 Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aIdxDirtyPages));
1462 Log(("Removed dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1463}
1464
1465/**
1466 * Add a new dirty page
1467 *
1468 * @param pVM VM Handle.
1469 * @param pPool The pool.
1470 * @param pPage The page.
1471 */
1472void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1473{
1474 unsigned idxFree;
1475
1476 Assert(PGMIsLocked(pVM));
1477 AssertCompile(RT_ELEMENTS(pPool->aIdxDirtyPages) == 8 || RT_ELEMENTS(pPool->aIdxDirtyPages) == 16);
1478
1479 if (pPage->fDirty)
1480 return;
1481
1482 idxFree = pPool->idxFreeDirtyPage;
1483 Assert(idxFree < RT_ELEMENTS(pPool->aIdxDirtyPages));
1484 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1485
1486 if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aIdxDirtyPages))
1487 pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* force removal */);
1488 Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aIdxDirtyPages));
1489 AssertMsg(pPool->aIdxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1490
1491 /* Make a copy of the guest page table as we require valid GCPhys addresses when removing
1492 * references to physical pages. (the HCPhys linear lookup is *extremely* expensive!)
1493 */
1494 void *pvGst;
1495 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1496 memcpy(&pPool->aDirtyPages[idxFree][0], pvGst, PAGE_SIZE);
1497
1498 STAM_COUNTER_INC(&pPool->StatDirtyPage);
1499 Log(("Mark dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1500 pPage->fDirty = true;
1501 pPage->idxDirty = idxFree;
1502 pPool->aIdxDirtyPages[idxFree] = pPage->idx;
1503 pPool->cDirtyPages++;
1504
1505 pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aIdxDirtyPages) - 1);
1506 if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aIdxDirtyPages)
1507 && pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1508 {
1509 unsigned i;
1510 for (i = 1; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1511 {
1512 idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aIdxDirtyPages) - 1);
1513 if (pPool->aIdxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1514 {
1515 pPool->idxFreeDirtyPage = idxFree;
1516 break;
1517 }
1518 }
1519 Assert(i != RT_ELEMENTS(pPool->aIdxDirtyPages));
1520 }
1521
1522 Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages) || pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1523 return;
1524}
1525
1526
1527/**
1528 * Reset all dirty pages by reinstating page monitoring.
1529 *
1530 * @param pVM VM Handle.
1531 * @param fForceRemoval Force removal of all dirty pages
1532 */
1533void pgmPoolResetDirtyPages(PVM pVM, bool fForceRemoval)
1534{
1535 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1536 Assert(PGMIsLocked(pVM));
1537 Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aIdxDirtyPages));
1538
1539 if (!pPool->cDirtyPages)
1540 return;
1541
1542 Log(("pgmPoolResetDirtyPages\n"));
1543 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1544 pgmPoolFlushDirtyPage(pVM, pPool, i, fForceRemoval);
1545
1546 pPool->idxFreeDirtyPage = 0;
1547 if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aIdxDirtyPages)
1548 && pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1549 {
1550 unsigned i;
1551 for (i = 1; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1552 {
1553 if (pPool->aIdxDirtyPages[i] == NIL_PGMPOOL_IDX)
1554 {
1555 pPool->idxFreeDirtyPage = i;
1556 break;
1557 }
1558 }
1559 AssertMsg(i != RT_ELEMENTS(pPool->aIdxDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
1560 }
1561
1562 Assert(pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX || pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages));
1563 return;
1564}
1565# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1566#endif /* PGMPOOL_WITH_MONITORING */
1567
1568#ifdef PGMPOOL_WITH_CACHE
1569
1570/**
1571 * Inserts a page into the GCPhys hash table.
1572 *
1573 * @param pPool The pool.
1574 * @param pPage The page.
1575 */
1576DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1577{
1578 Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1579 Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1580 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1581 pPage->iNext = pPool->aiHash[iHash];
1582 pPool->aiHash[iHash] = pPage->idx;
1583}
1584
1585
1586/**
1587 * Removes a page from the GCPhys hash table.
1588 *
1589 * @param pPool The pool.
1590 * @param pPage The page.
1591 */
1592DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1593{
1594 Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1595 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1596 if (pPool->aiHash[iHash] == pPage->idx)
1597 pPool->aiHash[iHash] = pPage->iNext;
1598 else
1599 {
1600 uint16_t iPrev = pPool->aiHash[iHash];
1601 for (;;)
1602 {
1603 const int16_t i = pPool->aPages[iPrev].iNext;
1604 if (i == pPage->idx)
1605 {
1606 pPool->aPages[iPrev].iNext = pPage->iNext;
1607 break;
1608 }
1609 if (i == NIL_PGMPOOL_IDX)
1610 {
1611 AssertReleaseMsgFailed(("GCPhys=%RGp idx=%#x\n", pPage->GCPhys, pPage->idx));
1612 break;
1613 }
1614 iPrev = i;
1615 }
1616 }
1617 pPage->iNext = NIL_PGMPOOL_IDX;
1618}
1619
1620
1621/**
1622 * Frees up one cache page.
1623 *
1624 * @returns VBox status code.
1625 * @retval VINF_SUCCESS on success.
1626 * @param pPool The pool.
1627 * @param iUser The user index.
1628 */
1629static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1630{
1631#ifndef IN_RC
1632 const PVM pVM = pPool->CTX_SUFF(pVM);
1633#endif
1634 Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1635 STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1636
1637 /*
1638 * Select one page from the tail of the age list.
1639 */
1640 PPGMPOOLPAGE pPage;
1641 for (unsigned iLoop = 0; ; iLoop++)
1642 {
1643 uint16_t iToFree = pPool->iAgeTail;
1644 if (iToFree == iUser)
1645 iToFree = pPool->aPages[iToFree].iAgePrev;
1646/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
1647 if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
1648 {
1649 uint16_t i = pPool->aPages[iToFree].iAgePrev;
1650 for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
1651 {
1652 if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
1653 continue;
1654 iToFree = i;
1655 break;
1656 }
1657 }
1658*/
1659 Assert(iToFree != iUser);
1660 AssertRelease(iToFree != NIL_PGMPOOL_IDX);
1661 pPage = &pPool->aPages[iToFree];
1662
1663 /*
1664 * Reject any attempts at flushing the currently active shadow CR3 mapping.
1665 * Call pgmPoolCacheUsed to move the page to the head of the age list.
1666 */
1667 if (!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage))
1668 break;
1669 LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
1670 pgmPoolCacheUsed(pPool, pPage);
1671 AssertLogRelReturn(iLoop < 8192, VERR_INTERNAL_ERROR);
1672 }
1673
1674 /*
1675 * Found a usable page, flush it and return.
1676 */
1677 int rc = pgmPoolFlushPage(pPool, pPage);
1678 /* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
1679 if (rc == VINF_SUCCESS)
1680 PGM_INVL_ALL_VCPU_TLBS(pVM);
1681 return rc;
1682}
1683
1684
1685/**
1686 * Checks if a kind mismatch is really a page being reused
1687 * or if it's just normal remappings.
1688 *
1689 * @returns true if reused and the cached page (enmKind1) should be flushed
1690 * @returns false if not reused.
1691 * @param enmKind1 The kind of the cached page.
1692 * @param enmKind2 The kind of the requested page.
1693 */
1694static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
1695{
1696 switch (enmKind1)
1697 {
1698 /*
1699 * Never reuse them. There is no remapping in non-paging mode.
1700 */
1701 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1702 case PGMPOOLKIND_32BIT_PD_PHYS:
1703 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1704 case PGMPOOLKIND_PAE_PD_PHYS:
1705 case PGMPOOLKIND_PAE_PDPT_PHYS:
1706 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1707 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1708 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1709 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1710 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1711 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
1712 return false;
1713
1714 /*
1715 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1716 */
1717 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1718 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1719 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1720 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1721 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1722 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1723 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1724 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1725 case PGMPOOLKIND_32BIT_PD:
1726 case PGMPOOLKIND_PAE_PDPT:
1727 switch (enmKind2)
1728 {
1729 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1730 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1731 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1732 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1733 case PGMPOOLKIND_64BIT_PML4:
1734 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1735 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1736 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1737 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1738 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1739 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1740 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1741 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1742 return true;
1743 default:
1744 return false;
1745 }
1746
1747 /*
1748 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1749 */
1750 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1751 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1752 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1753 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1754 case PGMPOOLKIND_64BIT_PML4:
1755 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1756 switch (enmKind2)
1757 {
1758 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1759 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1760 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1761 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1762 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1763 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1764 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1765 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1766 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1767 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1768 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1769 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1770 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1771 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1772 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1773 return true;
1774 default:
1775 return false;
1776 }
1777
1778 /*
1779 * These cannot be flushed, and it's common to reuse the PDs as PTs.
1780 */
1781 case PGMPOOLKIND_ROOT_NESTED:
1782 return false;
1783
1784 default:
1785 AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1786 }
1787}
1788
1789
1790/**
1791 * Attempts to satisfy a pgmPoolAlloc request from the cache.
1792 *
1793 * @returns VBox status code.
1794 * @retval VINF_PGM_CACHED_PAGE on success.
1795 * @retval VERR_FILE_NOT_FOUND if not found.
1796 * @param pPool The pool.
1797 * @param GCPhys The GC physical address of the page we're gonna shadow.
1798 * @param enmKind The kind of mapping.
1799 * @param enmAccess Access type for the mapping (only relevant for big pages)
1800 * @param iUser The shadow page pool index of the user table.
1801 * @param iUserTable The index into the user table (shadowed).
1802 * @param ppPage Where to store the pointer to the page.
1803 */
1804static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
1805{
1806#ifndef IN_RC
1807 const PVM pVM = pPool->CTX_SUFF(pVM);
1808#endif
1809 /*
1810 * Look up the GCPhys in the hash.
1811 */
1812 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1813 Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%x iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
1814 if (i != NIL_PGMPOOL_IDX)
1815 {
1816 do
1817 {
1818 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1819 Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
1820 if (pPage->GCPhys == GCPhys)
1821 {
1822 if ( (PGMPOOLKIND)pPage->enmKind == enmKind
1823 && (PGMPOOLACCESS)pPage->enmAccess == enmAccess)
1824 {
1825 /* Put it at the start of the use list to make sure pgmPoolTrackAddUser
1826 * doesn't flush it in case there are no more free use records.
1827 */
1828 pgmPoolCacheUsed(pPool, pPage);
1829
1830 int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1831 if (RT_SUCCESS(rc))
1832 {
1833 Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
1834 *ppPage = pPage;
1835 if (pPage->cModifications)
1836 pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
1837 STAM_COUNTER_INC(&pPool->StatCacheHits);
1838 return VINF_PGM_CACHED_PAGE;
1839 }
1840 return rc;
1841 }
1842
1843 if ((PGMPOOLKIND)pPage->enmKind != enmKind)
1844 {
1845 /*
1846 * The kind is different. In some cases we should now flush the page
1847 * as it has been reused, but in most cases this is normal remapping
1848 * of PDs as PT or big pages using the GCPhys field in a slightly
1849 * different way than the other kinds.
1850 */
1851 if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1852 {
1853 STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1854 pgmPoolFlushPage(pPool, pPage);
1855 PGM_INVL_VCPU_TLBS(VMMGetCpu(pVM)); /* see PT handler. */
1856 break;
1857 }
1858 }
1859 }
1860
1861 /* next */
1862 i = pPage->iNext;
1863 } while (i != NIL_PGMPOOL_IDX);
1864 }
1865
1866 Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
1867 STAM_COUNTER_INC(&pPool->StatCacheMisses);
1868 return VERR_FILE_NOT_FOUND;
1869}
1870
1871
1872/**
1873 * Inserts a page into the cache.
1874 *
1875 * @param pPool The pool.
1876 * @param pPage The cached page.
1877 * @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1878 */
1879static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1880{
1881 /*
1882 * Insert into the GCPhys hash if the page is fit for that.
1883 */
1884 Assert(!pPage->fCached);
1885 if (fCanBeCached)
1886 {
1887 pPage->fCached = true;
1888 pgmPoolHashInsert(pPool, pPage);
1889 Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1890 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1891 STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1892 }
1893 else
1894 {
1895 Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1896 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1897 STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1898 }
1899
1900 /*
1901 * Insert at the head of the age list.
1902 */
1903 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1904 pPage->iAgeNext = pPool->iAgeHead;
1905 if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1906 pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1907 else
1908 pPool->iAgeTail = pPage->idx;
1909 pPool->iAgeHead = pPage->idx;
1910}
1911
1912
1913/**
1914 * Flushes a cached page.
1915 *
1916 * @param pPool The pool.
1917 * @param pPage The cached page.
1918 */
1919static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1920{
1921 Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
1922
1923 /*
1924 * Remove the page from the hash.
1925 */
1926 if (pPage->fCached)
1927 {
1928 pPage->fCached = false;
1929 pgmPoolHashRemove(pPool, pPage);
1930 }
1931 else
1932 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1933
1934 /*
1935 * Remove it from the age list.
1936 */
1937 if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1938 pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1939 else
1940 pPool->iAgeTail = pPage->iAgePrev;
1941 if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1942 pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1943 else
1944 pPool->iAgeHead = pPage->iAgeNext;
1945 pPage->iAgeNext = NIL_PGMPOOL_IDX;
1946 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1947}
1948
1949#endif /* PGMPOOL_WITH_CACHE */
1950#ifdef PGMPOOL_WITH_MONITORING
1951
1952/**
1953 * Looks for pages sharing the monitor.
1954 *
1955 * @returns Pointer to the head page.
1956 * @returns NULL if not found.
1957 * @param pPool The Pool
1958 * @param pNewPage The page which is going to be monitored.
1959 */
1960static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1961{
1962#ifdef PGMPOOL_WITH_CACHE
1963 /*
1964 * Look up the GCPhys in the hash.
1965 */
1966 RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1967 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1968 if (i == NIL_PGMPOOL_IDX)
1969 return NULL;
1970 do
1971 {
1972 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1973 if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1974 && pPage != pNewPage)
1975 {
1976 switch (pPage->enmKind)
1977 {
1978 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1979 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1980 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1981 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1982 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1983 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1984 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1985 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1986 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1987 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1988 case PGMPOOLKIND_64BIT_PML4:
1989 case PGMPOOLKIND_32BIT_PD:
1990 case PGMPOOLKIND_PAE_PDPT:
1991 {
1992 /* find the head */
1993 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1994 {
1995 Assert(pPage->iMonitoredPrev != pPage->idx);
1996 pPage = &pPool->aPages[pPage->iMonitoredPrev];
1997 }
1998 return pPage;
1999 }
2000
2001 /* ignore, no monitoring. */
2002 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2003 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2004 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2005 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2006 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2007 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2008 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2009 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2010 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2011 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2012 case PGMPOOLKIND_ROOT_NESTED:
2013 case PGMPOOLKIND_PAE_PD_PHYS:
2014 case PGMPOOLKIND_PAE_PDPT_PHYS:
2015 case PGMPOOLKIND_32BIT_PD_PHYS:
2016 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2017 break;
2018 default:
2019 AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2020 }
2021 }
2022
2023 /* next */
2024 i = pPage->iNext;
2025 } while (i != NIL_PGMPOOL_IDX);
2026#endif
2027 return NULL;
2028}
2029
2030
2031/**
2032 * Enabled write monitoring of a guest page.
2033 *
2034 * @returns VBox status code.
2035 * @retval VINF_SUCCESS on success.
2036 * @param pPool The pool.
2037 * @param pPage The cached page.
2038 */
2039static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2040{
2041 LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
2042
2043 /*
2044 * Filter out the relevant kinds.
2045 */
2046 switch (pPage->enmKind)
2047 {
2048 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2049 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2050 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2051 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2052 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2053 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2054 case PGMPOOLKIND_64BIT_PML4:
2055 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2056 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2057 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2058 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2059 case PGMPOOLKIND_32BIT_PD:
2060 case PGMPOOLKIND_PAE_PDPT:
2061 break;
2062
2063 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2064 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2065 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2066 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2067 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2068 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2069 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2070 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2071 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2072 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2073 case PGMPOOLKIND_ROOT_NESTED:
2074 /* Nothing to monitor here. */
2075 return VINF_SUCCESS;
2076
2077 case PGMPOOLKIND_32BIT_PD_PHYS:
2078 case PGMPOOLKIND_PAE_PDPT_PHYS:
2079 case PGMPOOLKIND_PAE_PD_PHYS:
2080 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2081 /* Nothing to monitor here. */
2082 return VINF_SUCCESS;
2083#ifdef PGMPOOL_WITH_MIXED_PT_CR3
2084 break;
2085#else
2086 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2087#endif
2088 default:
2089 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2090 }
2091
2092 /*
2093 * Install handler.
2094 */
2095 int rc;
2096 PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2097 if (pPageHead)
2098 {
2099 Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2100 Assert(pPageHead->iMonitoredPrev != pPage->idx);
2101
2102#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2103 if (pPageHead->fDirty)
2104 pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirty, true /* force removal */);
2105#endif
2106
2107 pPage->iMonitoredPrev = pPageHead->idx;
2108 pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2109 if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2110 pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2111 pPageHead->iMonitoredNext = pPage->idx;
2112 rc = VINF_SUCCESS;
2113 }
2114 else
2115 {
2116 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2117 PVM pVM = pPool->CTX_SUFF(pVM);
2118 const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
2119 rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2120 GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
2121 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
2122 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
2123 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
2124 pPool->pszAccessHandler);
2125 /** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2126 * the heap size should suffice. */
2127 AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2128 Assert(!(VMMGetCpu(pVM)->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_ISSET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3));
2129 }
2130 pPage->fMonitored = true;
2131 return rc;
2132}
2133
2134
2135/**
2136 * Disables write monitoring of a guest page.
2137 *
2138 * @returns VBox status code.
2139 * @retval VINF_SUCCESS on success.
2140 * @param pPool The pool.
2141 * @param pPage The cached page.
2142 */
2143static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2144{
2145 /*
2146 * Filter out the relevant kinds.
2147 */
2148 switch (pPage->enmKind)
2149 {
2150 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2151 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2152 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2153 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2154 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2155 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2156 case PGMPOOLKIND_64BIT_PML4:
2157 case PGMPOOLKIND_32BIT_PD:
2158 case PGMPOOLKIND_PAE_PDPT:
2159 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2160 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2161 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2162 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2163 break;
2164
2165 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2166 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2167 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2168 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2169 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2170 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2171 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2172 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2173 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2174 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2175 case PGMPOOLKIND_ROOT_NESTED:
2176 case PGMPOOLKIND_PAE_PD_PHYS:
2177 case PGMPOOLKIND_PAE_PDPT_PHYS:
2178 case PGMPOOLKIND_32BIT_PD_PHYS:
2179 /* Nothing to monitor here. */
2180 return VINF_SUCCESS;
2181
2182#ifdef PGMPOOL_WITH_MIXED_PT_CR3
2183 break;
2184#endif
2185 default:
2186 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2187 }
2188
2189 /*
2190 * Remove the page from the monitored list or uninstall it if last.
2191 */
2192 const PVM pVM = pPool->CTX_SUFF(pVM);
2193 int rc;
2194 if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2195 || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2196 {
2197 if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2198 {
2199 PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2200 pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2201 rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
2202 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
2203 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
2204 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pNewHead),
2205 pPool->pszAccessHandler);
2206 AssertFatalRCSuccess(rc);
2207 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2208 }
2209 else
2210 {
2211 pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2212 if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2213 {
2214 pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2215 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2216 }
2217 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2218 rc = VINF_SUCCESS;
2219 }
2220 }
2221 else
2222 {
2223 rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
2224 AssertFatalRC(rc);
2225#ifdef VBOX_STRICT
2226 PVMCPU pVCpu = VMMGetCpu(pVM);
2227#endif
2228 AssertMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2229 ("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2230 }
2231 pPage->fMonitored = false;
2232
2233 /*
2234 * Remove it from the list of modified pages (if in it).
2235 */
2236 pgmPoolMonitorModifiedRemove(pPool, pPage);
2237
2238 return rc;
2239}
2240
2241
2242/**
2243 * Inserts the page into the list of modified pages.
2244 *
2245 * @param pPool The pool.
2246 * @param pPage The page.
2247 */
2248void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2249{
2250 Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2251 AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2252 && pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2253 && pPool->iModifiedHead != pPage->idx,
2254 ("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2255 pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2256 pPool->iModifiedHead, pPool->cModifiedPages));
2257
2258 pPage->iModifiedNext = pPool->iModifiedHead;
2259 if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2260 pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2261 pPool->iModifiedHead = pPage->idx;
2262 pPool->cModifiedPages++;
2263#ifdef VBOX_WITH_STATISTICS
2264 if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2265 pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2266#endif
2267}
2268
2269
2270/**
2271 * Removes the page from the list of modified pages and resets the
2272 * moficiation counter.
2273 *
2274 * @param pPool The pool.
2275 * @param pPage The page which is believed to be in the list of modified pages.
2276 */
2277static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2278{
2279 Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2280 if (pPool->iModifiedHead == pPage->idx)
2281 {
2282 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2283 pPool->iModifiedHead = pPage->iModifiedNext;
2284 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2285 {
2286 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2287 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2288 }
2289 pPool->cModifiedPages--;
2290 }
2291 else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2292 {
2293 pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2294 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2295 {
2296 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2297 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2298 }
2299 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2300 pPool->cModifiedPages--;
2301 }
2302 else
2303 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2304 pPage->cModifications = 0;
2305}
2306
2307
2308/**
2309 * Zaps the list of modified pages, resetting their modification counters in the process.
2310 *
2311 * @param pVM The VM handle.
2312 */
2313static void pgmPoolMonitorModifiedClearAll(PVM pVM)
2314{
2315 pgmLock(pVM);
2316 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2317 LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2318
2319 unsigned cPages = 0; NOREF(cPages);
2320
2321#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2322 pgmPoolResetDirtyPages(pVM, true /* force removal. */);
2323#endif
2324
2325 uint16_t idx = pPool->iModifiedHead;
2326 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2327 while (idx != NIL_PGMPOOL_IDX)
2328 {
2329 PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2330 idx = pPage->iModifiedNext;
2331 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2332 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2333 pPage->cModifications = 0;
2334 Assert(++cPages);
2335 }
2336 AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2337 pPool->cModifiedPages = 0;
2338 pgmUnlock(pVM);
2339}
2340
2341
2342#ifdef IN_RING3
2343/**
2344 * Callback to clear all shadow pages and clear all modification counters.
2345 *
2346 * @returns VBox status code.
2347 * @param pVM The VM handle.
2348 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
2349 * @param pvUser Unused parameter.
2350 *
2351 * @remark Should only be used when monitoring is available, thus placed in
2352 * the PGMPOOL_WITH_MONITORING \#ifdef.
2353 */
2354DECLCALLBACK(int) pgmPoolClearAll(PVM pVM, PVMCPU pVCpu, void *pvUser)
2355{
2356 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2357 STAM_PROFILE_START(&pPool->StatClearAll, c);
2358 LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
2359 NOREF(pvUser); NOREF(pVCpu);
2360
2361 pgmLock(pVM);
2362
2363 /*
2364 * Iterate all the pages until we've encountered all that in use.
2365 * This is simple but not quite optimal solution.
2366 */
2367 unsigned cModifiedPages = 0; NOREF(cModifiedPages);
2368 unsigned cLeft = pPool->cUsedPages;
2369 unsigned iPage = pPool->cCurPages;
2370 while (--iPage >= PGMPOOL_IDX_FIRST)
2371 {
2372 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2373 if (pPage->GCPhys != NIL_RTGCPHYS)
2374 {
2375 switch (pPage->enmKind)
2376 {
2377 /*
2378 * We only care about shadow page tables.
2379 */
2380 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2381 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2382 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2383 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2384 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2385 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2386 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2387 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2388 {
2389#ifdef PGMPOOL_WITH_USER_TRACKING
2390 if (pPage->cPresent)
2391#endif
2392 {
2393 void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
2394 STAM_PROFILE_START(&pPool->StatZeroPage, z);
2395 ASMMemZeroPage(pvShw);
2396 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
2397#ifdef PGMPOOL_WITH_USER_TRACKING
2398 pPage->cPresent = 0;
2399 pPage->iFirstPresent = ~0;
2400#endif
2401 }
2402 }
2403 /* fall thru */
2404
2405 default:
2406 Assert(!pPage->cModifications || ++cModifiedPages);
2407 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
2408 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
2409 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2410 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2411 pPage->cModifications = 0;
2412 break;
2413
2414 }
2415 if (!--cLeft)
2416 break;
2417 }
2418 }
2419
2420 /* swipe the special pages too. */
2421 for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
2422 {
2423 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2424 if (pPage->GCPhys != NIL_RTGCPHYS)
2425 {
2426 Assert(!pPage->cModifications || ++cModifiedPages);
2427 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
2428 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
2429 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2430 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2431 pPage->cModifications = 0;
2432 }
2433 }
2434
2435#ifndef DEBUG_michael
2436 AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
2437#endif
2438 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2439 pPool->cModifiedPages = 0;
2440
2441#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2442 /*
2443 * Clear all the GCPhys links and rebuild the phys ext free list.
2444 */
2445 for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
2446 pRam;
2447 pRam = pRam->CTX_SUFF(pNext))
2448 {
2449 unsigned iPage = pRam->cb >> PAGE_SHIFT;
2450 while (iPage-- > 0)
2451 PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
2452 }
2453
2454 pPool->iPhysExtFreeHead = 0;
2455 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
2456 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
2457 for (unsigned i = 0; i < cMaxPhysExts; i++)
2458 {
2459 paPhysExts[i].iNext = i + 1;
2460 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
2461 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
2462 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
2463 }
2464 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2465#endif
2466
2467#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2468 /* Clear all dirty pages. */
2469 pPool->idxFreeDirtyPage = 0;
2470 pPool->cDirtyPages = 0;
2471 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
2472 pPool->aIdxDirtyPages[i] = NIL_PGMPOOL_IDX;
2473#endif
2474
2475 /* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
2476 for (unsigned idCpu = 0; idCpu < pVM->cCPUs; idCpu++)
2477 {
2478 PVMCPU pVCpu = &pVM->aCpus[idCpu];
2479
2480 pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2481 }
2482
2483 pPool->cPresent = 0;
2484 pgmUnlock(pVM);
2485 PGM_INVL_ALL_VCPU_TLBS(pVM);
2486 STAM_PROFILE_STOP(&pPool->StatClearAll, c);
2487 return VINF_SUCCESS;
2488}
2489#endif /* IN_RING3 */
2490
2491
2492/**
2493 * Handle SyncCR3 pool tasks
2494 *
2495 * @returns VBox status code.
2496 * @retval VINF_SUCCESS if successfully added.
2497 * @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2498 * @param pVCpu The VMCPU handle.
2499 * @remark Should only be used when monitoring is available, thus placed in
2500 * the PGMPOOL_WITH_MONITORING #ifdef.
2501 */
2502int pgmPoolSyncCR3(PVMCPU pVCpu)
2503{
2504 PVM pVM = pVCpu->CTX_SUFF(pVM);
2505 LogFlow(("pgmPoolSyncCR3\n"));
2506
2507 /*
2508 * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2509 * Occasionally we will have to clear all the shadow page tables because we wanted
2510 * to monitor a page which was mapped by too many shadowed page tables. This operation
2511 * sometimes refered to as a 'lightweight flush'.
2512 */
2513# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2514 if (ASMBitTestAndClear(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_CLEAR_PGM_POOL_BIT))
2515 {
2516 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmPoolClearAll, NULL);
2517 AssertRC(rc);
2518 }
2519# else /* !IN_RING3 */
2520 if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2521 {
2522 LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2523 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2524 return VINF_PGM_SYNC_CR3;
2525 }
2526# endif /* !IN_RING3 */
2527 else
2528 pgmPoolMonitorModifiedClearAll(pVM);
2529
2530 return VINF_SUCCESS;
2531}
2532
2533#endif /* PGMPOOL_WITH_MONITORING */
2534#ifdef PGMPOOL_WITH_USER_TRACKING
2535
2536/**
2537 * Frees up at least one user entry.
2538 *
2539 * @returns VBox status code.
2540 * @retval VINF_SUCCESS if successfully added.
2541 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2542 * @param pPool The pool.
2543 * @param iUser The user index.
2544 */
2545static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2546{
2547 STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2548#ifdef PGMPOOL_WITH_CACHE
2549 /*
2550 * Just free cached pages in a braindead fashion.
2551 */
2552 /** @todo walk the age list backwards and free the first with usage. */
2553 int rc = VINF_SUCCESS;
2554 do
2555 {
2556 int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2557 if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2558 rc = rc2;
2559 } while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2560 return rc;
2561#else
2562 /*
2563 * Lazy approach.
2564 */
2565 /* @todo This path no longer works (CR3 root pages will be flushed)!! */
2566 AssertCompileFailed();
2567 Assert(!CPUMIsGuestInLongMode(pVM));
2568 pgmPoolFlushAllInt(pPool);
2569 return VERR_PGM_POOL_FLUSHED;
2570#endif
2571}
2572
2573
2574/**
2575 * Inserts a page into the cache.
2576 *
2577 * This will create user node for the page, insert it into the GCPhys
2578 * hash, and insert it into the age list.
2579 *
2580 * @returns VBox status code.
2581 * @retval VINF_SUCCESS if successfully added.
2582 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2583 * @param pPool The pool.
2584 * @param pPage The cached page.
2585 * @param GCPhys The GC physical address of the page we're gonna shadow.
2586 * @param iUser The user index.
2587 * @param iUserTable The user table index.
2588 */
2589DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2590{
2591 int rc = VINF_SUCCESS;
2592 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2593
2594 LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser %x iUserTable %x\n", GCPhys, iUser, iUserTable));
2595
2596#ifdef VBOX_STRICT
2597 /*
2598 * Check that the entry doesn't already exists.
2599 */
2600 if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2601 {
2602 uint16_t i = pPage->iUserHead;
2603 do
2604 {
2605 Assert(i < pPool->cMaxUsers);
2606 AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2607 i = paUsers[i].iNext;
2608 } while (i != NIL_PGMPOOL_USER_INDEX);
2609 }
2610#endif
2611
2612 /*
2613 * Find free a user node.
2614 */
2615 uint16_t i = pPool->iUserFreeHead;
2616 if (i == NIL_PGMPOOL_USER_INDEX)
2617 {
2618 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2619 if (RT_FAILURE(rc))
2620 return rc;
2621 i = pPool->iUserFreeHead;
2622 }
2623
2624 /*
2625 * Unlink the user node from the free list,
2626 * initialize and insert it into the user list.
2627 */
2628 pPool->iUserFreeHead = paUsers[i].iNext;
2629 paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2630 paUsers[i].iUser = iUser;
2631 paUsers[i].iUserTable = iUserTable;
2632 pPage->iUserHead = i;
2633
2634 /*
2635 * Insert into cache and enable monitoring of the guest page if enabled.
2636 *
2637 * Until we implement caching of all levels, including the CR3 one, we'll
2638 * have to make sure we don't try monitor & cache any recursive reuse of
2639 * a monitored CR3 page. Because all windows versions are doing this we'll
2640 * have to be able to do combined access monitoring, CR3 + PT and
2641 * PD + PT (guest PAE).
2642 *
2643 * Update:
2644 * We're now cooperating with the CR3 monitor if an uncachable page is found.
2645 */
2646#if defined(PGMPOOL_WITH_MONITORING) || defined(PGMPOOL_WITH_CACHE)
2647# ifdef PGMPOOL_WITH_MIXED_PT_CR3
2648 const bool fCanBeMonitored = true;
2649# else
2650 bool fCanBeMonitored = pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
2651 || (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
2652 || pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
2653# endif
2654# ifdef PGMPOOL_WITH_CACHE
2655 pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2656# endif
2657 if (fCanBeMonitored)
2658 {
2659# ifdef PGMPOOL_WITH_MONITORING
2660 rc = pgmPoolMonitorInsert(pPool, pPage);
2661 AssertRC(rc);
2662 }
2663# endif
2664#endif /* PGMPOOL_WITH_MONITORING */
2665 return rc;
2666}
2667
2668
2669# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
2670/**
2671 * Adds a user reference to a page.
2672 *
2673 * This will move the page to the head of the
2674 *
2675 * @returns VBox status code.
2676 * @retval VINF_SUCCESS if successfully added.
2677 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2678 * @param pPool The pool.
2679 * @param pPage The cached page.
2680 * @param iUser The user index.
2681 * @param iUserTable The user table.
2682 */
2683static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2684{
2685 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2686
2687 Log3(("pgmPoolTrackAddUser GCPhys = %RGp iUser %x iUserTable %x\n", pPage->GCPhys, iUser, iUserTable));
2688
2689# ifdef VBOX_STRICT
2690 /*
2691 * Check that the entry doesn't already exists. We only allow multiple users of top-level paging structures (SHW_POOL_ROOT_IDX).
2692 */
2693 if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2694 {
2695 uint16_t i = pPage->iUserHead;
2696 do
2697 {
2698 Assert(i < pPool->cMaxUsers);
2699 AssertMsg(iUser != PGMPOOL_IDX_PD || iUser != PGMPOOL_IDX_PDPT || iUser != PGMPOOL_IDX_NESTED_ROOT || iUser != PGMPOOL_IDX_AMD64_CR3 ||
2700 paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2701 i = paUsers[i].iNext;
2702 } while (i != NIL_PGMPOOL_USER_INDEX);
2703 }
2704# endif
2705
2706 /*
2707 * Allocate a user node.
2708 */
2709 uint16_t i = pPool->iUserFreeHead;
2710 if (i == NIL_PGMPOOL_USER_INDEX)
2711 {
2712 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2713 if (RT_FAILURE(rc))
2714 return rc;
2715 i = pPool->iUserFreeHead;
2716 }
2717 pPool->iUserFreeHead = paUsers[i].iNext;
2718
2719 /*
2720 * Initialize the user node and insert it.
2721 */
2722 paUsers[i].iNext = pPage->iUserHead;
2723 paUsers[i].iUser = iUser;
2724 paUsers[i].iUserTable = iUserTable;
2725 pPage->iUserHead = i;
2726
2727# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2728 if (pPage->fDirty)
2729 pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirty, true /* force removal */);
2730# endif
2731
2732# ifdef PGMPOOL_WITH_CACHE
2733 /*
2734 * Tell the cache to update its replacement stats for this page.
2735 */
2736 pgmPoolCacheUsed(pPool, pPage);
2737# endif
2738 return VINF_SUCCESS;
2739}
2740# endif /* PGMPOOL_WITH_CACHE */
2741
2742
2743/**
2744 * Frees a user record associated with a page.
2745 *
2746 * This does not clear the entry in the user table, it simply replaces the
2747 * user record to the chain of free records.
2748 *
2749 * @param pPool The pool.
2750 * @param HCPhys The HC physical address of the shadow page.
2751 * @param iUser The shadow page pool index of the user table.
2752 * @param iUserTable The index into the user table (shadowed).
2753 */
2754static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2755{
2756 /*
2757 * Unlink and free the specified user entry.
2758 */
2759 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2760
2761 Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
2762 /* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2763 uint16_t i = pPage->iUserHead;
2764 if ( i != NIL_PGMPOOL_USER_INDEX
2765 && paUsers[i].iUser == iUser
2766 && paUsers[i].iUserTable == iUserTable)
2767 {
2768 pPage->iUserHead = paUsers[i].iNext;
2769
2770 paUsers[i].iUser = NIL_PGMPOOL_IDX;
2771 paUsers[i].iNext = pPool->iUserFreeHead;
2772 pPool->iUserFreeHead = i;
2773 return;
2774 }
2775
2776 /* General: Linear search. */
2777 uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2778 while (i != NIL_PGMPOOL_USER_INDEX)
2779 {
2780 if ( paUsers[i].iUser == iUser
2781 && paUsers[i].iUserTable == iUserTable)
2782 {
2783 if (iPrev != NIL_PGMPOOL_USER_INDEX)
2784 paUsers[iPrev].iNext = paUsers[i].iNext;
2785 else
2786 pPage->iUserHead = paUsers[i].iNext;
2787
2788 paUsers[i].iUser = NIL_PGMPOOL_IDX;
2789 paUsers[i].iNext = pPool->iUserFreeHead;
2790 pPool->iUserFreeHead = i;
2791 return;
2792 }
2793 iPrev = i;
2794 i = paUsers[i].iNext;
2795 }
2796
2797 /* Fatal: didn't find it */
2798 AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%RGp\n",
2799 iUser, iUserTable, pPage->GCPhys));
2800}
2801
2802
2803/**
2804 * Gets the entry size of a shadow table.
2805 *
2806 * @param enmKind The kind of page.
2807 *
2808 * @returns The size of the entry in bytes. That is, 4 or 8.
2809 * @returns If the kind is not for a table, an assertion is raised and 0 is
2810 * returned.
2811 */
2812DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2813{
2814 switch (enmKind)
2815 {
2816 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2817 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2818 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2819 case PGMPOOLKIND_32BIT_PD:
2820 case PGMPOOLKIND_32BIT_PD_PHYS:
2821 return 4;
2822
2823 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2824 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2825 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2826 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2827 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2828 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2829 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2830 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2831 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2832 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2833 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2834 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2835 case PGMPOOLKIND_64BIT_PML4:
2836 case PGMPOOLKIND_PAE_PDPT:
2837 case PGMPOOLKIND_ROOT_NESTED:
2838 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2839 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2840 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2841 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2842 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2843 case PGMPOOLKIND_PAE_PD_PHYS:
2844 case PGMPOOLKIND_PAE_PDPT_PHYS:
2845 return 8;
2846
2847 default:
2848 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2849 }
2850}
2851
2852
2853/**
2854 * Gets the entry size of a guest table.
2855 *
2856 * @param enmKind The kind of page.
2857 *
2858 * @returns The size of the entry in bytes. That is, 0, 4 or 8.
2859 * @returns If the kind is not for a table, an assertion is raised and 0 is
2860 * returned.
2861 */
2862DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2863{
2864 switch (enmKind)
2865 {
2866 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2867 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2868 case PGMPOOLKIND_32BIT_PD:
2869 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2870 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2871 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2872 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2873 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2874 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2875 return 4;
2876
2877 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2878 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2879 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2880 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2881 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2882 case PGMPOOLKIND_64BIT_PML4:
2883 case PGMPOOLKIND_PAE_PDPT:
2884 return 8;
2885
2886 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2887 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2888 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2889 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2890 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2891 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2892 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2893 case PGMPOOLKIND_ROOT_NESTED:
2894 case PGMPOOLKIND_PAE_PD_PHYS:
2895 case PGMPOOLKIND_PAE_PDPT_PHYS:
2896 case PGMPOOLKIND_32BIT_PD_PHYS:
2897 /** @todo can we return 0? (nobody is calling this...) */
2898 AssertFailed();
2899 return 0;
2900
2901 default:
2902 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2903 }
2904}
2905
2906#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2907
2908/**
2909 * Scans one shadow page table for mappings of a physical page.
2910 *
2911 * @param pVM The VM handle.
2912 * @param pPhysPage The guest page in question.
2913 * @param iShw The shadow page table.
2914 * @param cRefs The number of references made in that PT.
2915 */
2916static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2917{
2918 LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
2919 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2920
2921 /*
2922 * Assert sanity.
2923 */
2924 Assert(cRefs == 1);
2925 AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2926 PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2927
2928 /*
2929 * Then, clear the actual mappings to the page in the shadow PT.
2930 */
2931 switch (pPage->enmKind)
2932 {
2933 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2934 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2935 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2936 {
2937 const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2938 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2939 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2940 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2941 {
2942 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2943 pPT->a[i].u = 0;
2944 cRefs--;
2945 if (!cRefs)
2946 return;
2947 }
2948#ifdef LOG_ENABLED
2949 Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2950 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2951 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2952 {
2953 Log(("i=%d cRefs=%d\n", i, cRefs--));
2954 }
2955#endif
2956 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2957 break;
2958 }
2959
2960 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2961 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2962 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2963 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2964 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2965 {
2966 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2967 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2968 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2969 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2970 {
2971 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2972 pPT->a[i].u = 0;
2973 cRefs--;
2974 if (!cRefs)
2975 return;
2976 }
2977#ifdef LOG_ENABLED
2978 Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2979 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2980 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2981 {
2982 Log(("i=%d cRefs=%d\n", i, cRefs--));
2983 }
2984#endif
2985 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d u64=%RX64\n", cRefs, pPage->iFirstPresent, pPage->cPresent, u64));
2986 break;
2987 }
2988
2989 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2990 {
2991 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2992 PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2993 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2994 if ((pPT->a[i].u & (EPT_PTE_PG_MASK | X86_PTE_P)) == u64)
2995 {
2996 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2997 pPT->a[i].u = 0;
2998 cRefs--;
2999 if (!cRefs)
3000 return;
3001 }
3002#ifdef LOG_ENABLED
3003 Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3004 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3005 if ((pPT->a[i].u & (EPT_PTE_PG_MASK | X86_PTE_P)) == u64)
3006 {
3007 Log(("i=%d cRefs=%d\n", i, cRefs--));
3008 }
3009#endif
3010 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3011 break;
3012 }
3013
3014 default:
3015 AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3016 }
3017}
3018
3019
3020/**
3021 * Scans one shadow page table for mappings of a physical page.
3022 *
3023 * @param pVM The VM handle.
3024 * @param pPhysPage The guest page in question.
3025 * @param iShw The shadow page table.
3026 * @param cRefs The number of references made in that PT.
3027 */
3028void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
3029{
3030 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3031 LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
3032 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3033 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
3034 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3035 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3036}
3037
3038
3039/**
3040 * Flushes a list of shadow page tables mapping the same physical page.
3041 *
3042 * @param pVM The VM handle.
3043 * @param pPhysPage The guest page in question.
3044 * @param iPhysExt The physical cross reference extent list to flush.
3045 */
3046void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
3047{
3048 Assert(PGMIsLockOwner(pVM));
3049 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3050 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3051 LogFlow(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%R[pgmpage] iPhysExt\n", pPhysPage, iPhysExt));
3052
3053 const uint16_t iPhysExtStart = iPhysExt;
3054 PPGMPOOLPHYSEXT pPhysExt;
3055 do
3056 {
3057 Assert(iPhysExt < pPool->cMaxPhysExts);
3058 pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3059 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3060 if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3061 {
3062 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
3063 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3064 }
3065
3066 /* next */
3067 iPhysExt = pPhysExt->iNext;
3068 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3069
3070 /* insert the list into the free list and clear the ram range entry. */
3071 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3072 pPool->iPhysExtFreeHead = iPhysExtStart;
3073 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3074
3075 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3076}
3077
3078#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3079
3080/**
3081 * Flushes all shadow page table mappings of the given guest page.
3082 *
3083 * This is typically called when the host page backing the guest one has been
3084 * replaced or when the page protection was changed due to an access handler.
3085 *
3086 * @returns VBox status code.
3087 * @retval VINF_SUCCESS if all references has been successfully cleared.
3088 * @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3089 * pool cleaning. FF and sync flags are set.
3090 *
3091 * @param pVM The VM handle.
3092 * @param pPhysPage The guest page in question.
3093 * @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3094 * flushed, it is NOT touched if this isn't necessary.
3095 * The caller MUST initialized this to @a false.
3096 */
3097int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs)
3098{
3099 PVMCPU pVCpu = VMMGetCpu(pVM);
3100 pgmLock(pVM);
3101 int rc = VINF_SUCCESS;
3102#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3103 const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3104 if (u16)
3105 {
3106 /*
3107 * The zero page is currently screwing up the tracking and we'll
3108 * have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3109 * is defined, zero pages won't normally be mapped. Some kind of solution
3110 * will be needed for this problem of course, but it will have to wait...
3111 */
3112 if (PGM_PAGE_IS_ZERO(pPhysPage))
3113 rc = VINF_PGM_GCPHYS_ALIASED;
3114 else
3115 {
3116# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3117 /* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
3118 pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
3119 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3120# endif
3121
3122 if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3123 pgmPoolTrackFlushGCPhysPT(pVM,
3124 pPhysPage,
3125 PGMPOOL_TD_GET_IDX(u16),
3126 PGMPOOL_TD_GET_CREFS(u16));
3127 else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3128 pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, PGMPOOL_TD_GET_IDX(u16));
3129 else
3130 rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3131 *pfFlushTLBs = true;
3132
3133# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3134 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3135# endif
3136 }
3137 }
3138
3139#elif defined(PGMPOOL_WITH_CACHE)
3140 if (PGM_PAGE_IS_ZERO(pPhysPage))
3141 rc = VINF_PGM_GCPHYS_ALIASED;
3142 else
3143 {
3144# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3145 /* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow kill the pool otherwise. */
3146 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3147# endif
3148 rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3149 if (rc == VINF_SUCCESS)
3150 *pfFlushTLBs = true;
3151 }
3152
3153# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3154 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3155# endif
3156
3157#else
3158 rc = VINF_PGM_GCPHYS_ALIASED;
3159#endif
3160
3161 if (rc == VINF_PGM_GCPHYS_ALIASED)
3162 {
3163 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
3164 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3165 rc = VINF_PGM_SYNC_CR3;
3166 }
3167 pgmUnlock(pVM);
3168 return rc;
3169}
3170
3171
3172/**
3173 * Scans all shadow page tables for mappings of a physical page.
3174 *
3175 * This may be slow, but it's most likely more efficient than cleaning
3176 * out the entire page pool / cache.
3177 *
3178 * @returns VBox status code.
3179 * @retval VINF_SUCCESS if all references has been successfully cleared.
3180 * @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3181 * a page pool cleaning.
3182 *
3183 * @param pVM The VM handle.
3184 * @param pPhysPage The guest page in question.
3185 */
3186int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
3187{
3188 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3189 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3190 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3191 pPool->cUsedPages, pPool->cPresent, pPhysPage));
3192
3193#if 1
3194 /*
3195 * There is a limit to what makes sense.
3196 */
3197 if (pPool->cPresent > 1024)
3198 {
3199 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3200 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3201 return VINF_PGM_GCPHYS_ALIASED;
3202 }
3203#endif
3204
3205 /*
3206 * Iterate all the pages until we've encountered all that in use.
3207 * This is simple but not quite optimal solution.
3208 */
3209 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
3210 const uint32_t u32 = u64;
3211 unsigned cLeft = pPool->cUsedPages;
3212 unsigned iPage = pPool->cCurPages;
3213 while (--iPage >= PGMPOOL_IDX_FIRST)
3214 {
3215 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3216 if (pPage->GCPhys != NIL_RTGCPHYS)
3217 {
3218 switch (pPage->enmKind)
3219 {
3220 /*
3221 * We only care about shadow page tables.
3222 */
3223 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3224 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3225 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3226 {
3227 unsigned cPresent = pPage->cPresent;
3228 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3229 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3230 if (pPT->a[i].n.u1Present)
3231 {
3232 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
3233 {
3234 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3235 pPT->a[i].u = 0;
3236 }
3237 if (!--cPresent)
3238 break;
3239 }
3240 break;
3241 }
3242
3243 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3244 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3245 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3246 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3247 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3248 {
3249 unsigned cPresent = pPage->cPresent;
3250 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3251 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3252 if (pPT->a[i].n.u1Present)
3253 {
3254 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
3255 {
3256 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3257 pPT->a[i].u = 0;
3258 }
3259 if (!--cPresent)
3260 break;
3261 }
3262 break;
3263 }
3264 }
3265 if (!--cLeft)
3266 break;
3267 }
3268 }
3269
3270 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3271 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3272 return VINF_SUCCESS;
3273}
3274
3275
3276/**
3277 * Clears the user entry in a user table.
3278 *
3279 * This is used to remove all references to a page when flushing it.
3280 */
3281static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3282{
3283 Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3284 Assert(pUser->iUser < pPool->cCurPages);
3285 uint32_t iUserTable = pUser->iUserTable;
3286
3287 /*
3288 * Map the user page.
3289 */
3290 PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3291 union
3292 {
3293 uint64_t *pau64;
3294 uint32_t *pau32;
3295 } u;
3296 u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3297
3298 LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3299
3300 /* Safety precaution in case we change the paging for other modes too in the future. */
3301 Assert(!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage));
3302
3303#ifdef VBOX_STRICT
3304 /*
3305 * Some sanity checks.
3306 */
3307 switch (pUserPage->enmKind)
3308 {
3309 case PGMPOOLKIND_32BIT_PD:
3310 case PGMPOOLKIND_32BIT_PD_PHYS:
3311 Assert(iUserTable < X86_PG_ENTRIES);
3312 break;
3313 case PGMPOOLKIND_PAE_PDPT:
3314 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3315 case PGMPOOLKIND_PAE_PDPT_PHYS:
3316 Assert(iUserTable < 4);
3317 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3318 break;
3319 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3320 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3321 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3322 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3323 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3324 case PGMPOOLKIND_PAE_PD_PHYS:
3325 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3326 break;
3327 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3328 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3329 Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
3330 break;
3331 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3332 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3333 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3334 break;
3335 case PGMPOOLKIND_64BIT_PML4:
3336 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3337 /* GCPhys >> PAGE_SHIFT is the index here */
3338 break;
3339 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3340 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3341 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3342 break;
3343
3344 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3345 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3346 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3347 break;
3348
3349 case PGMPOOLKIND_ROOT_NESTED:
3350 Assert(iUserTable < X86_PG_PAE_ENTRIES);
3351 break;
3352
3353 default:
3354 AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3355 break;
3356 }
3357#endif /* VBOX_STRICT */
3358
3359 /*
3360 * Clear the entry in the user page.
3361 */
3362 switch (pUserPage->enmKind)
3363 {
3364 /* 32-bit entries */
3365 case PGMPOOLKIND_32BIT_PD:
3366 case PGMPOOLKIND_32BIT_PD_PHYS:
3367 u.pau32[iUserTable] = 0;
3368 break;
3369
3370 /* 64-bit entries */
3371 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3372 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3373 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3374 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3375 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3376#if defined(IN_RC)
3377 /* In 32 bits PAE mode we *must* invalidate the TLB when changing a PDPT entry; the CPU fetches them only during cr3 load, so any
3378 * non-present PDPT will continue to cause page faults.
3379 */
3380 ASMReloadCR3();
3381#endif
3382 /* no break */
3383 case PGMPOOLKIND_PAE_PD_PHYS:
3384 case PGMPOOLKIND_PAE_PDPT_PHYS:
3385 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3386 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3387 case PGMPOOLKIND_64BIT_PML4:
3388 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3389 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3390 case PGMPOOLKIND_PAE_PDPT:
3391 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3392 case PGMPOOLKIND_ROOT_NESTED:
3393 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3394 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3395 u.pau64[iUserTable] = 0;
3396 break;
3397
3398 default:
3399 AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
3400 }
3401}
3402
3403
3404/**
3405 * Clears all users of a page.
3406 */
3407static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3408{
3409 /*
3410 * Free all the user records.
3411 */
3412 LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
3413
3414 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3415 uint16_t i = pPage->iUserHead;
3416 while (i != NIL_PGMPOOL_USER_INDEX)
3417 {
3418 /* Clear enter in user table. */
3419 pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
3420
3421 /* Free it. */
3422 const uint16_t iNext = paUsers[i].iNext;
3423 paUsers[i].iUser = NIL_PGMPOOL_IDX;
3424 paUsers[i].iNext = pPool->iUserFreeHead;
3425 pPool->iUserFreeHead = i;
3426
3427 /* Next. */
3428 i = iNext;
3429 }
3430 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3431}
3432
3433#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3434
3435/**
3436 * Allocates a new physical cross reference extent.
3437 *
3438 * @returns Pointer to the allocated extent on success. NULL if we're out of them.
3439 * @param pVM The VM handle.
3440 * @param piPhysExt Where to store the phys ext index.
3441 */
3442PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
3443{
3444 Assert(PGMIsLockOwner(pVM));
3445 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3446 uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3447 if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3448 {
3449 STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3450 return NULL;
3451 }
3452 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3453 pPool->iPhysExtFreeHead = pPhysExt->iNext;
3454 pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3455 *piPhysExt = iPhysExt;
3456 return pPhysExt;
3457}
3458
3459
3460/**
3461 * Frees a physical cross reference extent.
3462 *
3463 * @param pVM The VM handle.
3464 * @param iPhysExt The extent to free.
3465 */
3466void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
3467{
3468 Assert(PGMIsLockOwner(pVM));
3469 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3470 Assert(iPhysExt < pPool->cMaxPhysExts);
3471 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3472 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3473 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3474 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3475 pPool->iPhysExtFreeHead = iPhysExt;
3476}
3477
3478
3479/**
3480 * Frees a physical cross reference extent.
3481 *
3482 * @param pVM The VM handle.
3483 * @param iPhysExt The extent to free.
3484 */
3485void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
3486{
3487 Assert(PGMIsLockOwner(pVM));
3488 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3489
3490 const uint16_t iPhysExtStart = iPhysExt;
3491 PPGMPOOLPHYSEXT pPhysExt;
3492 do
3493 {
3494 Assert(iPhysExt < pPool->cMaxPhysExts);
3495 pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3496 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3497 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3498
3499 /* next */
3500 iPhysExt = pPhysExt->iNext;
3501 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3502
3503 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3504 pPool->iPhysExtFreeHead = iPhysExtStart;
3505}
3506
3507
3508/**
3509 * Insert a reference into a list of physical cross reference extents.
3510 *
3511 * @returns The new tracking data for PGMPAGE.
3512 *
3513 * @param pVM The VM handle.
3514 * @param iPhysExt The physical extent index of the list head.
3515 * @param iShwPT The shadow page table index.
3516 *
3517 */
3518static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
3519{
3520 Assert(PGMIsLockOwner(pVM));
3521 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3522 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3523
3524 /* special common case. */
3525 if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3526 {
3527 paPhysExts[iPhysExt].aidx[2] = iShwPT;
3528 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3529 LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d}\n", iPhysExt, iShwPT));
3530 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3531 }
3532
3533 /* general treatment. */
3534 const uint16_t iPhysExtStart = iPhysExt;
3535 unsigned cMax = 15;
3536 for (;;)
3537 {
3538 Assert(iPhysExt < pPool->cMaxPhysExts);
3539 for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3540 if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3541 {
3542 paPhysExts[iPhysExt].aidx[i] = iShwPT;
3543 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3544 LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
3545 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
3546 }
3547 if (!--cMax)
3548 {
3549 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3550 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3551 LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
3552 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3553 }
3554 }
3555
3556 /* add another extent to the list. */
3557 PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3558 if (!pNew)
3559 {
3560 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3561 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3562 LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
3563 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3564 }
3565 pNew->iNext = iPhysExtStart;
3566 pNew->aidx[0] = iShwPT;
3567 LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
3568 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3569}
3570
3571
3572/**
3573 * Add a reference to guest physical page where extents are in use.
3574 *
3575 * @returns The new tracking data for PGMPAGE.
3576 *
3577 * @param pVM The VM handle.
3578 * @param u16 The ram range flags (top 16-bits).
3579 * @param iShwPT The shadow page table index.
3580 */
3581uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
3582{
3583 pgmLock(pVM);
3584 if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3585 {
3586 /*
3587 * Convert to extent list.
3588 */
3589 Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3590 uint16_t iPhysExt;
3591 PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3592 if (pPhysExt)
3593 {
3594 LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
3595 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
3596 pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
3597 pPhysExt->aidx[1] = iShwPT;
3598 u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3599 }
3600 else
3601 u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3602 }
3603 else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3604 {
3605 /*
3606 * Insert into the extent list.
3607 */
3608 u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT);
3609 }
3610 else
3611 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
3612 pgmUnlock(pVM);
3613 return u16;
3614}
3615
3616
3617/**
3618 * Clear references to guest physical memory.
3619 *
3620 * @param pPool The pool.
3621 * @param pPage The page.
3622 * @param pPhysPage Pointer to the aPages entry in the ram range.
3623 */
3624void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
3625{
3626 const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
3627 AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3628
3629 uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
3630 if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
3631 {
3632 PVM pVM = pPool->CTX_SUFF(pVM);
3633 pgmLock(pVM);
3634
3635 uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
3636 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3637 do
3638 {
3639 Assert(iPhysExt < pPool->cMaxPhysExts);
3640
3641 /*
3642 * Look for the shadow page and check if it's all freed.
3643 */
3644 for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3645 {
3646 if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
3647 {
3648 paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
3649
3650 for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3651 if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
3652 {
3653 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3654 pgmUnlock(pVM);
3655 return;
3656 }
3657
3658 /* we can free the node. */
3659 const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
3660 if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
3661 && iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
3662 {
3663 /* lonely node */
3664 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3665 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
3666 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3667 }
3668 else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
3669 {
3670 /* head */
3671 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
3672 PGM_PAGE_SET_TRACKING(pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
3673 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3674 }
3675 else
3676 {
3677 /* in list */
3678 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3679 paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
3680 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3681 }
3682 iPhysExt = iPhysExtNext;
3683 pgmUnlock(pVM);
3684 return;
3685 }
3686 }
3687
3688 /* next */
3689 iPhysExtPrev = iPhysExt;
3690 iPhysExt = paPhysExts[iPhysExt].iNext;
3691 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3692
3693 pgmUnlock(pVM);
3694 AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3695 }
3696 else /* nothing to do */
3697 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
3698}
3699
3700
3701/**
3702 * Clear references to guest physical memory.
3703 *
3704 * This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
3705 * is assumed to be correct, so the linear search can be skipped and we can assert
3706 * at an earlier point.
3707 *
3708 * @param pPool The pool.
3709 * @param pPage The page.
3710 * @param HCPhys The host physical address corresponding to the guest page.
3711 * @param GCPhys The guest physical address corresponding to HCPhys.
3712 */
3713static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
3714{
3715 /*
3716 * Walk range list.
3717 */
3718 PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3719 while (pRam)
3720 {
3721 RTGCPHYS off = GCPhys - pRam->GCPhys;
3722 if (off < pRam->cb)
3723 {
3724 /* does it match? */
3725 const unsigned iPage = off >> PAGE_SHIFT;
3726 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3727#ifdef LOG_ENABLED
3728RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
3729Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
3730#endif
3731 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3732 {
3733 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3734 return;
3735 }
3736 break;
3737 }
3738 pRam = pRam->CTX_SUFF(pNext);
3739 }
3740 AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
3741}
3742
3743
3744/**
3745 * Clear references to guest physical memory.
3746 *
3747 * @param pPool The pool.
3748 * @param pPage The page.
3749 * @param HCPhys The host physical address corresponding to the guest page.
3750 * @param GCPhysHint The guest physical address which may corresponding to HCPhys.
3751 */
3752void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
3753{
3754 Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
3755
3756 /*
3757 * Walk range list.
3758 */
3759 PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3760 while (pRam)
3761 {
3762 RTGCPHYS off = GCPhysHint - pRam->GCPhys;
3763 if (off < pRam->cb)
3764 {
3765 /* does it match? */
3766 const unsigned iPage = off >> PAGE_SHIFT;
3767 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3768 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3769 {
3770 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3771 return;
3772 }
3773 break;
3774 }
3775 pRam = pRam->CTX_SUFF(pNext);
3776 }
3777
3778 /*
3779 * Damn, the hint didn't work. We'll have to do an expensive linear search.
3780 */
3781 STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
3782 pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3783 while (pRam)
3784 {
3785 unsigned iPage = pRam->cb >> PAGE_SHIFT;
3786 while (iPage-- > 0)
3787 {
3788 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3789 {
3790 Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
3791 HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
3792 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3793 return;
3794 }
3795 }
3796 pRam = pRam->CTX_SUFF(pNext);
3797 }
3798
3799 AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp\n", HCPhys, GCPhysHint));
3800}
3801
3802
3803/**
3804 * Clear references to guest physical memory in a 32-bit / 32-bit page table.
3805 *
3806 * @param pPool The pool.
3807 * @param pPage The page.
3808 * @param pShwPT The shadow page table (mapping of the page).
3809 * @param pGstPT The guest page table.
3810 */
3811DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
3812{
3813 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
3814 if (pShwPT->a[i].n.u1Present)
3815 {
3816 Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
3817 i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3818 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3819 if (!--pPage->cPresent)
3820 break;
3821 }
3822}
3823
3824
3825/**
3826 * Clear references to guest physical memory in a PAE / 32-bit page table.
3827 *
3828 * @param pPool The pool.
3829 * @param pPage The page.
3830 * @param pShwPT The shadow page table (mapping of the page).
3831 * @param pGstPT The guest page table (just a half one).
3832 */
3833DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
3834{
3835 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3836 if (pShwPT->a[i].n.u1Present)
3837 {
3838 Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
3839 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3840 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3841 }
3842}
3843
3844
3845/**
3846 * Clear references to guest physical memory in a PAE / PAE page table.
3847 *
3848 * @param pPool The pool.
3849 * @param pPage The page.
3850 * @param pShwPT The shadow page table (mapping of the page).
3851 * @param pGstPT The guest page table.
3852 */
3853DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
3854{
3855 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3856 if (pShwPT->a[i].n.u1Present)
3857 {
3858 Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
3859 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
3860 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
3861 }
3862}
3863
3864
3865/**
3866 * Clear references to guest physical memory in a 32-bit / 4MB page table.
3867 *
3868 * @param pPool The pool.
3869 * @param pPage The page.
3870 * @param pShwPT The shadow page table (mapping of the page).
3871 */
3872DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
3873{
3874 RTGCPHYS GCPhys = pPage->GCPhys;
3875 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3876 if (pShwPT->a[i].n.u1Present)
3877 {
3878 Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
3879 i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
3880 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
3881 }
3882}
3883
3884
3885/**
3886 * Clear references to guest physical memory in a PAE / 2/4MB page table.
3887 *
3888 * @param pPool The pool.
3889 * @param pPage The page.
3890 * @param pShwPT The shadow page table (mapping of the page).
3891 */
3892DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
3893{
3894 RTGCPHYS GCPhys = pPage->GCPhys;
3895 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3896 if (pShwPT->a[i].n.u1Present)
3897 {
3898 Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
3899 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
3900 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
3901 }
3902}
3903
3904#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3905
3906
3907/**
3908 * Clear references to shadowed pages in a 32 bits page directory.
3909 *
3910 * @param pPool The pool.
3911 * @param pPage The page.
3912 * @param pShwPD The shadow page directory (mapping of the page).
3913 */
3914DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
3915{
3916 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3917 {
3918 if ( pShwPD->a[i].n.u1Present
3919 && !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3920 )
3921 {
3922 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
3923 if (pSubPage)
3924 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3925 else
3926 AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
3927 }
3928 }
3929}
3930
3931/**
3932 * Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
3933 *
3934 * @param pPool The pool.
3935 * @param pPage The page.
3936 * @param pShwPD The shadow page directory (mapping of the page).
3937 */
3938DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
3939{
3940 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3941 {
3942 if ( pShwPD->a[i].n.u1Present
3943 && !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3944 )
3945 {
3946 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
3947 if (pSubPage)
3948 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3949 else
3950 AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
3951 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3952 }
3953 }
3954}
3955
3956/**
3957 * Clear references to shadowed pages in a PAE page directory pointer table.
3958 *
3959 * @param pPool The pool.
3960 * @param pPage The page.
3961 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3962 */
3963DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3964{
3965 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
3966 {
3967 if ( pShwPDPT->a[i].n.u1Present
3968 && !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
3969 )
3970 {
3971 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3972 if (pSubPage)
3973 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3974 else
3975 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3976 }
3977 }
3978}
3979
3980
3981/**
3982 * Clear references to shadowed pages in a 64-bit page directory pointer table.
3983 *
3984 * @param pPool The pool.
3985 * @param pPage The page.
3986 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3987 */
3988DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3989{
3990 for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3991 {
3992 Assert(!(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING));
3993 if (pShwPDPT->a[i].n.u1Present)
3994 {
3995 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3996 if (pSubPage)
3997 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3998 else
3999 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
4000 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4001 }
4002 }
4003}
4004
4005
4006/**
4007 * Clear references to shadowed pages in a 64-bit level 4 page table.
4008 *
4009 * @param pPool The pool.
4010 * @param pPage The page.
4011 * @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4012 */
4013DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4014{
4015 for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4016 {
4017 if (pShwPML4->a[i].n.u1Present)
4018 {
4019 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
4020 if (pSubPage)
4021 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4022 else
4023 AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
4024 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4025 }
4026 }
4027}
4028
4029
4030/**
4031 * Clear references to shadowed pages in an EPT page table.
4032 *
4033 * @param pPool The pool.
4034 * @param pPage The page.
4035 * @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4036 */
4037DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4038{
4039 RTGCPHYS GCPhys = pPage->GCPhys;
4040 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4041 if (pShwPT->a[i].n.u1Present)
4042 {
4043 Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4044 i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
4045 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys);
4046 }
4047}
4048
4049
4050/**
4051 * Clear references to shadowed pages in an EPT page directory.
4052 *
4053 * @param pPool The pool.
4054 * @param pPage The page.
4055 * @param pShwPD The shadow page directory (mapping of the page).
4056 */
4057DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4058{
4059 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4060 {
4061 if (pShwPD->a[i].n.u1Present)
4062 {
4063 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
4064 if (pSubPage)
4065 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4066 else
4067 AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4068 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4069 }
4070 }
4071}
4072
4073
4074/**
4075 * Clear references to shadowed pages in an EPT page directory pointer table.
4076 *
4077 * @param pPool The pool.
4078 * @param pPage The page.
4079 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4080 */
4081DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4082{
4083 for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4084 {
4085 if (pShwPDPT->a[i].n.u1Present)
4086 {
4087 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
4088 if (pSubPage)
4089 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4090 else
4091 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
4092 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4093 }
4094 }
4095}
4096
4097
4098/**
4099 * Clears all references made by this page.
4100 *
4101 * This includes other shadow pages and GC physical addresses.
4102 *
4103 * @param pPool The pool.
4104 * @param pPage The page.
4105 */
4106static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4107{
4108 /*
4109 * Map the shadow page and take action according to the page kind.
4110 */
4111 void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
4112 switch (pPage->enmKind)
4113 {
4114#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4115 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4116 {
4117 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4118 void *pvGst;
4119 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4120 pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4121 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4122 break;
4123 }
4124
4125 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4126 {
4127 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4128 void *pvGst;
4129 int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4130 pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
4131 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4132 break;
4133 }
4134
4135 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4136 {
4137 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4138 void *pvGst;
4139 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4140 pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
4141 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4142 break;
4143 }
4144
4145 case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4146 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4147 {
4148 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4149 pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4150 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4151 break;
4152 }
4153
4154 case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
4155 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4156 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4157 {
4158 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4159 pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
4160 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4161 break;
4162 }
4163
4164#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
4165 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4166 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4167 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4168 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4169 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4170 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4171 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4172 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4173 break;
4174#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
4175
4176 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4177 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4178 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4179 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4180 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4181 case PGMPOOLKIND_PAE_PD_PHYS:
4182 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4183 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4184 pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
4185 break;
4186
4187 case PGMPOOLKIND_32BIT_PD_PHYS:
4188 case PGMPOOLKIND_32BIT_PD:
4189 pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
4190 break;
4191
4192 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4193 case PGMPOOLKIND_PAE_PDPT:
4194 case PGMPOOLKIND_PAE_PDPT_PHYS:
4195 pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
4196 break;
4197
4198 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4199 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4200 pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
4201 break;
4202
4203 case PGMPOOLKIND_64BIT_PML4:
4204 pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
4205 break;
4206
4207 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4208 pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
4209 break;
4210
4211 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4212 pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
4213 break;
4214
4215 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4216 pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
4217 break;
4218
4219 default:
4220 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
4221 }
4222
4223 /* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
4224 STAM_PROFILE_START(&pPool->StatZeroPage, z);
4225 ASMMemZeroPage(pvShw);
4226 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4227 pPage->fZeroed = true;
4228 PGMPOOL_UNLOCK_PTR(pPool->CTX_SUFF(pVM), pvShw);
4229}
4230#endif /* PGMPOOL_WITH_USER_TRACKING */
4231
4232/**
4233 * Flushes a pool page.
4234 *
4235 * This moves the page to the free list after removing all user references to it.
4236 *
4237 * @returns VBox status code.
4238 * @retval VINF_SUCCESS on success.
4239 * @param pPool The pool.
4240 * @param HCPhys The HC physical address of the shadow page.
4241 */
4242int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4243{
4244 PVM pVM = pPool->CTX_SUFF(pVM);
4245
4246 int rc = VINF_SUCCESS;
4247 STAM_PROFILE_START(&pPool->StatFlushPage, f);
4248 LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4249 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4250
4251 /*
4252 * Quietly reject any attempts at flushing any of the special root pages.
4253 */
4254 if (pPage->idx < PGMPOOL_IDX_FIRST)
4255 {
4256 AssertFailed(); /* can no longer happen */
4257 Log(("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4258 return VINF_SUCCESS;
4259 }
4260
4261 pgmLock(pVM);
4262
4263 /*
4264 * Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4265 */
4266 if (pgmPoolIsPageLocked(&pVM->pgm.s, pPage))
4267 {
4268 AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4269 || pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4270 || pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4271 || pPage->enmKind == PGMPOOLKIND_32BIT_PD
4272 || pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4273 || pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4274 || pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4275 || pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4276 || pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
4277 ("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
4278 Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4279 pgmUnlock(pVM);
4280 return VINF_SUCCESS;
4281 }
4282
4283#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4284 /* Start a subset so we won't run out of mapping space. */
4285 PVMCPU pVCpu = VMMGetCpu(pVM);
4286 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
4287#endif
4288
4289 /*
4290 * Mark the page as being in need of an ASMMemZeroPage().
4291 */
4292 pPage->fZeroed = false;
4293
4294#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4295 if (pPage->fDirty)
4296 pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirty, true /* force removal */);
4297#endif
4298
4299#ifdef PGMPOOL_WITH_USER_TRACKING
4300 /*
4301 * Clear the page.
4302 */
4303 pgmPoolTrackClearPageUsers(pPool, pPage);
4304 STAM_PROFILE_START(&pPool->StatTrackDeref,a);
4305 pgmPoolTrackDeref(pPool, pPage);
4306 STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
4307#endif
4308
4309#ifdef PGMPOOL_WITH_CACHE
4310 /*
4311 * Flush it from the cache.
4312 */
4313 pgmPoolCacheFlushPage(pPool, pPage);
4314#endif /* PGMPOOL_WITH_CACHE */
4315
4316#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4317 /* Heavy stuff done. */
4318 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
4319#endif
4320
4321#ifdef PGMPOOL_WITH_MONITORING
4322 /*
4323 * Deregistering the monitoring.
4324 */
4325 if (pPage->fMonitored)
4326 rc = pgmPoolMonitorFlush(pPool, pPage);
4327#endif
4328
4329 /*
4330 * Free the page.
4331 */
4332 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
4333 pPage->iNext = pPool->iFreeHead;
4334 pPool->iFreeHead = pPage->idx;
4335 pPage->enmKind = PGMPOOLKIND_FREE;
4336 pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4337 pPage->GCPhys = NIL_RTGCPHYS;
4338 pPage->fReusedFlushPending = false;
4339
4340 pPool->cUsedPages--;
4341 pgmUnlock(pVM);
4342 STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
4343 return rc;
4344}
4345
4346
4347/**
4348 * Frees a usage of a pool page.
4349 *
4350 * The caller is responsible to updating the user table so that it no longer
4351 * references the shadow page.
4352 *
4353 * @param pPool The pool.
4354 * @param HCPhys The HC physical address of the shadow page.
4355 * @param iUser The shadow page pool index of the user table.
4356 * @param iUserTable The index into the user table (shadowed).
4357 */
4358void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
4359{
4360 PVM pVM = pPool->CTX_SUFF(pVM);
4361
4362 STAM_PROFILE_START(&pPool->StatFree, a);
4363 LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%#x iUserTable=%#x\n",
4364 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
4365 Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
4366 pgmLock(pVM);
4367#ifdef PGMPOOL_WITH_USER_TRACKING
4368 pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
4369#endif
4370#ifdef PGMPOOL_WITH_CACHE
4371 if (!pPage->fCached)
4372#endif
4373 pgmPoolFlushPage(pPool, pPage);
4374 pgmUnlock(pVM);
4375 STAM_PROFILE_STOP(&pPool->StatFree, a);
4376}
4377
4378
4379/**
4380 * Makes one or more free page free.
4381 *
4382 * @returns VBox status code.
4383 * @retval VINF_SUCCESS on success.
4384 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4385 *
4386 * @param pPool The pool.
4387 * @param enmKind Page table kind
4388 * @param iUser The user of the page.
4389 */
4390static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
4391{
4392 PVM pVM = pPool->CTX_SUFF(pVM);
4393
4394 LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
4395
4396 /*
4397 * If the pool isn't full grown yet, expand it.
4398 */
4399 if ( pPool->cCurPages < pPool->cMaxPages
4400#if defined(IN_RC)
4401 /* Hack alert: we can't deal with jumps to ring 3 when called from MapCR3 and allocating pages for PAE PDs. */
4402 && enmKind != PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4403 && (enmKind < PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD || enmKind > PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD)
4404#endif
4405 )
4406 {
4407 STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
4408#ifdef IN_RING3
4409 int rc = PGMR3PoolGrow(pVM);
4410#else
4411 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
4412#endif
4413 if (RT_FAILURE(rc))
4414 return rc;
4415 STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
4416 if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
4417 return VINF_SUCCESS;
4418 }
4419
4420#ifdef PGMPOOL_WITH_CACHE
4421 /*
4422 * Free one cached page.
4423 */
4424 return pgmPoolCacheFreeOne(pPool, iUser);
4425#else
4426 /*
4427 * Flush the pool.
4428 *
4429 * If we have tracking enabled, it should be possible to come up with
4430 * a cheap replacement strategy...
4431 */
4432 /* @todo This path no longer works (CR3 root pages will be flushed)!! */
4433 AssertCompileFailed();
4434 Assert(!CPUMIsGuestInLongMode(pVM));
4435 pgmPoolFlushAllInt(pPool);
4436 return VERR_PGM_POOL_FLUSHED;
4437#endif
4438}
4439
4440/**
4441 * Allocates a page from the pool.
4442 *
4443 * This page may actually be a cached page and not in need of any processing
4444 * on the callers part.
4445 *
4446 * @returns VBox status code.
4447 * @retval VINF_SUCCESS if a NEW page was allocated.
4448 * @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4449 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4450 * @param pVM The VM handle.
4451 * @param GCPhys The GC physical address of the page we're gonna shadow.
4452 * For 4MB and 2MB PD entries, it's the first address the
4453 * shadow PT is covering.
4454 * @param enmKind The kind of mapping.
4455 * @param enmAccess Access type for the mapping (only relevant for big pages)
4456 * @param iUser The shadow page pool index of the user table.
4457 * @param iUserTable The index into the user table (shadowed).
4458 * @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4459 * @param fLockPage Lock the page
4460 */
4461int pgmPoolAllocEx(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage, bool fLockPage)
4462{
4463 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4464 STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4465 LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%#x iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
4466 *ppPage = NULL;
4467 /** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4468 * (TRPMR3SyncIDT) because of FF priority. Try fix that?
4469 * Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
4470
4471 pgmLock(pVM);
4472
4473#ifdef PGMPOOL_WITH_CACHE
4474 if (pPool->fCacheEnabled)
4475 {
4476 int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, iUser, iUserTable, ppPage);
4477 if (RT_SUCCESS(rc2))
4478 {
4479 if (fLockPage)
4480 pgmPoolLockPage(pPool, *ppPage);
4481 pgmUnlock(pVM);
4482 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4483 LogFlow(("pgmPoolAlloc: cached returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, *ppPage, (*ppPage)->Core.Key, (*ppPage)->idx));
4484 return rc2;
4485 }
4486 }
4487#endif
4488
4489 /*
4490 * Allocate a new one.
4491 */
4492 int rc = VINF_SUCCESS;
4493 uint16_t iNew = pPool->iFreeHead;
4494 if (iNew == NIL_PGMPOOL_IDX)
4495 {
4496 rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
4497 if (RT_FAILURE(rc))
4498 {
4499 pgmUnlock(pVM);
4500 Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
4501 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4502 return rc;
4503 }
4504 iNew = pPool->iFreeHead;
4505 AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
4506 }
4507
4508 /* unlink the free head */
4509 PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
4510 pPool->iFreeHead = pPage->iNext;
4511 pPage->iNext = NIL_PGMPOOL_IDX;
4512
4513 /*
4514 * Initialize it.
4515 */
4516 pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
4517 pPage->enmKind = enmKind;
4518 pPage->enmAccess = enmAccess;
4519 pPage->GCPhys = GCPhys;
4520 pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
4521 pPage->fMonitored = false;
4522 pPage->fCached = false;
4523#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4524 pPage->fDirty = false;
4525#endif
4526 pPage->fReusedFlushPending = false;
4527#ifdef PGMPOOL_WITH_MONITORING
4528 pPage->cModifications = 0;
4529 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4530 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4531#else
4532 pPage->fCR3Mix = false;
4533#endif
4534#ifdef PGMPOOL_WITH_USER_TRACKING
4535 pPage->cPresent = 0;
4536 pPage->iFirstPresent = ~0;
4537 pPage->pvLastAccessHandlerFault = 0;
4538 pPage->cLastAccessHandlerCount = 0;
4539 pPage->pvLastAccessHandlerRip = 0;
4540
4541 /*
4542 * Insert into the tracking and cache. If this fails, free the page.
4543 */
4544 int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
4545 if (RT_FAILURE(rc3))
4546 {
4547 pPool->cUsedPages--;
4548 pPage->enmKind = PGMPOOLKIND_FREE;
4549 pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4550 pPage->GCPhys = NIL_RTGCPHYS;
4551 pPage->iNext = pPool->iFreeHead;
4552 pPool->iFreeHead = pPage->idx;
4553 pgmUnlock(pVM);
4554 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4555 Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
4556 return rc3;
4557 }
4558#endif /* PGMPOOL_WITH_USER_TRACKING */
4559
4560 /*
4561 * Commit the allocation, clear the page and return.
4562 */
4563#ifdef VBOX_WITH_STATISTICS
4564 if (pPool->cUsedPages > pPool->cUsedPagesHigh)
4565 pPool->cUsedPagesHigh = pPool->cUsedPages;
4566#endif
4567
4568 if (!pPage->fZeroed)
4569 {
4570 STAM_PROFILE_START(&pPool->StatZeroPage, z);
4571 void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4572 ASMMemZeroPage(pv);
4573 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4574 }
4575
4576 *ppPage = pPage;
4577 if (fLockPage)
4578 pgmPoolLockPage(pPool, pPage);
4579 pgmUnlock(pVM);
4580 LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
4581 rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
4582 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4583 return rc;
4584}
4585
4586
4587/**
4588 * Frees a usage of a pool page.
4589 *
4590 * @param pVM The VM handle.
4591 * @param HCPhys The HC physical address of the shadow page.
4592 * @param iUser The shadow page pool index of the user table.
4593 * @param iUserTable The index into the user table (shadowed).
4594 */
4595void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
4596{
4597 LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
4598 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4599 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
4600}
4601
4602/**
4603 * Internal worker for finding a 'in-use' shadow page give by it's physical address.
4604 *
4605 * @returns Pointer to the shadow page structure.
4606 * @param pPool The pool.
4607 * @param HCPhys The HC physical address of the shadow page.
4608 */
4609PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
4610{
4611 PVM pVM = pPool->CTX_SUFF(pVM);
4612
4613 Assert(PGMIsLockOwner(pVM));
4614
4615 /*
4616 * Look up the page.
4617 */
4618 pgmLock(pVM);
4619 PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
4620 pgmUnlock(pVM);
4621
4622 AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
4623 return pPage;
4624}
4625
4626
4627#ifdef IN_RING3
4628/**
4629 * Flushes the entire cache.
4630 *
4631 * It will assert a global CR3 flush (FF) and assumes the caller is aware of this
4632 * and execute this CR3 flush.
4633 *
4634 * @param pPool The pool.
4635 */
4636void pgmR3PoolReset(PVM pVM)
4637{
4638 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4639
4640 Assert(PGMIsLockOwner(pVM));
4641 STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
4642 LogFlow(("pgmPoolFlushAllInt:\n"));
4643
4644 /*
4645 * If there are no pages in the pool, there is nothing to do.
4646 */
4647 if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
4648 {
4649 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4650 return;
4651 }
4652
4653 /*
4654 * Exit the shadow mode since we're going to clear everything,
4655 * including the root page.
4656 */
4657 for (unsigned i=0;i<pVM->cCPUs;i++)
4658 {
4659 PVMCPU pVCpu = &pVM->aCpus[i];
4660 pgmR3ExitShadowModeBeforePoolFlush(pVM, pVCpu);
4661 }
4662
4663 /*
4664 * Nuke the free list and reinsert all pages into it.
4665 */
4666 for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
4667 {
4668 PPGMPOOLPAGE pPage = &pPool->aPages[i];
4669
4670 Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
4671#ifdef PGMPOOL_WITH_MONITORING
4672 if (pPage->fMonitored)
4673 pgmPoolMonitorFlush(pPool, pPage);
4674 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4675 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4676 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4677 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4678 pPage->cModifications = 0;
4679#endif
4680 pPage->GCPhys = NIL_RTGCPHYS;
4681 pPage->enmKind = PGMPOOLKIND_FREE;
4682 pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4683 Assert(pPage->idx == i);
4684 pPage->iNext = i + 1;
4685 pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
4686 pPage->fSeenNonGlobal = false;
4687 pPage->fMonitored = false;
4688#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4689 pPage->fDirty = false;
4690#endif
4691 pPage->fCached = false;
4692 pPage->fReusedFlushPending = false;
4693#ifdef PGMPOOL_WITH_USER_TRACKING
4694 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4695#else
4696 pPage->fCR3Mix = false;
4697#endif
4698#ifdef PGMPOOL_WITH_CACHE
4699 pPage->iAgeNext = NIL_PGMPOOL_IDX;
4700 pPage->iAgePrev = NIL_PGMPOOL_IDX;
4701#endif
4702 pPage->cLocked = 0;
4703 }
4704 pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
4705 pPool->iFreeHead = PGMPOOL_IDX_FIRST;
4706 pPool->cUsedPages = 0;
4707
4708#ifdef PGMPOOL_WITH_USER_TRACKING
4709 /*
4710 * Zap and reinitialize the user records.
4711 */
4712 pPool->cPresent = 0;
4713 pPool->iUserFreeHead = 0;
4714 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4715 const unsigned cMaxUsers = pPool->cMaxUsers;
4716 for (unsigned i = 0; i < cMaxUsers; i++)
4717 {
4718 paUsers[i].iNext = i + 1;
4719 paUsers[i].iUser = NIL_PGMPOOL_IDX;
4720 paUsers[i].iUserTable = 0xfffffffe;
4721 }
4722 paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
4723#endif
4724
4725#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4726 /*
4727 * Clear all the GCPhys links and rebuild the phys ext free list.
4728 */
4729 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
4730 pRam;
4731 pRam = pRam->CTX_SUFF(pNext))
4732 {
4733 unsigned iPage = pRam->cb >> PAGE_SHIFT;
4734 while (iPage-- > 0)
4735 PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
4736 }
4737
4738 pPool->iPhysExtFreeHead = 0;
4739 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4740 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
4741 for (unsigned i = 0; i < cMaxPhysExts; i++)
4742 {
4743 paPhysExts[i].iNext = i + 1;
4744 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
4745 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
4746 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
4747 }
4748 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4749#endif
4750
4751#ifdef PGMPOOL_WITH_MONITORING
4752 /*
4753 * Just zap the modified list.
4754 */
4755 pPool->cModifiedPages = 0;
4756 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
4757#endif
4758
4759#ifdef PGMPOOL_WITH_CACHE
4760 /*
4761 * Clear the GCPhys hash and the age list.
4762 */
4763 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
4764 pPool->aiHash[i] = NIL_PGMPOOL_IDX;
4765 pPool->iAgeHead = NIL_PGMPOOL_IDX;
4766 pPool->iAgeTail = NIL_PGMPOOL_IDX;
4767#endif
4768
4769#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4770 /* Clear all dirty pages. */
4771 pPool->idxFreeDirtyPage = 0;
4772 pPool->cDirtyPages = 0;
4773 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
4774 pPool->aIdxDirtyPages[i] = NIL_PGMPOOL_IDX;
4775#endif
4776
4777 /*
4778 * Reinsert active pages into the hash and ensure monitoring chains are correct.
4779 */
4780 for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
4781 {
4782 PPGMPOOLPAGE pPage = &pPool->aPages[i];
4783 pPage->iNext = NIL_PGMPOOL_IDX;
4784#ifdef PGMPOOL_WITH_MONITORING
4785 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4786 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4787 pPage->cModifications = 0;
4788 /* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
4789 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4790 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4791 if (pPage->fMonitored)
4792 {
4793 int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
4794 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
4795 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
4796 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
4797 pPool->pszAccessHandler);
4798 AssertFatalRCSuccess(rc);
4799# ifdef PGMPOOL_WITH_CACHE
4800 pgmPoolHashInsert(pPool, pPage);
4801# endif
4802 }
4803#endif
4804#ifdef PGMPOOL_WITH_USER_TRACKING
4805 Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
4806#endif
4807#ifdef PGMPOOL_WITH_CACHE
4808 Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
4809 Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
4810#endif
4811 }
4812
4813 for (unsigned i=0;i<pVM->cCPUs;i++)
4814 {
4815 PVMCPU pVCpu = &pVM->aCpus[i];
4816 /*
4817 * Re-enter the shadowing mode and assert Sync CR3 FF.
4818 */
4819 pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
4820 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4821 }
4822
4823 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4824}
4825#endif /* IN_RING3 */
4826
4827#ifdef LOG_ENABLED
4828static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
4829{
4830 switch(enmKind)
4831 {
4832 case PGMPOOLKIND_INVALID:
4833 return "PGMPOOLKIND_INVALID";
4834 case PGMPOOLKIND_FREE:
4835 return "PGMPOOLKIND_FREE";
4836 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4837 return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
4838 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4839 return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
4840 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4841 return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
4842 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4843 return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
4844 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4845 return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
4846 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4847 return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
4848 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4849 return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
4850 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4851 return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
4852 case PGMPOOLKIND_32BIT_PD:
4853 return "PGMPOOLKIND_32BIT_PD";
4854 case PGMPOOLKIND_32BIT_PD_PHYS:
4855 return "PGMPOOLKIND_32BIT_PD_PHYS";
4856 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4857 return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
4858 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4859 return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
4860 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4861 return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
4862 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4863 return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
4864 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4865 return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
4866 case PGMPOOLKIND_PAE_PD_PHYS:
4867 return "PGMPOOLKIND_PAE_PD_PHYS";
4868 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4869 return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
4870 case PGMPOOLKIND_PAE_PDPT:
4871 return "PGMPOOLKIND_PAE_PDPT";
4872 case PGMPOOLKIND_PAE_PDPT_PHYS:
4873 return "PGMPOOLKIND_PAE_PDPT_PHYS";
4874 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4875 return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
4876 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4877 return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
4878 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4879 return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
4880 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4881 return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
4882 case PGMPOOLKIND_64BIT_PML4:
4883 return "PGMPOOLKIND_64BIT_PML4";
4884 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4885 return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
4886 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4887 return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
4888 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4889 return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
4890 case PGMPOOLKIND_ROOT_NESTED:
4891 return "PGMPOOLKIND_ROOT_NESTED";
4892 }
4893 return "Unknown kind!";
4894}
4895#endif /* LOG_ENABLED*/
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette