PGMAllPool.cpp@ 17641

Last change on this file since 17641 was 17611, checked in by vboxsync, 16 years ago
Fixed PAE PDPT dereferencing.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 160.6 KB

Line
1	/* $Id: PGMAllPool.cpp 17611 2009-03-10 10:15:50Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_PGM_POOL
27	#include <VBox/pgm.h>
28	#include <VBox/mm.h>
29	#include <VBox/em.h>
30	#include <VBox/cpum.h>
31	#ifdef IN_RC
32	# include <VBox/patm.h>
33	#endif
34	#include "PGMInternal.h"
35	#include <VBox/vm.h>
36	#include <VBox/disopcode.h>
37	#include <VBox/hwacc_vmx.h>
38
39	#include <VBox/log.h>
40	#include <VBox/err.h>
41	#include <iprt/asm.h>
42	#include <iprt/string.h>
43
44
45	/*******************************************************************************
46	* Internal Functions *
47	*******************************************************************************/
48	__BEGIN_DECLS
49	static void pgmPoolFlushAllInt(PPGMPOOL pPool);
50	#ifdef PGMPOOL_WITH_USER_TRACKING
51	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
52	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
53	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
54	#endif
55	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
56	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint);
57	#endif
58	#ifdef PGMPOOL_WITH_CACHE
59	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
60	#endif
61	#ifdef PGMPOOL_WITH_MONITORING
62	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
63	#endif
64	#ifndef IN_RING3
65	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
66	#endif
67	#ifdef LOG_ENABLED
68	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
69	#endif
70	__END_DECLS
71
72
73	/**
74	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
75	*
76	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
77	* @param enmKind The page kind.
78	*/
79	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
80	{
81	switch (enmKind)
82	{
83	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
84	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
85	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
86	return true;
87	default:
88	return false;
89	}
90	}
91
92	/** @def PGMPOOL_PAGE_2_LOCKED_PTR
93	* Maps a pool page pool into the current context and lock it (RC only).
94	*
95	* @returns VBox status code.
96	* @param pVM The VM handle.
97	* @param pPage The pool page.
98	*
99	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
100	* small page window employeed by that function. Be careful.
101	* @remark There is no need to assert on the result.
102	*/
103	#if defined(IN_RC)
104	DECLINLINE(void *) PGMPOOL_PAGE_2_LOCKED_PTR(PVM pVM, PPGMPOOLPAGE pPage)
105	{
106	void *pv = pgmPoolMapPageInlined(&pVM->pgm.s, pPage);
107
108	/* Make sure the dynamic mapping will not be reused. */
109	if (pv)
110	PGMDynLockHCPage(pVM, (uint8_t *)pv);
111
112	return pv;
113	}
114	#else
115	# define PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage) PGMPOOL_PAGE_2_PTR(pVM, pPage)
116	#endif
117
118	/** @def PGMPOOL_UNLOCK_PTR
119	* Unlock a previously locked dynamic caching (RC only).
120	*
121	* @returns VBox status code.
122	* @param pVM The VM handle.
123	* @param pPage The pool page.
124	*
125	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
126	* small page window employeed by that function. Be careful.
127	* @remark There is no need to assert on the result.
128	*/
129	#if defined(IN_RC)
130	DECLINLINE(void) PGMPOOL_UNLOCK_PTR(PVM pVM, void *pvPage)
131	{
132	if (pvPage)
133	PGMDynUnlockHCPage(pVM, (uint8_t *)pvPage);
134	}
135	#else
136	# define PGMPOOL_UNLOCK_PTR(pVM, pPage) do {} while (0)
137	#endif
138
139
140	#ifdef PGMPOOL_WITH_MONITORING
141	/**
142	* Determin the size of a write instruction.
143	* @returns number of bytes written.
144	* @param pDis The disassembler state.
145	*/
146	static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
147	{
148	/*
149	* This is very crude and possibly wrong for some opcodes,
150	* but since it's not really supposed to be called we can
151	* probably live with that.
152	*/
153	return DISGetParamSize(pDis, &pDis->param1);
154	}
155
156
157	/**
158	* Flushes a chain of pages sharing the same access monitor.
159	*
160	* @returns VBox status code suitable for scheduling.
161	* @param pPool The pool.
162	* @param pPage A page in the chain.
163	*/
164	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
165	{
166	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
167
168	/*
169	* Find the list head.
170	*/
171	uint16_t idx = pPage->idx;
172	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
173	{
174	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
175	{
176	idx = pPage->iMonitoredPrev;
177	Assert(idx != pPage->idx);
178	pPage = &pPool->aPages[idx];
179	}
180	}
181
182	/*
183	* Iterate the list flushing each shadow page.
184	*/
185	int rc = VINF_SUCCESS;
186	for (;;)
187	{
188	idx = pPage->iMonitoredNext;
189	Assert(idx != pPage->idx);
190	if (pPage->idx >= PGMPOOL_IDX_FIRST)
191	{
192	int rc2 = pgmPoolFlushPage(pPool, pPage);
193	if (rc2 == VERR_PGM_POOL_CLEARED && rc == VINF_SUCCESS)
194	rc = VINF_PGM_SYNC_CR3;
195	}
196	/* next */
197	if (idx == NIL_PGMPOOL_IDX)
198	break;
199	pPage = &pPool->aPages[idx];
200	}
201	return rc;
202	}
203
204
205	/**
206	* Wrapper for getting the current context pointer to the entry being modified.
207	*
208	* @returns VBox status code suitable for scheduling.
209	* @param pVM VM Handle.
210	* @param pvDst Destination address
211	* @param pvSrc Source guest virtual address.
212	* @param GCPhysSrc The source guest physical address.
213	* @param cb Size of data to read
214	*/
215	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVM pVM, void *pvDst, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
216	{
217	#if defined(IN_RING3)
218	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
219	return VINF_SUCCESS;
220	#else
221	/* @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
222	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
223	#endif
224	}
225
226	/**
227	* Process shadow entries before they are changed by the guest.
228	*
229	* For PT entries we will clear them. For PD entries, we'll simply check
230	* for mapping conflicts and set the SyncCR3 FF if found.
231	*
232	* @param pPool The pool.
233	* @param pPage The head page.
234	* @param GCPhysFault The guest physical fault address.
235	* @param uAddress In R0 and GC this is the guest context fault address (flat).
236	* In R3 this is the host context 'fault' address.
237	* @param pCpu The disassembler state for figuring out the write size.
238	* This need not be specified if the caller knows we won't do cross entry accesses.
239	*/
240	void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu)
241	{
242	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
243	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
244	const unsigned cbWrite = (pCpu) ? pgmPoolDisasWriteSize(pCpu) : 0;
245	PVM pVM = pPool->CTX_SUFF(pVM);
246
247	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp kind=%s cbWrite=%d\n", (RTGCPTR)pvAddress, GCPhysFault, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
248	for (;;)
249	{
250	union
251	{
252	void *pv;
253	PX86PT pPT;
254	PX86PTPAE pPTPae;
255	PX86PD pPD;
256	PX86PDPAE pPDPae;
257	PX86PDPT pPDPT;
258	PX86PML4 pPML4;
259	} uShw;
260
261	uShw.pv = NULL;
262	switch (pPage->enmKind)
263	{
264	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
265	{
266	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
267	const unsigned iShw = off / sizeof(X86PTE);
268	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
269	if (uShw.pPT->a[iShw].n.u1Present)
270	{
271	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
272	X86PTE GstPte;
273
274	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
275	AssertRC(rc);
276	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
277	pgmPoolTracDerefGCPhysHint(pPool, pPage,
278	uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
279	GstPte.u & X86_PTE_PG_MASK);
280	# endif
281	uShw.pPT->a[iShw].u = 0;
282	}
283	break;
284	}
285
286	/* page/2 sized */
287	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
288	{
289	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
290	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
291	{
292	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
293	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
294	if (uShw.pPTPae->a[iShw].n.u1Present)
295	{
296	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
297	X86PTE GstPte;
298	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
299	AssertRC(rc);
300
301	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
302	pgmPoolTracDerefGCPhysHint(pPool, pPage,
303	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
304	GstPte.u & X86_PTE_PG_MASK);
305	# endif
306	uShw.pPTPae->a[iShw].u = 0;
307	}
308	}
309	break;
310	}
311
312	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
313	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
314	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
315	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
316	{
317	unsigned iGst = off / sizeof(X86PDE);
318	unsigned iShwPdpt = iGst / 256;
319	unsigned iShw = (iGst % 256) * 2;
320	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
321
322	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
323	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
324	{
325	for (unsigned i = 0; i < 2; i++)
326	{
327	# ifndef IN_RING0
328	if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
329	{
330	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
331	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
332	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
333	break;
334	}
335	else
336	# endif /* !IN_RING0 */
337	if (uShw.pPDPae->a[iShw+i].n.u1Present)
338	{
339	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
340	pgmPoolFree(pVM,
341	uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
342	pPage->idx,
343	iShw + i);
344	uShw.pPDPae->a[iShw+i].u = 0;
345	}
346
347	/* paranoia / a bit assumptive. */
348	if ( pCpu
349	&& (off & 3)
350	&& (off & 3) + cbWrite > 4)
351	{
352	const unsigned iShw2 = iShw + 2 + i;
353	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
354	{
355	# ifndef IN_RING0
356	if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
357	{
358	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
359	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
360	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
361	break;
362	}
363	else
364	# endif /* !IN_RING0 */
365	if (uShw.pPDPae->a[iShw2].n.u1Present)
366	{
367	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
368	pgmPoolFree(pVM,
369	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
370	pPage->idx,
371	iShw2);
372	uShw.pPDPae->a[iShw2].u = 0;
373	}
374	}
375	}
376	}
377	}
378	break;
379	}
380
381	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
382	{
383	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
384	const unsigned iShw = off / sizeof(X86PTEPAE);
385	if (uShw.pPTPae->a[iShw].n.u1Present)
386	{
387	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
388	X86PTEPAE GstPte;
389	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
390	AssertRC(rc);
391
392	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
393	pgmPoolTracDerefGCPhysHint(pPool, pPage,
394	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
395	GstPte.u & X86_PTE_PAE_PG_MASK);
396	# endif
397	uShw.pPTPae->a[iShw].u = 0;
398	}
399
400	/* paranoia / a bit assumptive. */
401	if ( pCpu
402	&& (off & 7)
403	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
404	{
405	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
406	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
407
408	if (uShw.pPTPae->a[iShw2].n.u1Present)
409	{
410	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
411	X86PTEPAE GstPte;
412	# ifdef IN_RING3
413	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, (RTHCPTR)((RTHCUINTPTR)pvAddress + sizeof(GstPte)), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
414	# else
415	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress + sizeof(GstPte), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
416	# endif
417	AssertRC(rc);
418	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
419	pgmPoolTracDerefGCPhysHint(pPool, pPage,
420	uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK,
421	GstPte.u & X86_PTE_PAE_PG_MASK);
422	# endif
423	uShw.pPTPae->a[iShw2].u = 0;
424	}
425	}
426	break;
427	}
428
429	case PGMPOOLKIND_32BIT_PD:
430	{
431	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
432	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
433
434	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
435	# ifndef IN_RING0
436	if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
437	{
438	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
439	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
440	STAM_COUNTER_INC(&(pVM->pgm.s.StatRZGuestCR3WriteConflict));
441	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
442	break;
443	}
444	# endif /* !IN_RING0 */
445	# ifndef IN_RING0
446	else
447	# endif /* !IN_RING0 */
448	{
449	if (uShw.pPD->a[iShw].n.u1Present)
450	{
451	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
452	pgmPoolFree(pVM,
453	uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
454	pPage->idx,
455	iShw);
456	uShw.pPD->a[iShw].u = 0;
457	}
458	}
459	/* paranoia / a bit assumptive. */
460	if ( pCpu
461	&& (off & 3)
462	&& (off & 3) + cbWrite > sizeof(X86PTE))
463	{
464	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
465	if ( iShw2 != iShw
466	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
467	{
468	# ifndef IN_RING0
469	if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
470	{
471	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
472	STAM_COUNTER_INC(&(pVM->pgm.s.StatRZGuestCR3WriteConflict));
473	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
474	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
475	break;
476	}
477	# endif /* !IN_RING0 */
478	# ifndef IN_RING0
479	else
480	# endif /* !IN_RING0 */
481	{
482	if (uShw.pPD->a[iShw2].n.u1Present)
483	{
484	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
485	pgmPoolFree(pVM,
486	uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
487	pPage->idx,
488	iShw2);
489	uShw.pPD->a[iShw2].u = 0;
490	}
491	}
492	}
493	}
494	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
495	if ( uShw.pPD->a[iShw].n.u1Present
496	&& !VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
497	{
498	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
499	# ifdef IN_RC /* TLB load - we're pushing things a bit... */
500	ASMProbeReadByte(pvAddress);
501	# endif
502	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
503	uShw.pPD->a[iShw].u = 0;
504	}
505	#endif
506	break;
507	}
508
509	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
510	{
511	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
512	const unsigned iShw = off / sizeof(X86PDEPAE);
513	#ifndef IN_RING0
514	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
515	{
516	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
517	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
518	STAM_COUNTER_INC(&(pVM->pgm.s.StatRZGuestCR3WriteConflict));
519	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
520	break;
521	}
522	#endif /* !IN_RING0 */
523	/*
524	* Causes trouble when the guest uses a PDE to refer to the whole page table level
525	* structure. (Invalidate here; faults later on when it tries to change the page
526	* table entries -> recheck; probably only applies to the RC case.)
527	*/
528	# ifndef IN_RING0
529	else
530	# endif /* !IN_RING0 */
531	{
532	if (uShw.pPDPae->a[iShw].n.u1Present)
533	{
534	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
535	pgmPoolFree(pVM,
536	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
537	pPage->idx,
538	iShw);
539	uShw.pPDPae->a[iShw].u = 0;
540	}
541	}
542	/* paranoia / a bit assumptive. */
543	if ( pCpu
544	&& (off & 7)
545	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
546	{
547	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
548	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
549
550	#ifndef IN_RING0
551	if ( iShw2 != iShw
552	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
553	{
554	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
555	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
556	STAM_COUNTER_INC(&(pVM->pgm.s.StatRZGuestCR3WriteConflict));
557	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
558	break;
559	}
560	#endif /* !IN_RING0 */
561	# ifndef IN_RING0
562	else
563	# endif /* !IN_RING0 */
564	if (uShw.pPDPae->a[iShw2].n.u1Present)
565	{
566	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
567	pgmPoolFree(pVM,
568	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
569	pPage->idx,
570	iShw2);
571	uShw.pPDPae->a[iShw2].u = 0;
572	}
573	}
574	break;
575	}
576
577	case PGMPOOLKIND_PAE_PDPT:
578	{
579	/*
580	* Hopefully this doesn't happen very often:
581	* - touching unused parts of the page
582	* - messing with the bits of pd pointers without changing the physical address
583	*/
584	/* PDPT roots are not page aligned; 32 byte only! */
585	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
586
587	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
588	const unsigned iShw = offPdpt / sizeof(X86PDPE);
589	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
590	{
591	# ifndef IN_RING0
592	if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
593	{
594	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
595	STAM_COUNTER_INC(&(pVM->pgm.s.StatRZGuestCR3WriteConflict));
596	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
597	LogFlow(("pgmPoolMonitorChainChanging: Detected pdpt conflict at iShw=%#x!\n", iShw));
598	break;
599	}
600	# endif /* !IN_RING0 */
601	# ifndef IN_RING0
602	else
603	# endif /* !IN_RING0 */
604	if (uShw.pPDPT->a[iShw].n.u1Present)
605	{
606	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
607	pgmPoolFree(pVM,
608	uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
609	pPage->idx,
610	iShw);
611	uShw.pPDPT->a[iShw].u = 0;
612	}
613
614	/* paranoia / a bit assumptive. */
615	if ( pCpu
616	&& (offPdpt & 7)
617	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
618	{
619	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
620	if ( iShw2 != iShw
621	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
622	{
623	# ifndef IN_RING0
624	if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
625	{
626	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
627	STAM_COUNTER_INC(&(pVM->pgm.s.StatRZGuestCR3WriteConflict));
628	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
629	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
630	break;
631	}
632	# endif /* !IN_RING0 */
633	# ifndef IN_RING0
634	else
635	# endif /* !IN_RING0 */
636	if (uShw.pPDPT->a[iShw2].n.u1Present)
637	{
638	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
639	pgmPoolFree(pVM,
640	uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
641	pPage->idx,
642	iShw2);
643	uShw.pPDPT->a[iShw2].u = 0;
644	}
645	}
646	}
647	}
648	break;
649	}
650
651	#ifndef IN_RC
652	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
653	{
654	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
655	const unsigned iShw = off / sizeof(X86PDEPAE);
656	Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
657	if (uShw.pPDPae->a[iShw].n.u1Present)
658	{
659	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
660	pgmPoolFree(pVM,
661	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
662	pPage->idx,
663	iShw);
664	uShw.pPDPae->a[iShw].u = 0;
665	}
666	/* paranoia / a bit assumptive. */
667	if ( pCpu
668	&& (off & 7)
669	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
670	{
671	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
672	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
673
674	Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
675	if (uShw.pPDPae->a[iShw2].n.u1Present)
676	{
677	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
678	pgmPoolFree(pVM,
679	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
680	pPage->idx,
681	iShw2);
682	uShw.pPDPae->a[iShw2].u = 0;
683	}
684	}
685	break;
686	}
687
688	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
689	{
690	/*
691	* Hopefully this doesn't happen very often:
692	* - messing with the bits of pd pointers without changing the physical address
693	*/
694	if (!VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
695	{
696	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
697	const unsigned iShw = off / sizeof(X86PDPE);
698	if (uShw.pPDPT->a[iShw].n.u1Present)
699	{
700	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
701	pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
702	uShw.pPDPT->a[iShw].u = 0;
703	}
704	/* paranoia / a bit assumptive. */
705	if ( pCpu
706	&& (off & 7)
707	&& (off & 7) + cbWrite > sizeof(X86PDPE))
708	{
709	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
710	if (uShw.pPDPT->a[iShw2].n.u1Present)
711	{
712	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
713	pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
714	uShw.pPDPT->a[iShw2].u = 0;
715	}
716	}
717	}
718	break;
719	}
720
721	case PGMPOOLKIND_64BIT_PML4:
722	{
723	/*
724	* Hopefully this doesn't happen very often:
725	* - messing with the bits of pd pointers without changing the physical address
726	*/
727	if (!VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
728	{
729	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
730	const unsigned iShw = off / sizeof(X86PDPE);
731	if (uShw.pPML4->a[iShw].n.u1Present)
732	{
733	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
734	pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
735	uShw.pPML4->a[iShw].u = 0;
736	}
737	/* paranoia / a bit assumptive. */
738	if ( pCpu
739	&& (off & 7)
740	&& (off & 7) + cbWrite > sizeof(X86PDPE))
741	{
742	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
743	if (uShw.pPML4->a[iShw2].n.u1Present)
744	{
745	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
746	pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
747	uShw.pPML4->a[iShw2].u = 0;
748	}
749	}
750	}
751	break;
752	}
753	#endif /* IN_RING0 */
754
755	default:
756	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
757	}
758	PGMPOOL_UNLOCK_PTR(pVM, uShw.pv);
759
760	/* next */
761	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
762	return;
763	pPage = &pPool->aPages[pPage->iMonitoredNext];
764	}
765	}
766
767	# ifndef IN_RING3
768	/**
769	* Checks if a access could be a fork operation in progress.
770	*
771	* Meaning, that the guest is setuping up the parent process for Copy-On-Write.
772	*
773	* @returns true if it's likly that we're forking, otherwise false.
774	* @param pPool The pool.
775	* @param pCpu The disassembled instruction.
776	* @param offFault The access offset.
777	*/
778	DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pCpu, unsigned offFault)
779	{
780	/*
781	* i386 linux is using btr to clear X86_PTE_RW.
782	* The functions involved are (2.6.16 source inspection):
783	* clear_bit
784	* ptep_set_wrprotect
785	* copy_one_pte
786	* copy_pte_range
787	* copy_pmd_range
788	* copy_pud_range
789	* copy_page_range
790	* dup_mmap
791	* dup_mm
792	* copy_mm
793	* copy_process
794	* do_fork
795	*/
796	if ( pCpu->pCurInstr->opcode == OP_BTR
797	&& !(offFault & 4)
798	/** @todo Validate that the bit index is X86_PTE_RW. */
799	)
800	{
801	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,Fork));
802	return true;
803	}
804	return false;
805	}
806
807
808	/**
809	* Determine whether the page is likely to have been reused.
810	*
811	* @returns true if we consider the page as being reused for a different purpose.
812	* @returns false if we consider it to still be a paging page.
813	* @param pVM VM Handle.
814	* @param pPage The page in question.
815	* @param pRegFrame Trap register frame.
816	* @param pCpu The disassembly info for the faulting instruction.
817	* @param pvFault The fault address.
818	*
819	* @remark The REP prefix check is left to the caller because of STOSD/W.
820	*/
821	DECLINLINE(bool) pgmPoolMonitorIsReused(PVM pVM, PPGMPOOLPAGE pPage, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pCpu, RTGCPTR pvFault)
822	{
823	#ifndef IN_RC
824	/** @todo could make this general, faulting close to rsp should be safe reuse heuristic. */
825	if ( HWACCMHasPendingIrq(pVM)
826	&& (pRegFrame->rsp - pvFault) < 32)
827	{
828	/* Fault caused by stack writes while trying to inject an interrupt event. */
829	Log(("pgmPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
830	return true;
831	}
832	#else
833	NOREF(pVM); NOREF(pvFault);
834	#endif
835
836	switch (pCpu->pCurInstr->opcode)
837	{
838	/* call implies the actual push of the return address faulted */
839	case OP_CALL:
840	Log4(("pgmPoolMonitorIsReused: CALL\n"));
841	return true;
842	case OP_PUSH:
843	Log4(("pgmPoolMonitorIsReused: PUSH\n"));
844	return true;
845	case OP_PUSHF:
846	Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
847	return true;
848	case OP_PUSHA:
849	Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
850	return true;
851	case OP_FXSAVE:
852	Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
853	return true;
854	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
855	Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
856	return true;
857	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
858	Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
859	return true;
860	case OP_MOVSWD:
861	case OP_STOSWD:
862	if ( pCpu->prefix == (PREFIX_REP\|PREFIX_REX)
863	&& pRegFrame->rcx >= 0x40
864	)
865	{
866	Assert(pCpu->mode == CPUMODE_64BIT);
867
868	Log(("pgmPoolMonitorIsReused: OP_STOSQ\n"));
869	return true;
870	}
871	return false;
872	}
873	if ( (pCpu->param1.flags & USE_REG_GEN32)
874	&& (pCpu->param1.base.reg_gen == USE_REG_ESP))
875	{
876	Log4(("pgmPoolMonitorIsReused: ESP\n"));
877	return true;
878	}
879
880	//if (pPage->fCR3Mix)
881	// return false;
882	return false;
883	}
884
885
886	/**
887	* Flushes the page being accessed.
888	*
889	* @returns VBox status code suitable for scheduling.
890	* @param pVM The VM handle.
891	* @param pPool The pool.
892	* @param pPage The pool page (head).
893	* @param pCpu The disassembly of the write instruction.
894	* @param pRegFrame The trap register frame.
895	* @param GCPhysFault The fault address as guest physical address.
896	* @param pvFault The fault address.
897	*/
898	static int pgmPoolAccessHandlerFlush(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
899	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
900	{
901	/*
902	* First, do the flushing.
903	*/
904	int rc = pgmPoolMonitorChainFlush(pPool, pPage);
905
906	/*
907	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
908	*/
909	uint32_t cbWritten;
910	int rc2 = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cbWritten);
911	if (RT_SUCCESS(rc2))
912	pRegFrame->rip += pCpu->opsize;
913	else if (rc2 == VERR_EM_INTERPRETER)
914	{
915	#ifdef IN_RC
916	if (PATMIsPatchGCAddr(pVM, (RTRCPTR)pRegFrame->eip))
917	{
918	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
919	pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
920	rc = VINF_SUCCESS;
921	STAM_COUNTER_INC(&pPool->StatMonitorRZIntrFailPatch2);
922	}
923	else
924	#endif
925	{
926	rc = VINF_EM_RAW_EMULATE_INSTR;
927	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
928	}
929	}
930	else
931	rc = rc2;
932
933	/* See use in pgmPoolAccessHandlerSimple(). */
934	PGM_INVL_GUEST_TLBS();
935
936	LogFlow(("pgmPoolAccessHandlerPT: returns %Rrc (flushed)\n", rc));
937	return rc;
938
939	}
940
941
942	/**
943	* Handles the STOSD write accesses.
944	*
945	* @returns VBox status code suitable for scheduling.
946	* @param pVM The VM handle.
947	* @param pPool The pool.
948	* @param pPage The pool page (head).
949	* @param pCpu The disassembly of the write instruction.
950	* @param pRegFrame The trap register frame.
951	* @param GCPhysFault The fault address as guest physical address.
952	* @param pvFault The fault address.
953	*/
954	DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
955	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
956	{
957	Assert(pCpu->mode == CPUMODE_32BIT);
958
959	Log3(("pgmPoolAccessHandlerSTOSD\n"));
960
961	/*
962	* Increment the modification counter and insert it into the list
963	* of modified pages the first time.
964	*/
965	if (!pPage->cModifications++)
966	pgmPoolMonitorModifiedInsert(pPool, pPage);
967
968	/*
969	* Execute REP STOSD.
970	*
971	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
972	* write situation, meaning that it's safe to write here.
973	*/
974	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
975	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
976	#endif
977	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
978	while (pRegFrame->ecx)
979	{
980	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
981	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
982	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
983	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
984	#else
985	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
986	#endif
987	#ifdef IN_RC
988	(uint32_t )pu32 = pRegFrame->eax;
989	#else
990	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->eax, 4);
991	#endif
992	pu32 += 4;
993	GCPhysFault += 4;
994	pRegFrame->edi += 4;
995	pRegFrame->ecx--;
996	}
997	pRegFrame->rip += pCpu->opsize;
998
999	#ifdef IN_RC
1000	/* See use in pgmPoolAccessHandlerSimple(). */
1001	PGM_INVL_GUEST_TLBS();
1002	#endif
1003
1004	LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
1005	return VINF_SUCCESS;
1006	}
1007
1008
1009	/**
1010	* Handles the simple write accesses.
1011	*
1012	* @returns VBox status code suitable for scheduling.
1013	* @param pVM The VM handle.
1014	* @param pPool The pool.
1015	* @param pPage The pool page (head).
1016	* @param pCpu The disassembly of the write instruction.
1017	* @param pRegFrame The trap register frame.
1018	* @param GCPhysFault The fault address as guest physical address.
1019	* @param pvFault The fault address.
1020	*/
1021	DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
1022	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
1023	{
1024	Log3(("pgmPoolAccessHandlerSimple\n"));
1025	/*
1026	* Increment the modification counter and insert it into the list
1027	* of modified pages the first time.
1028	*/
1029	if (!pPage->cModifications++)
1030	pgmPoolMonitorModifiedInsert(pPool, pPage);
1031
1032	/*
1033	* Clear all the pages. ASSUMES that pvFault is readable.
1034	*/
1035	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1036	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
1037	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1038	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pvFault, pCpu);
1039	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1040	#else
1041	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pvFault, pCpu);
1042	#endif
1043
1044	/*
1045	* Interpret the instruction.
1046	*/
1047	uint32_t cb;
1048	int rc = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cb);
1049	if (RT_SUCCESS(rc))
1050	pRegFrame->rip += pCpu->opsize;
1051	else if (rc == VERR_EM_INTERPRETER)
1052	{
1053	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for %04x:%RGv - opcode=%d\n",
1054	pRegFrame->cs, (RTGCPTR)pRegFrame->rip, pCpu->pCurInstr->opcode));
1055	rc = VINF_EM_RAW_EMULATE_INSTR;
1056	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
1057	}
1058
1059	#ifdef IN_RC
1060	/*
1061	* Quick hack, with logging enabled we're getting stale
1062	* code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
1063	* Flushing here is BAD and expensive, I think EMInterpretDisasOne will
1064	* have to be fixed to support this. But that'll have to wait till next week.
1065	*
1066	* An alternative is to keep track of the changed PTEs together with the
1067	* GCPhys from the guest PT. This may proove expensive though.
1068	*
1069	* At the moment, it's VITAL that it's done AFTER the instruction interpreting
1070	* because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
1071	*/
1072	PGM_INVL_GUEST_TLBS();
1073	#endif
1074
1075	LogFlow(("pgmPoolAccessHandlerSimple: returns %Rrc cb=%d\n", rc, cb));
1076	return rc;
1077	}
1078
1079	/**
1080	* \#PF Handler callback for PT write accesses.
1081	*
1082	* @returns VBox status code (appropriate for GC return).
1083	* @param pVM VM Handle.
1084	* @param uErrorCode CPU Error code.
1085	* @param pRegFrame Trap register frame.
1086	* NULL on DMA and other non CPU access.
1087	* @param pvFault The fault address (cr2).
1088	* @param GCPhysFault The GC physical address corresponding to pvFault.
1089	* @param pvUser User argument.
1090	*/
1091	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1092	{
1093	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1094	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1095	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1096	LogFlow(("pgmPoolAccessHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1097
1098	/*
1099	* We should ALWAYS have the list head as user parameter. This
1100	* is because we use that page to record the changes.
1101	*/
1102	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1103
1104	/*
1105	* Disassemble the faulting instruction.
1106	*/
1107	DISCPUSTATE Cpu;
1108	int rc = EMInterpretDisasOne(pVM, pRegFrame, &Cpu, NULL);
1109	AssertRCReturn(rc, rc);
1110
1111	/*
1112	* Check if it's worth dealing with.
1113	*/
1114	bool fReused = false;
1115	if ( ( pPage->cModifications < 48 /** @todo #define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1116	\|\| pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
1117	)
1118	&& !(fReused = pgmPoolMonitorIsReused(pVM, pPage, pRegFrame, &Cpu, pvFault))
1119	&& !pgmPoolMonitorIsForking(pPool, &Cpu, GCPhysFault & PAGE_OFFSET_MASK))
1120	{
1121	/*
1122	* Simple instructions, no REP prefix.
1123	*/
1124	if (!(Cpu.prefix & (PREFIX_REP \| PREFIX_REPNE)))
1125	{
1126	rc = pgmPoolAccessHandlerSimple(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1127	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1128	return rc;
1129	}
1130
1131	/*
1132	* Windows is frequently doing small memset() operations (netio test 4k+).
1133	* We have to deal with these or we'll kill the cache and performance.
1134	*/
1135	if ( Cpu.pCurInstr->opcode == OP_STOSWD
1136	&& CPUMGetGuestCPL(pVM, pRegFrame) == 0
1137	&& pRegFrame->ecx <= 0x20
1138	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1139	&& !((uintptr_t)pvFault & 3)
1140	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1141	&& Cpu.mode == CPUMODE_32BIT
1142	&& Cpu.opmode == CPUMODE_32BIT
1143	&& Cpu.addrmode == CPUMODE_32BIT
1144	&& Cpu.prefix == PREFIX_REP
1145	&& !pRegFrame->eflags.Bits.u1DF
1146	)
1147	{
1148	rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1149	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,RepStosd), a);
1150	return rc;
1151	}
1152
1153	/* REP prefix, don't bother. */
1154	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,RepPrefix));
1155	Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1156	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, Cpu.pCurInstr->opcode, Cpu.prefix));
1157	}
1158
1159	/*
1160	* Not worth it, so flush it.
1161	*
1162	* If we considered it to be reused, don't to back to ring-3
1163	* to emulate failed instructions since we usually cannot
1164	* interpret then. This may be a bit risky, in which case
1165	* the reuse detection must be fixed.
1166	*/
1167	rc = pgmPoolAccessHandlerFlush(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1168	if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
1169	rc = VINF_SUCCESS;
1170	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1171	return rc;
1172	}
1173
1174	# endif /* !IN_RING3 */
1175	#endif /* PGMPOOL_WITH_MONITORING */
1176
1177	#ifdef PGMPOOL_WITH_CACHE
1178
1179	/**
1180	* Inserts a page into the GCPhys hash table.
1181	*
1182	* @param pPool The pool.
1183	* @param pPage The page.
1184	*/
1185	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1186	{
1187	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1188	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1189	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1190	pPage->iNext = pPool->aiHash[iHash];
1191	pPool->aiHash[iHash] = pPage->idx;
1192	}
1193
1194
1195	/**
1196	* Removes a page from the GCPhys hash table.
1197	*
1198	* @param pPool The pool.
1199	* @param pPage The page.
1200	*/
1201	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1202	{
1203	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1204	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1205	if (pPool->aiHash[iHash] == pPage->idx)
1206	pPool->aiHash[iHash] = pPage->iNext;
1207	else
1208	{
1209	uint16_t iPrev = pPool->aiHash[iHash];
1210	for (;;)
1211	{
1212	const int16_t i = pPool->aPages[iPrev].iNext;
1213	if (i == pPage->idx)
1214	{
1215	pPool->aPages[iPrev].iNext = pPage->iNext;
1216	break;
1217	}
1218	if (i == NIL_PGMPOOL_IDX)
1219	{
1220	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%#x\n", pPage->GCPhys, pPage->idx));
1221	break;
1222	}
1223	iPrev = i;
1224	}
1225	}
1226	pPage->iNext = NIL_PGMPOOL_IDX;
1227	}
1228
1229
1230	/**
1231	* Frees up one cache page.
1232	*
1233	* @returns VBox status code.
1234	* @retval VINF_SUCCESS on success.
1235	* @retval VERR_PGM_POOL_CLEARED if the deregistration of a physical handler will cause a light weight pool flush.
1236	* @param pPool The pool.
1237	* @param iUser The user index.
1238	*/
1239	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1240	{
1241	#ifndef IN_RC
1242	const PVM pVM = pPool->CTX_SUFF(pVM);
1243	#endif
1244	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1245	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1246
1247	/*
1248	* Select one page from the tail of the age list.
1249	*/
1250	uint16_t iToFree = pPool->iAgeTail;
1251	if (iToFree == iUser)
1252	iToFree = pPool->aPages[iToFree].iAgePrev;
1253	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
1254	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
1255	{
1256	uint16_t i = pPool->aPages[iToFree].iAgePrev;
1257	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
1258	{
1259	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
1260	continue;
1261	iToFree = i;
1262	break;
1263	}
1264	}
1265	*/
1266
1267	Assert(iToFree != iUser);
1268	AssertRelease(iToFree != NIL_PGMPOOL_IDX);
1269
1270	PPGMPOOLPAGE pPage = &pPool->aPages[iToFree];
1271
1272	/*
1273	* Reject any attempts at flushing the currently active shadow CR3 mapping
1274	*/
1275	if (pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage))
1276	{
1277	/* Refresh the cr3 mapping by putting it at the head of the age list. */
1278	LogFlow(("pgmPoolCacheFreeOne refuse CR3 mapping\n"));
1279	pgmPoolCacheUsed(pPool, pPage);
1280	return pgmPoolCacheFreeOne(pPool, iUser);
1281	}
1282
1283	int rc = pgmPoolFlushPage(pPool, pPage);
1284	if (rc == VINF_SUCCESS)
1285	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
1286	return rc;
1287	}
1288
1289
1290	/**
1291	* Checks if a kind mismatch is really a page being reused
1292	* or if it's just normal remappings.
1293	*
1294	* @returns true if reused and the cached page (enmKind1) should be flushed
1295	* @returns false if not reused.
1296	* @param enmKind1 The kind of the cached page.
1297	* @param enmKind2 The kind of the requested page.
1298	*/
1299	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
1300	{
1301	switch (enmKind1)
1302	{
1303	/*
1304	* Never reuse them. There is no remapping in non-paging mode.
1305	*/
1306	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1307	case PGMPOOLKIND_32BIT_PD_PHYS:
1308	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1309	case PGMPOOLKIND_PAE_PD_PHYS:
1310	case PGMPOOLKIND_PAE_PDPT_PHYS:
1311	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1312	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1313	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1314	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1315	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1316	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
1317	return false;
1318
1319	/*
1320	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1321	*/
1322	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1323	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1324	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1325	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1326	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1327	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1328	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1329	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1330	case PGMPOOLKIND_32BIT_PD:
1331	case PGMPOOLKIND_PAE_PDPT:
1332	switch (enmKind2)
1333	{
1334	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1335	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1336	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1337	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1338	case PGMPOOLKIND_64BIT_PML4:
1339	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1340	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1341	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1342	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1343	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1344	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1345	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1346	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1347	return true;
1348	default:
1349	return false;
1350	}
1351
1352	/*
1353	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1354	*/
1355	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1356	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1357	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1358	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1359	case PGMPOOLKIND_64BIT_PML4:
1360	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1361	switch (enmKind2)
1362	{
1363	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1364	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1365	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1366	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1367	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1368	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1369	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1370	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1371	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1372	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1373	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1374	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1375	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1376	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1377	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1378	return true;
1379	default:
1380	return false;
1381	}
1382
1383	/*
1384	* These cannot be flushed, and it's common to reuse the PDs as PTs.
1385	*/
1386	case PGMPOOLKIND_ROOT_NESTED:
1387	return false;
1388
1389	default:
1390	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1391	}
1392	}
1393
1394
1395	/**
1396	* Attempts to satisfy a pgmPoolAlloc request from the cache.
1397	*
1398	* @returns VBox status code.
1399	* @retval VINF_PGM_CACHED_PAGE on success.
1400	* @retval VERR_FILE_NOT_FOUND if not found.
1401	* @param pPool The pool.
1402	* @param GCPhys The GC physical address of the page we're gonna shadow.
1403	* @param enmKind The kind of mapping.
1404	* @param iUser The shadow page pool index of the user table.
1405	* @param iUserTable The index into the user table (shadowed).
1406	* @param ppPage Where to store the pointer to the page.
1407	*/
1408	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
1409	{
1410	#ifndef IN_RC
1411	const PVM pVM = pPool->CTX_SUFF(pVM);
1412	#endif
1413	/*
1414	* Look up the GCPhys in the hash.
1415	*/
1416	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1417	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%x iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
1418	if (i != NIL_PGMPOOL_IDX)
1419	{
1420	do
1421	{
1422	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1423	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
1424	if (pPage->GCPhys == GCPhys)
1425	{
1426	if ((PGMPOOLKIND)pPage->enmKind == enmKind)
1427	{
1428	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
1429	* doesn't flush it in case there are no more free use records.
1430	*/
1431	pgmPoolCacheUsed(pPool, pPage);
1432
1433	int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1434	if (RT_SUCCESS(rc))
1435	{
1436	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
1437	*ppPage = pPage;
1438	STAM_COUNTER_INC(&pPool->StatCacheHits);
1439	return VINF_PGM_CACHED_PAGE;
1440	}
1441	return rc;
1442	}
1443
1444	/*
1445	* The kind is different. In some cases we should now flush the page
1446	* as it has been reused, but in most cases this is normal remapping
1447	* of PDs as PT or big pages using the GCPhys field in a slightly
1448	* different way than the other kinds.
1449	*/
1450	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1451	{
1452	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1453	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED will be returned by pgmPoolTracInsert. */
1454	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
1455	break;
1456	}
1457	}
1458
1459	/* next */
1460	i = pPage->iNext;
1461	} while (i != NIL_PGMPOOL_IDX);
1462	}
1463
1464	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
1465	STAM_COUNTER_INC(&pPool->StatCacheMisses);
1466	return VERR_FILE_NOT_FOUND;
1467	}
1468
1469
1470	/**
1471	* Inserts a page into the cache.
1472	*
1473	* @param pPool The pool.
1474	* @param pPage The cached page.
1475	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1476	*/
1477	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1478	{
1479	/*
1480	* Insert into the GCPhys hash if the page is fit for that.
1481	*/
1482	Assert(!pPage->fCached);
1483	if (fCanBeCached)
1484	{
1485	pPage->fCached = true;
1486	pgmPoolHashInsert(pPool, pPage);
1487	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1488	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1489	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1490	}
1491	else
1492	{
1493	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1494	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1495	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1496	}
1497
1498	/*
1499	* Insert at the head of the age list.
1500	*/
1501	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1502	pPage->iAgeNext = pPool->iAgeHead;
1503	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1504	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1505	else
1506	pPool->iAgeTail = pPage->idx;
1507	pPool->iAgeHead = pPage->idx;
1508	}
1509
1510
1511	/**
1512	* Flushes a cached page.
1513	*
1514	* @param pPool The pool.
1515	* @param pPage The cached page.
1516	*/
1517	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1518	{
1519	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
1520
1521	/*
1522	* Remove the page from the hash.
1523	*/
1524	if (pPage->fCached)
1525	{
1526	pPage->fCached = false;
1527	pgmPoolHashRemove(pPool, pPage);
1528	}
1529	else
1530	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1531
1532	/*
1533	* Remove it from the age list.
1534	*/
1535	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1536	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1537	else
1538	pPool->iAgeTail = pPage->iAgePrev;
1539	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1540	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1541	else
1542	pPool->iAgeHead = pPage->iAgeNext;
1543	pPage->iAgeNext = NIL_PGMPOOL_IDX;
1544	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1545	}
1546
1547	#endif /* PGMPOOL_WITH_CACHE */
1548	#ifdef PGMPOOL_WITH_MONITORING
1549
1550	/**
1551	* Looks for pages sharing the monitor.
1552	*
1553	* @returns Pointer to the head page.
1554	* @returns NULL if not found.
1555	* @param pPool The Pool
1556	* @param pNewPage The page which is going to be monitored.
1557	*/
1558	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1559	{
1560	#ifdef PGMPOOL_WITH_CACHE
1561	/*
1562	* Look up the GCPhys in the hash.
1563	*/
1564	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1565	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1566	if (i == NIL_PGMPOOL_IDX)
1567	return NULL;
1568	do
1569	{
1570	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1571	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1572	&& pPage != pNewPage)
1573	{
1574	switch (pPage->enmKind)
1575	{
1576	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1577	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1578	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1579	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1580	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1581	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1582	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1583	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1584	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1585	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1586	case PGMPOOLKIND_64BIT_PML4:
1587	case PGMPOOLKIND_32BIT_PD:
1588	case PGMPOOLKIND_PAE_PDPT:
1589	{
1590	/* find the head */
1591	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1592	{
1593	Assert(pPage->iMonitoredPrev != pPage->idx);
1594	pPage = &pPool->aPages[pPage->iMonitoredPrev];
1595	}
1596	return pPage;
1597	}
1598
1599	/* ignore, no monitoring. */
1600	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1601	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1602	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1603	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1604	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1605	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1606	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1607	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1608	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1609	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1610	case PGMPOOLKIND_ROOT_NESTED:
1611	case PGMPOOLKIND_PAE_PD_PHYS:
1612	case PGMPOOLKIND_PAE_PDPT_PHYS:
1613	case PGMPOOLKIND_32BIT_PD_PHYS:
1614	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
1615	break;
1616	default:
1617	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
1618	}
1619	}
1620
1621	/* next */
1622	i = pPage->iNext;
1623	} while (i != NIL_PGMPOOL_IDX);
1624	#endif
1625	return NULL;
1626	}
1627
1628
1629	/**
1630	* Enabled write monitoring of a guest page.
1631	*
1632	* @returns VBox status code.
1633	* @retval VINF_SUCCESS on success.
1634	* @retval VERR_PGM_POOL_CLEARED if the registration of the physical handler will cause a light weight pool flush.
1635	* @param pPool The pool.
1636	* @param pPage The cached page.
1637	*/
1638	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1639	{
1640	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
1641
1642	/*
1643	* Filter out the relevant kinds.
1644	*/
1645	switch (pPage->enmKind)
1646	{
1647	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1648	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1649	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1650	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1651	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1652	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1653	case PGMPOOLKIND_64BIT_PML4:
1654	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1655	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1656	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1657	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1658	case PGMPOOLKIND_32BIT_PD:
1659	case PGMPOOLKIND_PAE_PDPT:
1660	break;
1661
1662	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1663	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1664	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1665	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1666	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1667	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1668	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1669	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1670	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1671	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1672	case PGMPOOLKIND_ROOT_NESTED:
1673	/* Nothing to monitor here. */
1674	return VINF_SUCCESS;
1675
1676	case PGMPOOLKIND_32BIT_PD_PHYS:
1677	case PGMPOOLKIND_PAE_PDPT_PHYS:
1678	case PGMPOOLKIND_PAE_PD_PHYS:
1679	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
1680	/* Nothing to monitor here. */
1681	return VINF_SUCCESS;
1682	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1683	break;
1684	#else
1685	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1686	#endif
1687	default:
1688	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1689	}
1690
1691	/*
1692	* Install handler.
1693	*/
1694	int rc;
1695	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
1696	if (pPageHead)
1697	{
1698	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
1699	Assert(pPageHead->iMonitoredPrev != pPage->idx);
1700	pPage->iMonitoredPrev = pPageHead->idx;
1701	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
1702	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
1703	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
1704	pPageHead->iMonitoredNext = pPage->idx;
1705	rc = VINF_SUCCESS;
1706	}
1707	else
1708	{
1709	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1710	PVM pVM = pPool->CTX_SUFF(pVM);
1711	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1712	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
1713	GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
1714	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
1715	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
1716	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
1717	pPool->pszAccessHandler);
1718	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
1719	* the heap size should suffice. */
1720	AssertFatalRC(rc);
1721	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1722	rc = VERR_PGM_POOL_CLEARED;
1723	}
1724	pPage->fMonitored = true;
1725	return rc;
1726	}
1727
1728
1729	/**
1730	* Disables write monitoring of a guest page.
1731	*
1732	* @returns VBox status code.
1733	* @retval VINF_SUCCESS on success.
1734	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1735	* @param pPool The pool.
1736	* @param pPage The cached page.
1737	*/
1738	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1739	{
1740	/*
1741	* Filter out the relevant kinds.
1742	*/
1743	switch (pPage->enmKind)
1744	{
1745	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1746	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1747	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1748	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1749	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1750	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1751	case PGMPOOLKIND_64BIT_PML4:
1752	case PGMPOOLKIND_32BIT_PD:
1753	case PGMPOOLKIND_PAE_PDPT:
1754	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1755	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1756	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1757	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1758	break;
1759
1760	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1761	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1762	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1763	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1764	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1765	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1766	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1767	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1768	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1769	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1770	case PGMPOOLKIND_ROOT_NESTED:
1771	case PGMPOOLKIND_PAE_PD_PHYS:
1772	case PGMPOOLKIND_PAE_PDPT_PHYS:
1773	case PGMPOOLKIND_32BIT_PD_PHYS:
1774	/* Nothing to monitor here. */
1775	return VINF_SUCCESS;
1776
1777	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1778	break;
1779	#endif
1780	default:
1781	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1782	}
1783
1784	/*
1785	* Remove the page from the monitored list or uninstall it if last.
1786	*/
1787	const PVM pVM = pPool->CTX_SUFF(pVM);
1788	int rc;
1789	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1790	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1791	{
1792	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1793	{
1794	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
1795	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
1796	rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
1797	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
1798	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
1799	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pNewHead),
1800	pPool->pszAccessHandler);
1801	AssertFatalRCSuccess(rc);
1802	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1803	}
1804	else
1805	{
1806	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
1807	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
1808	{
1809	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
1810	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1811	}
1812	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
1813	rc = VINF_SUCCESS;
1814	}
1815	}
1816	else
1817	{
1818	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
1819	AssertFatalRC(rc);
1820	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1821	rc = VERR_PGM_POOL_CLEARED;
1822	}
1823	pPage->fMonitored = false;
1824
1825	/*
1826	* Remove it from the list of modified pages (if in it).
1827	*/
1828	pgmPoolMonitorModifiedRemove(pPool, pPage);
1829
1830	return rc;
1831	}
1832
1833
1834	/**
1835	* Inserts the page into the list of modified pages.
1836	*
1837	* @param pPool The pool.
1838	* @param pPage The page.
1839	*/
1840	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1841	{
1842	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
1843	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
1844	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
1845	&& pPool->iModifiedHead != pPage->idx,
1846	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
1847	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
1848	pPool->iModifiedHead, pPool->cModifiedPages));
1849
1850	pPage->iModifiedNext = pPool->iModifiedHead;
1851	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
1852	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
1853	pPool->iModifiedHead = pPage->idx;
1854	pPool->cModifiedPages++;
1855	#ifdef VBOX_WITH_STATISTICS
1856	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
1857	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
1858	#endif
1859	}
1860
1861
1862	/**
1863	* Removes the page from the list of modified pages and resets the
1864	* moficiation counter.
1865	*
1866	* @param pPool The pool.
1867	* @param pPage The page which is believed to be in the list of modified pages.
1868	*/
1869	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1870	{
1871	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
1872	if (pPool->iModifiedHead == pPage->idx)
1873	{
1874	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1875	pPool->iModifiedHead = pPage->iModifiedNext;
1876	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1877	{
1878	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
1879	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1880	}
1881	pPool->cModifiedPages--;
1882	}
1883	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
1884	{
1885	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
1886	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1887	{
1888	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
1889	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1890	}
1891	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1892	pPool->cModifiedPages--;
1893	}
1894	else
1895	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1896	pPage->cModifications = 0;
1897	}
1898
1899
1900	/**
1901	* Zaps the list of modified pages, resetting their modification counters in the process.
1902	*
1903	* @param pVM The VM handle.
1904	*/
1905	void pgmPoolMonitorModifiedClearAll(PVM pVM)
1906	{
1907	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1908	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
1909
1910	unsigned cPages = 0; NOREF(cPages);
1911	uint16_t idx = pPool->iModifiedHead;
1912	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1913	while (idx != NIL_PGMPOOL_IDX)
1914	{
1915	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
1916	idx = pPage->iModifiedNext;
1917	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1918	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1919	pPage->cModifications = 0;
1920	Assert(++cPages);
1921	}
1922	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
1923	pPool->cModifiedPages = 0;
1924	}
1925
1926
1927	#ifdef IN_RING3
1928	/**
1929	* Clear all shadow pages and clear all modification counters.
1930	*
1931	* @param pVM The VM handle.
1932	* @remark Should only be used when monitoring is available, thus placed in
1933	* the PGMPOOL_WITH_MONITORING #ifdef.
1934	*/
1935	void pgmPoolClearAll(PVM pVM)
1936	{
1937	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1938	STAM_PROFILE_START(&pPool->StatClearAll, c);
1939	LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
1940
1941	/*
1942	* Iterate all the pages until we've encountered all that in use.
1943	* This is simple but not quite optimal solution.
1944	*/
1945	unsigned cModifiedPages = 0; NOREF(cModifiedPages);
1946	unsigned cLeft = pPool->cUsedPages;
1947	unsigned iPage = pPool->cCurPages;
1948	while (--iPage >= PGMPOOL_IDX_FIRST)
1949	{
1950	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1951	if (pPage->GCPhys != NIL_RTGCPHYS)
1952	{
1953	switch (pPage->enmKind)
1954	{
1955	/*
1956	* We only care about shadow page tables.
1957	*/
1958	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1959	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1960	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1961	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1962	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1963	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1964	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1965	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1966	{
1967	#ifdef PGMPOOL_WITH_USER_TRACKING
1968	if (pPage->cPresent)
1969	#endif
1970	{
1971	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
1972	STAM_PROFILE_START(&pPool->StatZeroPage, z);
1973	ASMMemZeroPage(pvShw);
1974	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
1975	#ifdef PGMPOOL_WITH_USER_TRACKING
1976	pPage->cPresent = 0;
1977	pPage->iFirstPresent = ~0;
1978	#endif
1979	}
1980	}
1981	/* fall thru */
1982
1983	default:
1984	Assert(!pPage->cModifications \|\| ++cModifiedPages);
1985	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1986	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1987	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1988	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1989	pPage->cModifications = 0;
1990	break;
1991
1992	}
1993	if (!--cLeft)
1994	break;
1995	}
1996	}
1997
1998	/* swipe the special pages too. */
1999	for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
2000	{
2001	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2002	if (pPage->GCPhys != NIL_RTGCPHYS)
2003	{
2004	Assert(!pPage->cModifications \|\| ++cModifiedPages);
2005	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2006	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2007	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2008	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2009	pPage->cModifications = 0;
2010	}
2011	}
2012
2013	#ifndef DEBUG_michael
2014	AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
2015	#endif
2016	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2017	pPool->cModifiedPages = 0;
2018
2019	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2020	/*
2021	* Clear all the GCPhys links and rebuild the phys ext free list.
2022	*/
2023	for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
2024	pRam;
2025	pRam = pRam->CTX_SUFF(pNext))
2026	{
2027	unsigned iPage = pRam->cb >> PAGE_SHIFT;
2028	while (iPage-- > 0)
2029	PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
2030	}
2031
2032	pPool->iPhysExtFreeHead = 0;
2033	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
2034	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
2035	for (unsigned i = 0; i < cMaxPhysExts; i++)
2036	{
2037	paPhysExts[i].iNext = i + 1;
2038	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
2039	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
2040	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
2041	}
2042	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2043	#endif
2044
2045
2046	pPool->cPresent = 0;
2047	STAM_PROFILE_STOP(&pPool->StatClearAll, c);
2048	}
2049	#endif /* IN_RING3 */
2050
2051
2052	/**
2053	* Handle SyncCR3 pool tasks
2054	*
2055	* @returns VBox status code.
2056	* @retval VINF_SUCCESS if successfully added.
2057	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2058	* @param pVM The VM handle.
2059	* @remark Should only be used when monitoring is available, thus placed in
2060	* the PGMPOOL_WITH_MONITORING #ifdef.
2061	*/
2062	int pgmPoolSyncCR3(PVM pVM)
2063	{
2064	LogFlow(("pgmPoolSyncCR3\n"));
2065	/*
2066	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2067	* Occasionally we will have to clear all the shadow page tables because we wanted
2068	* to monitor a page which was mapped by too many shadowed page tables. This operation
2069	* sometimes refered to as a 'lightweight flush'.
2070	*/
2071	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2072	pgmPoolMonitorModifiedClearAll(pVM);
2073	else
2074	{
2075	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2076	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2077	pgmPoolClearAll(pVM);
2078	# else /* !IN_RING3 */
2079	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2080	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2081	return VINF_PGM_SYNC_CR3;
2082	# endif /* !IN_RING3 */
2083	}
2084	return VINF_SUCCESS;
2085	}
2086
2087	#endif /* PGMPOOL_WITH_MONITORING */
2088	#ifdef PGMPOOL_WITH_USER_TRACKING
2089
2090	/**
2091	* Frees up at least one user entry.
2092	*
2093	* @returns VBox status code.
2094	* @retval VINF_SUCCESS if successfully added.
2095	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2096	* @param pPool The pool.
2097	* @param iUser The user index.
2098	*/
2099	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2100	{
2101	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2102	#ifdef PGMPOOL_WITH_CACHE
2103	/*
2104	* Just free cached pages in a braindead fashion.
2105	*/
2106	/** @todo walk the age list backwards and free the first with usage. */
2107	int rc = VINF_SUCCESS;
2108	do
2109	{
2110	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2111	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2112	rc = rc2;
2113	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2114	return rc;
2115	#else
2116	/*
2117	* Lazy approach.
2118	*/
2119	/* @todo incompatible with long mode paging (cr3 root will be flushed) */
2120	Assert(!CPUMIsGuestInLongMode(pVM));
2121	pgmPoolFlushAllInt(pPool);
2122	return VERR_PGM_POOL_FLUSHED;
2123	#endif
2124	}
2125
2126
2127	/**
2128	* Inserts a page into the cache.
2129	*
2130	* This will create user node for the page, insert it into the GCPhys
2131	* hash, and insert it into the age list.
2132	*
2133	* @returns VBox status code.
2134	* @retval VINF_SUCCESS if successfully added.
2135	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2136	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
2137	* @param pPool The pool.
2138	* @param pPage The cached page.
2139	* @param GCPhys The GC physical address of the page we're gonna shadow.
2140	* @param iUser The user index.
2141	* @param iUserTable The user table index.
2142	*/
2143	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2144	{
2145	int rc = VINF_SUCCESS;
2146	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2147
2148	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser %x iUserTable %x\n", GCPhys, iUser, iUserTable));
2149
2150	#ifdef VBOX_STRICT
2151	/*
2152	* Check that the entry doesn't already exists.
2153	*/
2154	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2155	{
2156	uint16_t i = pPage->iUserHead;
2157	do
2158	{
2159	Assert(i < pPool->cMaxUsers);
2160	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2161	i = paUsers[i].iNext;
2162	} while (i != NIL_PGMPOOL_USER_INDEX);
2163	}
2164	#endif
2165
2166	/*
2167	* Find free a user node.
2168	*/
2169	uint16_t i = pPool->iUserFreeHead;
2170	if (i == NIL_PGMPOOL_USER_INDEX)
2171	{
2172	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2173	if (RT_FAILURE(rc))
2174	return rc;
2175	i = pPool->iUserFreeHead;
2176	}
2177
2178	/*
2179	* Unlink the user node from the free list,
2180	* initialize and insert it into the user list.
2181	*/
2182	pPool->iUserFreeHead = paUsers[i].iNext;
2183	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2184	paUsers[i].iUser = iUser;
2185	paUsers[i].iUserTable = iUserTable;
2186	pPage->iUserHead = i;
2187
2188	/*
2189	* Insert into cache and enable monitoring of the guest page if enabled.
2190	*
2191	* Until we implement caching of all levels, including the CR3 one, we'll
2192	* have to make sure we don't try monitor & cache any recursive reuse of
2193	* a monitored CR3 page. Because all windows versions are doing this we'll
2194	* have to be able to do combined access monitoring, CR3 + PT and
2195	* PD + PT (guest PAE).
2196	*
2197	* Update:
2198	* We're now cooperating with the CR3 monitor if an uncachable page is found.
2199	*/
2200	#if defined(PGMPOOL_WITH_MONITORING) \|\| defined(PGMPOOL_WITH_CACHE)
2201	# ifdef PGMPOOL_WITH_MIXED_PT_CR3
2202	const bool fCanBeMonitored = true;
2203	# else
2204	bool fCanBeMonitored = pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
2205	\|\| (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
2206	\|\| pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
2207	# endif
2208	# ifdef PGMPOOL_WITH_CACHE
2209	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2210	# endif
2211	if (fCanBeMonitored)
2212	{
2213	# ifdef PGMPOOL_WITH_MONITORING
2214	rc = pgmPoolMonitorInsert(pPool, pPage);
2215	if (rc == VERR_PGM_POOL_CLEARED)
2216	{
2217	/* 'Failed' - free the usage, and keep it in the cache (if enabled). */
2218	# ifndef PGMPOOL_WITH_CACHE
2219	pgmPoolMonitorFlush(pPool, pPage);
2220	rc = VERR_PGM_POOL_FLUSHED;
2221	# endif
2222	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2223	paUsers[i].iNext = pPool->iUserFreeHead;
2224	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2225	pPool->iUserFreeHead = i;
2226	}
2227	}
2228	# endif
2229	#endif /* PGMPOOL_WITH_MONITORING */
2230	return rc;
2231	}
2232
2233
2234	# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
2235	/**
2236	* Adds a user reference to a page.
2237	*
2238	* This will move the page to the head of the
2239	*
2240	* @returns VBox status code.
2241	* @retval VINF_SUCCESS if successfully added.
2242	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2243	* @param pPool The pool.
2244	* @param pPage The cached page.
2245	* @param iUser The user index.
2246	* @param iUserTable The user table.
2247	*/
2248	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2249	{
2250	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2251
2252	Log3(("pgmPoolTrackAddUser GCPhys = %RGp iUser %x iUserTable %x\n", pPage->GCPhys, iUser, iUserTable));
2253	# ifdef VBOX_STRICT
2254	/*
2255	* Check that the entry doesn't already exists.
2256	*/
2257	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2258	{
2259	uint16_t i = pPage->iUserHead;
2260	do
2261	{
2262	Assert(i < pPool->cMaxUsers);
2263	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2264	i = paUsers[i].iNext;
2265	} while (i != NIL_PGMPOOL_USER_INDEX);
2266	}
2267	# endif
2268
2269	/*
2270	* Allocate a user node.
2271	*/
2272	uint16_t i = pPool->iUserFreeHead;
2273	if (i == NIL_PGMPOOL_USER_INDEX)
2274	{
2275	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2276	if (RT_FAILURE(rc))
2277	return rc;
2278	i = pPool->iUserFreeHead;
2279	}
2280	pPool->iUserFreeHead = paUsers[i].iNext;
2281
2282	/*
2283	* Initialize the user node and insert it.
2284	*/
2285	paUsers[i].iNext = pPage->iUserHead;
2286	paUsers[i].iUser = iUser;
2287	paUsers[i].iUserTable = iUserTable;
2288	pPage->iUserHead = i;
2289
2290	# ifdef PGMPOOL_WITH_CACHE
2291	/*
2292	* Tell the cache to update its replacement stats for this page.
2293	*/
2294	pgmPoolCacheUsed(pPool, pPage);
2295	# endif
2296	return VINF_SUCCESS;
2297	}
2298	# endif /* PGMPOOL_WITH_CACHE */
2299
2300
2301	/**
2302	* Frees a user record associated with a page.
2303	*
2304	* This does not clear the entry in the user table, it simply replaces the
2305	* user record to the chain of free records.
2306	*
2307	* @param pPool The pool.
2308	* @param HCPhys The HC physical address of the shadow page.
2309	* @param iUser The shadow page pool index of the user table.
2310	* @param iUserTable The index into the user table (shadowed).
2311	*/
2312	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2313	{
2314	/*
2315	* Unlink and free the specified user entry.
2316	*/
2317	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2318
2319	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
2320	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2321	uint16_t i = pPage->iUserHead;
2322	if ( i != NIL_PGMPOOL_USER_INDEX
2323	&& paUsers[i].iUser == iUser
2324	&& paUsers[i].iUserTable == iUserTable)
2325	{
2326	pPage->iUserHead = paUsers[i].iNext;
2327
2328	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2329	paUsers[i].iNext = pPool->iUserFreeHead;
2330	pPool->iUserFreeHead = i;
2331	return;
2332	}
2333
2334	/* General: Linear search. */
2335	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2336	while (i != NIL_PGMPOOL_USER_INDEX)
2337	{
2338	if ( paUsers[i].iUser == iUser
2339	&& paUsers[i].iUserTable == iUserTable)
2340	{
2341	if (iPrev != NIL_PGMPOOL_USER_INDEX)
2342	paUsers[iPrev].iNext = paUsers[i].iNext;
2343	else
2344	pPage->iUserHead = paUsers[i].iNext;
2345
2346	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2347	paUsers[i].iNext = pPool->iUserFreeHead;
2348	pPool->iUserFreeHead = i;
2349	return;
2350	}
2351	iPrev = i;
2352	i = paUsers[i].iNext;
2353	}
2354
2355	/* Fatal: didn't find it */
2356	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%RGp\n",
2357	iUser, iUserTable, pPage->GCPhys));
2358	}
2359
2360
2361	/**
2362	* Gets the entry size of a shadow table.
2363	*
2364	* @param enmKind The kind of page.
2365	*
2366	* @returns The size of the entry in bytes. That is, 4 or 8.
2367	* @returns If the kind is not for a table, an assertion is raised and 0 is
2368	* returned.
2369	*/
2370	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2371	{
2372	switch (enmKind)
2373	{
2374	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2375	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2376	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2377	case PGMPOOLKIND_32BIT_PD:
2378	case PGMPOOLKIND_32BIT_PD_PHYS:
2379	return 4;
2380
2381	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2382	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2383	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2384	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2385	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2386	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2387	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2388	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2389	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2390	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2391	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2392	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2393	case PGMPOOLKIND_64BIT_PML4:
2394	case PGMPOOLKIND_PAE_PDPT:
2395	case PGMPOOLKIND_ROOT_NESTED:
2396	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2397	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2398	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2399	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2400	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2401	case PGMPOOLKIND_PAE_PD_PHYS:
2402	case PGMPOOLKIND_PAE_PDPT_PHYS:
2403	return 8;
2404
2405	default:
2406	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2407	}
2408	}
2409
2410
2411	/**
2412	* Gets the entry size of a guest table.
2413	*
2414	* @param enmKind The kind of page.
2415	*
2416	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
2417	* @returns If the kind is not for a table, an assertion is raised and 0 is
2418	* returned.
2419	*/
2420	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2421	{
2422	switch (enmKind)
2423	{
2424	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2425	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2426	case PGMPOOLKIND_32BIT_PD:
2427	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2428	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2429	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2430	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2431	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2432	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2433	return 4;
2434
2435	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2436	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2437	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2438	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2439	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2440	case PGMPOOLKIND_64BIT_PML4:
2441	case PGMPOOLKIND_PAE_PDPT:
2442	return 8;
2443
2444	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2445	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2446	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2447	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2448	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2449	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2450	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2451	case PGMPOOLKIND_ROOT_NESTED:
2452	case PGMPOOLKIND_PAE_PD_PHYS:
2453	case PGMPOOLKIND_PAE_PDPT_PHYS:
2454	case PGMPOOLKIND_32BIT_PD_PHYS:
2455	/** @todo can we return 0? (nobody is calling this...) */
2456	AssertFailed();
2457	return 0;
2458
2459	default:
2460	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2461	}
2462	}
2463
2464	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2465
2466	/**
2467	* Scans one shadow page table for mappings of a physical page.
2468	*
2469	* @param pVM The VM handle.
2470	* @param pPhysPage The guest page in question.
2471	* @param iShw The shadow page table.
2472	* @param cRefs The number of references made in that PT.
2473	*/
2474	static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2475	{
2476	LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
2477	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2478
2479	/*
2480	* Assert sanity.
2481	*/
2482	Assert(cRefs == 1);
2483	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2484	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2485
2486	/*
2487	* Then, clear the actual mappings to the page in the shadow PT.
2488	*/
2489	switch (pPage->enmKind)
2490	{
2491	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2492	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2493	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2494	{
2495	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2496	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2497	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2498	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2499	{
2500	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2501	pPT->a[i].u = 0;
2502	cRefs--;
2503	if (!cRefs)
2504	return;
2505	}
2506	#ifdef LOG_ENABLED
2507	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2508	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2509	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2510	{
2511	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2512	pPT->a[i].u = 0;
2513	}
2514	#endif
2515	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2516	break;
2517	}
2518
2519	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2520	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2521	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2522	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2523	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2524	{
2525	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2526	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2527	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2528	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2529	{
2530	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2531	pPT->a[i].u = 0;
2532	cRefs--;
2533	if (!cRefs)
2534	return;
2535	}
2536	#ifdef LOG_ENABLED
2537	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2538	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2539	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2540	{
2541	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2542	pPT->a[i].u = 0;
2543	}
2544	#endif
2545	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d u64=%RX64\n", cRefs, pPage->iFirstPresent, pPage->cPresent, u64));
2546	break;
2547	}
2548
2549	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2550	{
2551	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2552	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2553	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2554	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
2555	{
2556	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2557	pPT->a[i].u = 0;
2558	cRefs--;
2559	if (!cRefs)
2560	return;
2561	}
2562	#ifdef LOG_ENABLED
2563	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2564	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2565	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
2566	{
2567	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2568	pPT->a[i].u = 0;
2569	}
2570	#endif
2571	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2572	break;
2573	}
2574
2575	default:
2576	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
2577	}
2578	}
2579
2580
2581	/**
2582	* Scans one shadow page table for mappings of a physical page.
2583	*
2584	* @param pVM The VM handle.
2585	* @param pPhysPage The guest page in question.
2586	* @param iShw The shadow page table.
2587	* @param cRefs The number of references made in that PT.
2588	*/
2589	void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2590	{
2591	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2592	LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
2593	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
2594	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
2595	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
2596	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
2597	}
2598
2599
2600	/**
2601	* Flushes a list of shadow page tables mapping the same physical page.
2602	*
2603	* @param pVM The VM handle.
2604	* @param pPhysPage The guest page in question.
2605	* @param iPhysExt The physical cross reference extent list to flush.
2606	*/
2607	void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
2608	{
2609	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2610	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
2611	LogFlow(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%R[pgmpage] iPhysExt\n", pPhysPage, iPhysExt));
2612
2613	const uint16_t iPhysExtStart = iPhysExt;
2614	PPGMPOOLPHYSEXT pPhysExt;
2615	do
2616	{
2617	Assert(iPhysExt < pPool->cMaxPhysExts);
2618	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
2619	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
2620	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
2621	{
2622	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
2623	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2624	}
2625
2626	/* next */
2627	iPhysExt = pPhysExt->iNext;
2628	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2629
2630	/* insert the list into the free list and clear the ram range entry. */
2631	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2632	pPool->iPhysExtFreeHead = iPhysExtStart;
2633	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
2634
2635	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
2636	}
2637
2638	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2639
2640	/**
2641	* Flushes all shadow page table mappings of the given guest page.
2642	*
2643	* This is typically called when the host page backing the guest one has been
2644	* replaced or when the page protection was changed due to an access handler.
2645	*
2646	* @returns VBox status code.
2647	* @retval VINF_SUCCESS if all references has been successfully cleared.
2648	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
2649	* a page pool cleaning.
2650	*
2651	* @param pVM The VM handle.
2652	* @param pPhysPage The guest page in question.
2653	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
2654	* flushed, it is NOT touched if this isn't necessary.
2655	* The caller MUST initialized this to @a false.
2656	*/
2657	int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs)
2658	{
2659	int rc = VINF_SUCCESS;
2660	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2661	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
2662	if (u16)
2663	{
2664	# ifdef VBOX_WITH_NEW_PHYS_CODE
2665	/*
2666	* The zero page is currently screwing up the tracking and we'll
2667	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2668	* is defined, zero pages won't normally be mapped. Some kind of solution
2669	* will be needed for this problem of course, but it will have to wait...
2670	*/
2671	if (PGM_PAGE_IS_ZERO(pPhysPage))
2672	rc = VINF_PGM_GCPHYS_ALIASED;
2673	else
2674	# endif
2675	{
2676	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2677	/* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
2678	pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
2679	PVMCPU pVCpu = VMMGetCpu(pVM);
2680	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
2681	# endif
2682
2683	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
2684	pgmPoolTrackFlushGCPhysPT(pVM,
2685	pPhysPage,
2686	PGMPOOL_TD_GET_IDX(u16),
2687	PGMPOOL_TD_GET_CREFS(u16));
2688	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
2689	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, PGMPOOL_TD_GET_IDX(u16));
2690	else
2691	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
2692	*pfFlushTLBs = true;
2693
2694	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2695	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
2696	# endif
2697	}
2698	}
2699
2700	#elif defined(PGMPOOL_WITH_CACHE)
2701	# ifdef VBOX_WITH_NEW_PHYS_CODE
2702	if (PGM_PAGE_IS_ZERO(pPhysPage))
2703	rc = VINF_PGM_GCPHYS_ALIASED;
2704	else
2705	# endif
2706	{
2707	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2708	/* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow kill the pool otherwise. */
2709	PVMCPU pVCpu = VMMGetCpu(pVM);
2710	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
2711	# endif
2712	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
2713	if (rc == VINF_SUCCESS)
2714	*pfFlushTLBs = true;
2715	}
2716
2717	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2718	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
2719	# endif
2720
2721	#else
2722	rc = VINF_PGM_GCPHYS_ALIASED;
2723	#endif
2724
2725	return rc;
2726	}
2727
2728
2729	/**
2730	* Scans all shadow page tables for mappings of a physical page.
2731	*
2732	* This may be slow, but it's most likely more efficient than cleaning
2733	* out the entire page pool / cache.
2734	*
2735	* @returns VBox status code.
2736	* @retval VINF_SUCCESS if all references has been successfully cleared.
2737	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
2738	* a page pool cleaning.
2739	*
2740	* @param pVM The VM handle.
2741	* @param pPhysPage The guest page in question.
2742	*/
2743	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
2744	{
2745	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2746	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2747	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
2748	pPool->cUsedPages, pPool->cPresent, pPhysPage));
2749
2750	#if 1
2751	/*
2752	* There is a limit to what makes sense.
2753	*/
2754	if (pPool->cPresent > 1024)
2755	{
2756	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
2757	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2758	return VINF_PGM_GCPHYS_ALIASED;
2759	}
2760	#endif
2761
2762	/*
2763	* Iterate all the pages until we've encountered all that in use.
2764	* This is simple but not quite optimal solution.
2765	*/
2766	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2767	const uint32_t u32 = u64;
2768	unsigned cLeft = pPool->cUsedPages;
2769	unsigned iPage = pPool->cCurPages;
2770	while (--iPage >= PGMPOOL_IDX_FIRST)
2771	{
2772	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2773	if (pPage->GCPhys != NIL_RTGCPHYS)
2774	{
2775	switch (pPage->enmKind)
2776	{
2777	/*
2778	* We only care about shadow page tables.
2779	*/
2780	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2781	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2782	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2783	{
2784	unsigned cPresent = pPage->cPresent;
2785	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2786	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2787	if (pPT->a[i].n.u1Present)
2788	{
2789	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2790	{
2791	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
2792	pPT->a[i].u = 0;
2793	}
2794	if (!--cPresent)
2795	break;
2796	}
2797	break;
2798	}
2799
2800	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2801	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2802	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2803	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2804	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2805	{
2806	unsigned cPresent = pPage->cPresent;
2807	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2808	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2809	if (pPT->a[i].n.u1Present)
2810	{
2811	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2812	{
2813	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
2814	pPT->a[i].u = 0;
2815	}
2816	if (!--cPresent)
2817	break;
2818	}
2819	break;
2820	}
2821	}
2822	if (!--cLeft)
2823	break;
2824	}
2825	}
2826
2827	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
2828	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2829	return VINF_SUCCESS;
2830	}
2831
2832
2833	/**
2834	* Clears the user entry in a user table.
2835	*
2836	* This is used to remove all references to a page when flushing it.
2837	*/
2838	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
2839	{
2840	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
2841	Assert(pUser->iUser < pPool->cCurPages);
2842	uint32_t iUserTable = pUser->iUserTable;
2843
2844	/*
2845	* Map the user page.
2846	*/
2847	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
2848	union
2849	{
2850	uint64_t *pau64;
2851	uint32_t *pau32;
2852	} u;
2853	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
2854
2855	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
2856
2857	/* Safety precaution in case we change the paging for other modes too in the future. */
2858	Assert(!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage));
2859
2860	#ifdef VBOX_STRICT
2861	/*
2862	* Some sanity checks.
2863	*/
2864	switch (pUserPage->enmKind)
2865	{
2866	case PGMPOOLKIND_32BIT_PD:
2867	case PGMPOOLKIND_32BIT_PD_PHYS:
2868	Assert(iUserTable < X86_PG_ENTRIES);
2869	break;
2870	case PGMPOOLKIND_PAE_PDPT:
2871	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2872	case PGMPOOLKIND_PAE_PDPT_PHYS:
2873	Assert(iUserTable < 4);
2874	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
2875	break;
2876	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2877	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2878	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2879	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2880	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2881	case PGMPOOLKIND_PAE_PD_PHYS:
2882	Assert(iUserTable < X86_PG_PAE_ENTRIES);
2883	break;
2884	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2885	Assert(iUserTable < X86_PG_PAE_ENTRIES);
2886	Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
2887	break;
2888	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2889	Assert(iUserTable < X86_PG_PAE_ENTRIES);
2890	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
2891	break;
2892	case PGMPOOLKIND_64BIT_PML4:
2893	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
2894	/* GCPhys >> PAGE_SHIFT is the index here */
2895	break;
2896	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2897	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2898	Assert(iUserTable < X86_PG_PAE_ENTRIES);
2899	break;
2900
2901	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2902	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2903	Assert(iUserTable < X86_PG_PAE_ENTRIES);
2904	break;
2905
2906	case PGMPOOLKIND_ROOT_NESTED:
2907	Assert(iUserTable < X86_PG_PAE_ENTRIES);
2908	break;
2909
2910	default:
2911	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
2912	break;
2913	}
2914	#endif /* VBOX_STRICT */
2915
2916	/*
2917	* Clear the entry in the user page.
2918	*/
2919	switch (pUserPage->enmKind)
2920	{
2921	/* 32-bit entries */
2922	case PGMPOOLKIND_32BIT_PD:
2923	case PGMPOOLKIND_32BIT_PD_PHYS:
2924	u.pau32[iUserTable] = 0;
2925	break;
2926
2927	/* 64-bit entries */
2928	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2929	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2930	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2931	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2932	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2933	#if defined(IN_RC)
2934	/* In 32 bits PAE mode we must invalidate the TLB when changing a PDPT entry; the CPU fetches them only during cr3 load, so any
2935	* non-present PDPT will continue to cause page faults.
2936	*/
2937	ASMReloadCR3();
2938	#endif
2939	/* no break */
2940	case PGMPOOLKIND_PAE_PD_PHYS:
2941	case PGMPOOLKIND_PAE_PDPT_PHYS:
2942	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2943	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2944	case PGMPOOLKIND_64BIT_PML4:
2945	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2946	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2947	case PGMPOOLKIND_PAE_PDPT:
2948	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2949	case PGMPOOLKIND_ROOT_NESTED:
2950	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2951	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2952	u.pau64[iUserTable] = 0;
2953	break;
2954
2955	default:
2956	AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
2957	}
2958	}
2959
2960
2961	/**
2962	* Clears all users of a page.
2963	*/
2964	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2965	{
2966	/*
2967	* Free all the user records.
2968	*/
2969	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
2970
2971	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2972	uint16_t i = pPage->iUserHead;
2973	while (i != NIL_PGMPOOL_USER_INDEX)
2974	{
2975	/* Clear enter in user table. */
2976	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
2977
2978	/* Free it. */
2979	const uint16_t iNext = paUsers[i].iNext;
2980	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2981	paUsers[i].iNext = pPool->iUserFreeHead;
2982	pPool->iUserFreeHead = i;
2983
2984	/* Next. */
2985	i = iNext;
2986	}
2987	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2988	}
2989
2990	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2991
2992	/**
2993	* Allocates a new physical cross reference extent.
2994	*
2995	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
2996	* @param pVM The VM handle.
2997	* @param piPhysExt Where to store the phys ext index.
2998	*/
2999	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
3000	{
3001	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3002	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3003	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3004	{
3005	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3006	return NULL;
3007	}
3008	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3009	pPool->iPhysExtFreeHead = pPhysExt->iNext;
3010	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3011	*piPhysExt = iPhysExt;
3012	return pPhysExt;
3013	}
3014
3015
3016	/**
3017	* Frees a physical cross reference extent.
3018	*
3019	* @param pVM The VM handle.
3020	* @param iPhysExt The extent to free.
3021	*/
3022	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
3023	{
3024	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3025	Assert(iPhysExt < pPool->cMaxPhysExts);
3026	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3027	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3028	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3029	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3030	pPool->iPhysExtFreeHead = iPhysExt;
3031	}
3032
3033
3034	/**
3035	* Frees a physical cross reference extent.
3036	*
3037	* @param pVM The VM handle.
3038	* @param iPhysExt The extent to free.
3039	*/
3040	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
3041	{
3042	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3043
3044	const uint16_t iPhysExtStart = iPhysExt;
3045	PPGMPOOLPHYSEXT pPhysExt;
3046	do
3047	{
3048	Assert(iPhysExt < pPool->cMaxPhysExts);
3049	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3050	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3051	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3052
3053	/* next */
3054	iPhysExt = pPhysExt->iNext;
3055	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3056
3057	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3058	pPool->iPhysExtFreeHead = iPhysExtStart;
3059	}
3060
3061
3062	/**
3063	* Insert a reference into a list of physical cross reference extents.
3064	*
3065	* @returns The new tracking data for PGMPAGE.
3066	*
3067	* @param pVM The VM handle.
3068	* @param iPhysExt The physical extent index of the list head.
3069	* @param iShwPT The shadow page table index.
3070	*
3071	*/
3072	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
3073	{
3074	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3075	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3076
3077	/* special common case. */
3078	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3079	{
3080	paPhysExts[iPhysExt].aidx[2] = iShwPT;
3081	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3082	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{,,%d}\n", iPhysExt, iShwPT));
3083	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3084	}
3085
3086	/* general treatment. */
3087	const uint16_t iPhysExtStart = iPhysExt;
3088	unsigned cMax = 15;
3089	for (;;)
3090	{
3091	Assert(iPhysExt < pPool->cMaxPhysExts);
3092	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3093	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3094	{
3095	paPhysExts[iPhysExt].aidx[i] = iShwPT;
3096	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3097	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
3098	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
3099	}
3100	if (!--cMax)
3101	{
3102	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3103	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3104	LogFlow(("pgmPoolTrackPhysExtAddref: overflow (1) iShwPT=%d\n", iShwPT));
3105	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3106	}
3107	}
3108
3109	/* add another extent to the list. */
3110	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3111	if (!pNew)
3112	{
3113	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3114	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3115	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3116	}
3117	pNew->iNext = iPhysExtStart;
3118	pNew->aidx[0] = iShwPT;
3119	LogFlow(("pgmPoolTrackPhysExtAddref: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
3120	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3121	}
3122
3123
3124	/**
3125	* Add a reference to guest physical page where extents are in use.
3126	*
3127	* @returns The new tracking data for PGMPAGE.
3128	*
3129	* @param pVM The VM handle.
3130	* @param u16 The ram range flags (top 16-bits).
3131	* @param iShwPT The shadow page table index.
3132	*/
3133	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
3134	{
3135	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3136	{
3137	/*
3138	* Convert to extent list.
3139	*/
3140	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3141	uint16_t iPhysExt;
3142	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3143	if (pPhysExt)
3144	{
3145	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
3146	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
3147	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
3148	pPhysExt->aidx[1] = iShwPT;
3149	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3150	}
3151	else
3152	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3153	}
3154	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3155	{
3156	/*
3157	* Insert into the extent list.
3158	*/
3159	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT);
3160	}
3161	else
3162	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
3163	return u16;
3164	}
3165
3166
3167	/**
3168	* Clear references to guest physical memory.
3169	*
3170	* @param pPool The pool.
3171	* @param pPage The page.
3172	* @param pPhysPage Pointer to the aPages entry in the ram range.
3173	*/
3174	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
3175	{
3176	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
3177	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3178
3179	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
3180	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
3181	{
3182	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
3183	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3184	do
3185	{
3186	Assert(iPhysExt < pPool->cMaxPhysExts);
3187
3188	/*
3189	* Look for the shadow page and check if it's all freed.
3190	*/
3191	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3192	{
3193	if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
3194	{
3195	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
3196
3197	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3198	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
3199	{
3200	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3201	return;
3202	}
3203
3204	/* we can free the node. */
3205	PVM pVM = pPool->CTX_SUFF(pVM);
3206	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
3207	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
3208	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
3209	{
3210	/* lonely node */
3211	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3212	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
3213	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3214	}
3215	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
3216	{
3217	/* head */
3218	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
3219	PGM_PAGE_SET_TRACKING(pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
3220	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3221	}
3222	else
3223	{
3224	/* in list */
3225	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3226	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
3227	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3228	}
3229	iPhysExt = iPhysExtNext;
3230	return;
3231	}
3232	}
3233
3234	/* next */
3235	iPhysExtPrev = iPhysExt;
3236	iPhysExt = paPhysExts[iPhysExt].iNext;
3237	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3238
3239	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3240	}
3241	else /* nothing to do */
3242	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
3243	}
3244
3245
3246	/**
3247	* Clear references to guest physical memory.
3248	*
3249	* This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
3250	* is assumed to be correct, so the linear search can be skipped and we can assert
3251	* at an earlier point.
3252	*
3253	* @param pPool The pool.
3254	* @param pPage The page.
3255	* @param HCPhys The host physical address corresponding to the guest page.
3256	* @param GCPhys The guest physical address corresponding to HCPhys.
3257	*/
3258	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
3259	{
3260	/*
3261	* Walk range list.
3262	*/
3263	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3264	while (pRam)
3265	{
3266	RTGCPHYS off = GCPhys - pRam->GCPhys;
3267	if (off < pRam->cb)
3268	{
3269	/* does it match? */
3270	const unsigned iPage = off >> PAGE_SHIFT;
3271	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3272	#ifdef LOG_ENABLED
3273	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
3274	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
3275	#endif
3276	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3277	{
3278	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3279	return;
3280	}
3281	break;
3282	}
3283	pRam = pRam->CTX_SUFF(pNext);
3284	}
3285	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
3286	}
3287
3288
3289	/**
3290	* Clear references to guest physical memory.
3291	*
3292	* @param pPool The pool.
3293	* @param pPage The page.
3294	* @param HCPhys The host physical address corresponding to the guest page.
3295	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
3296	*/
3297	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
3298	{
3299	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
3300
3301	/*
3302	* Walk range list.
3303	*/
3304	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3305	while (pRam)
3306	{
3307	RTGCPHYS off = GCPhysHint - pRam->GCPhys;
3308	if (off < pRam->cb)
3309	{
3310	/* does it match? */
3311	const unsigned iPage = off >> PAGE_SHIFT;
3312	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3313	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3314	{
3315	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3316	return;
3317	}
3318	break;
3319	}
3320	pRam = pRam->CTX_SUFF(pNext);
3321	}
3322
3323	/*
3324	* Damn, the hint didn't work. We'll have to do an expensive linear search.
3325	*/
3326	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
3327	pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3328	while (pRam)
3329	{
3330	unsigned iPage = pRam->cb >> PAGE_SHIFT;
3331	while (iPage-- > 0)
3332	{
3333	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3334	{
3335	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
3336	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
3337	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3338	return;
3339	}
3340	}
3341	pRam = pRam->CTX_SUFF(pNext);
3342	}
3343
3344	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp\n", HCPhys, GCPhysHint));
3345	}
3346
3347
3348	/**
3349	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
3350	*
3351	* @param pPool The pool.
3352	* @param pPage The page.
3353	* @param pShwPT The shadow page table (mapping of the page).
3354	* @param pGstPT The guest page table.
3355	*/
3356	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
3357	{
3358	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
3359	if (pShwPT->a[i].n.u1Present)
3360	{
3361	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
3362	i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3363	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3364	if (!--pPage->cPresent)
3365	break;
3366	}
3367	}
3368
3369
3370	/**
3371	* Clear references to guest physical memory in a PAE / 32-bit page table.
3372	*
3373	* @param pPool The pool.
3374	* @param pPage The page.
3375	* @param pShwPT The shadow page table (mapping of the page).
3376	* @param pGstPT The guest page table (just a half one).
3377	*/
3378	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
3379	{
3380	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3381	if (pShwPT->a[i].n.u1Present)
3382	{
3383	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
3384	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3385	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3386	}
3387	}
3388
3389
3390	/**
3391	* Clear references to guest physical memory in a PAE / PAE page table.
3392	*
3393	* @param pPool The pool.
3394	* @param pPage The page.
3395	* @param pShwPT The shadow page table (mapping of the page).
3396	* @param pGstPT The guest page table.
3397	*/
3398	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
3399	{
3400	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3401	if (pShwPT->a[i].n.u1Present)
3402	{
3403	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
3404	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
3405	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
3406	}
3407	}
3408
3409
3410	/**
3411	* Clear references to guest physical memory in a 32-bit / 4MB page table.
3412	*
3413	* @param pPool The pool.
3414	* @param pPage The page.
3415	* @param pShwPT The shadow page table (mapping of the page).
3416	*/
3417	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
3418	{
3419	RTGCPHYS GCPhys = pPage->GCPhys;
3420	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3421	if (pShwPT->a[i].n.u1Present)
3422	{
3423	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
3424	i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
3425	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
3426	}
3427	}
3428
3429
3430	/**
3431	* Clear references to guest physical memory in a PAE / 2/4MB page table.
3432	*
3433	* @param pPool The pool.
3434	* @param pPage The page.
3435	* @param pShwPT The shadow page table (mapping of the page).
3436	*/
3437	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
3438	{
3439	RTGCPHYS GCPhys = pPage->GCPhys;
3440	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3441	if (pShwPT->a[i].n.u1Present)
3442	{
3443	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
3444	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
3445	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
3446	}
3447	}
3448
3449	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3450
3451
3452	/**
3453	* Clear references to shadowed pages in a 32 bits page directory.
3454	*
3455	* @param pPool The pool.
3456	* @param pPage The page.
3457	* @param pShwPD The shadow page directory (mapping of the page).
3458	*/
3459	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
3460	{
3461	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3462	{
3463	if ( pShwPD->a[i].n.u1Present
3464	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3465	)
3466	{
3467	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
3468	if (pSubPage)
3469	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3470	else
3471	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
3472	}
3473	}
3474	}
3475
3476	/**
3477	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
3478	*
3479	* @param pPool The pool.
3480	* @param pPage The page.
3481	* @param pShwPD The shadow page directory (mapping of the page).
3482	*/
3483	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
3484	{
3485	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3486	{
3487	if ( pShwPD->a[i].n.u1Present
3488	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3489	)
3490	{
3491	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
3492	if (pSubPage)
3493	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3494	else
3495	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
3496	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3497	}
3498	}
3499	}
3500
3501	/**
3502	* Clear references to shadowed pages in a PAE page directory pointer table.
3503	*
3504	* @param pPool The pool.
3505	* @param pPage The page.
3506	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3507	*/
3508	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3509	{
3510	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
3511	{
3512	if ( pShwPDPT->a[i].n.u1Present
3513	&& !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
3514	)
3515	{
3516	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3517	if (pSubPage)
3518	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3519	else
3520	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3521	}
3522	}
3523	}
3524
3525
3526	/**
3527	* Clear references to shadowed pages in a 64-bit page directory pointer table.
3528	*
3529	* @param pPool The pool.
3530	* @param pPage The page.
3531	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3532	*/
3533	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3534	{
3535	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3536	{
3537	Assert(!(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING));
3538	if (pShwPDPT->a[i].n.u1Present)
3539	{
3540	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3541	if (pSubPage)
3542	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3543	else
3544	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3545	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3546	}
3547	}
3548	}
3549
3550
3551	/**
3552	* Clear references to shadowed pages in a 64-bit level 4 page table.
3553	*
3554	* @param pPool The pool.
3555	* @param pPage The page.
3556	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
3557	*/
3558	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
3559	{
3560	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
3561	{
3562	if (pShwPML4->a[i].n.u1Present)
3563	{
3564	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
3565	if (pSubPage)
3566	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3567	else
3568	AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
3569	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3570	}
3571	}
3572	}
3573
3574
3575	/**
3576	* Clear references to shadowed pages in an EPT page table.
3577	*
3578	* @param pPool The pool.
3579	* @param pPage The page.
3580	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
3581	*/
3582	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
3583	{
3584	RTGCPHYS GCPhys = pPage->GCPhys;
3585	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3586	if (pShwPT->a[i].n.u1Present)
3587	{
3588	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
3589	i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
3590	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys);
3591	}
3592	}
3593
3594
3595	/**
3596	* Clear references to shadowed pages in an EPT page directory.
3597	*
3598	* @param pPool The pool.
3599	* @param pPage The page.
3600	* @param pShwPD The shadow page directory (mapping of the page).
3601	*/
3602	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
3603	{
3604	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3605	{
3606	if (pShwPD->a[i].n.u1Present)
3607	{
3608	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
3609	if (pSubPage)
3610	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3611	else
3612	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
3613	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3614	}
3615	}
3616	}
3617
3618
3619	/**
3620	* Clear references to shadowed pages in an EPT page directory pointer table.
3621	*
3622	* @param pPool The pool.
3623	* @param pPage The page.
3624	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3625	*/
3626	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
3627	{
3628	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3629	{
3630	if (pShwPDPT->a[i].n.u1Present)
3631	{
3632	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
3633	if (pSubPage)
3634	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3635	else
3636	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
3637	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3638	}
3639	}
3640	}
3641
3642
3643	/**
3644	* Clears all references made by this page.
3645	*
3646	* This includes other shadow pages and GC physical addresses.
3647	*
3648	* @param pPool The pool.
3649	* @param pPage The page.
3650	*/
3651	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3652	{
3653	/*
3654	* Map the shadow page and take action according to the page kind.
3655	*/
3656	void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
3657	switch (pPage->enmKind)
3658	{
3659	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3660	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3661	{
3662	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3663	void *pvGst;
3664	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3665	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
3666	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3667	break;
3668	}
3669
3670	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3671	{
3672	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3673	void *pvGst;
3674	int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3675	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
3676	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3677	break;
3678	}
3679
3680	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3681	{
3682	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3683	void *pvGst;
3684	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3685	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
3686	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3687	break;
3688	}
3689
3690	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
3691	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3692	{
3693	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3694	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
3695	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3696	break;
3697	}
3698
3699	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
3700	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3701	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3702	{
3703	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3704	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
3705	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3706	break;
3707	}
3708
3709	#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3710	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3711	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3712	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3713	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3714	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3715	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3716	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3717	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3718	break;
3719	#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3720
3721	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3722	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3723	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3724	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3725	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3726	case PGMPOOLKIND_PAE_PD_PHYS:
3727	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3728	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3729	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
3730	break;
3731
3732	case PGMPOOLKIND_32BIT_PD_PHYS:
3733	case PGMPOOLKIND_32BIT_PD:
3734	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
3735	break;
3736
3737	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3738	case PGMPOOLKIND_PAE_PDPT:
3739	case PGMPOOLKIND_PAE_PDPT_PHYS:
3740	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
3741	break;
3742
3743	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3744	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3745	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
3746	break;
3747
3748	case PGMPOOLKIND_64BIT_PML4:
3749	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
3750	break;
3751
3752	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3753	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
3754	break;
3755
3756	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3757	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
3758	break;
3759
3760	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3761	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
3762	break;
3763
3764	default:
3765	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
3766	}
3767
3768	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
3769	STAM_PROFILE_START(&pPool->StatZeroPage, z);
3770	ASMMemZeroPage(pvShw);
3771	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3772	pPage->fZeroed = true;
3773	PGMPOOL_UNLOCK_PTR(pPool->CTX_SUFF(pVM), pvShw);
3774	}
3775
3776	#endif /* PGMPOOL_WITH_USER_TRACKING */
3777
3778
3779	#ifdef IN_RING3
3780	/**
3781	* Flushes all the special root pages as part of a pgmPoolFlushAllInt operation.
3782	*
3783	* @param pPool The pool.
3784	*/
3785	static void pgmPoolFlushAllSpecialRoots(PPGMPOOL pPool)
3786	{
3787	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3788	/* Start a subset so we won't run out of mapping space. */
3789	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
3790	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3791	#endif
3792
3793	/*
3794	* These special pages are all mapped into the indexes 1..PGMPOOL_IDX_FIRST.
3795	*/
3796	Assert(NIL_PGMPOOL_IDX == 0);
3797
3798	/*
3799	* Paranoia (to be removed), flag a global CR3 sync.
3800	*/
3801	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
3802
3803	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3804	/* Pop the subset. */
3805	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3806	#endif
3807	}
3808
3809
3810	/**
3811	* Flushes the entire cache.
3812	*
3813	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3814	* and execute this CR3 flush.
3815	*
3816	* @param pPool The pool.
3817	*/
3818	static void pgmPoolFlushAllInt(PPGMPOOL pPool)
3819	{
3820	PVM pVM = pPool->CTX_SUFF(pVM);
3821
3822	STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
3823	LogFlow(("pgmPoolFlushAllInt:\n"));
3824
3825	/*
3826	* If there are no pages in the pool, there is nothing to do.
3827	*/
3828	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
3829	{
3830	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3831	return;
3832	}
3833
3834	/* Unmap the old CR3 value before flushing everything. */
3835	int rc = PGM_BTH_PFN(UnmapCR3, pVM)(pVM);
3836	AssertRC(rc);
3837
3838	/* Exit the current shadow paging mode as well; nested paging and EPT use a root CR3 which will get flushed here. */
3839	rc = PGM_SHW_PFN(Exit, pVM)(pVM);
3840	AssertRC(rc);
3841
3842	/*
3843	* Nuke the free list and reinsert all pages into it.
3844	*/
3845	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
3846	{
3847	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3848
3849	#ifdef IN_RING3
3850	Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
3851	#endif
3852	#ifdef PGMPOOL_WITH_MONITORING
3853	if (pPage->fMonitored)
3854	pgmPoolMonitorFlush(pPool, pPage);
3855	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3856	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3857	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3858	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3859	pPage->cModifications = 0;
3860	#endif
3861	pPage->GCPhys = NIL_RTGCPHYS;
3862	pPage->enmKind = PGMPOOLKIND_FREE;
3863	Assert(pPage->idx == i);
3864	pPage->iNext = i + 1;
3865	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
3866	pPage->fSeenNonGlobal = false;
3867	pPage->fMonitored= false;
3868	pPage->fCached = false;
3869	pPage->fReusedFlushPending = false;
3870	#ifdef PGMPOOL_WITH_USER_TRACKING
3871	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3872	#else
3873	pPage->fCR3Mix = false;
3874	#endif
3875	#ifdef PGMPOOL_WITH_CACHE
3876	pPage->iAgeNext = NIL_PGMPOOL_IDX;
3877	pPage->iAgePrev = NIL_PGMPOOL_IDX;
3878	#endif
3879	pPage->fLocked = false;
3880	}
3881	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
3882	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
3883	pPool->cUsedPages = 0;
3884
3885	#ifdef PGMPOOL_WITH_USER_TRACKING
3886	/*
3887	* Zap and reinitialize the user records.
3888	*/
3889	pPool->cPresent = 0;
3890	pPool->iUserFreeHead = 0;
3891	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3892	const unsigned cMaxUsers = pPool->cMaxUsers;
3893	for (unsigned i = 0; i < cMaxUsers; i++)
3894	{
3895	paUsers[i].iNext = i + 1;
3896	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3897	paUsers[i].iUserTable = 0xfffffffe;
3898	}
3899	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
3900	#endif
3901
3902	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3903	/*
3904	* Clear all the GCPhys links and rebuild the phys ext free list.
3905	*/
3906	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
3907	pRam;
3908	pRam = pRam->CTX_SUFF(pNext))
3909	{
3910	unsigned iPage = pRam->cb >> PAGE_SHIFT;
3911	while (iPage-- > 0)
3912	PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
3913	}
3914
3915	pPool->iPhysExtFreeHead = 0;
3916	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3917	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
3918	for (unsigned i = 0; i < cMaxPhysExts; i++)
3919	{
3920	paPhysExts[i].iNext = i + 1;
3921	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
3922	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
3923	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
3924	}
3925	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3926	#endif
3927
3928	#ifdef PGMPOOL_WITH_MONITORING
3929	/*
3930	* Just zap the modified list.
3931	*/
3932	pPool->cModifiedPages = 0;
3933	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
3934	#endif
3935
3936	#ifdef PGMPOOL_WITH_CACHE
3937	/*
3938	* Clear the GCPhys hash and the age list.
3939	*/
3940	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
3941	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
3942	pPool->iAgeHead = NIL_PGMPOOL_IDX;
3943	pPool->iAgeTail = NIL_PGMPOOL_IDX;
3944	#endif
3945
3946	/*
3947	* Flush all the special root pages.
3948	* Reinsert active pages into the hash and ensure monitoring chains are correct.
3949	*/
3950	pgmPoolFlushAllSpecialRoots(pPool);
3951	for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
3952	{
3953	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3954	pPage->iNext = NIL_PGMPOOL_IDX;
3955	#ifdef PGMPOOL_WITH_MONITORING
3956	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3957	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3958	pPage->cModifications = 0;
3959	/* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
3960	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3961	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3962	if (pPage->fMonitored)
3963	{
3964	int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
3965	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
3966	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
3967	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
3968	pPool->pszAccessHandler);
3969	AssertFatalRCSuccess(rc);
3970	# ifdef PGMPOOL_WITH_CACHE
3971	pgmPoolHashInsert(pPool, pPage);
3972	# endif
3973	}
3974	#endif
3975	#ifdef PGMPOOL_WITH_USER_TRACKING
3976	Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
3977	#endif
3978	#ifdef PGMPOOL_WITH_CACHE
3979	Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
3980	Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
3981	#endif
3982	}
3983
3984	/* Force a shadow mode reinit (necessary for nested paging and ept). */
3985	pVM->pgm.s.enmShadowMode = PGMMODE_INVALID;
3986
3987	/* Reinit the current shadow paging mode as well; nested paging and EPT use a root CR3 which will get flushed here. */
3988	rc = PGMR3ChangeMode(pVM, PGMGetGuestMode(pVM));
3989	AssertRC(rc);
3990
3991	/*
3992	* Finally, assert the FF.
3993	*/
3994	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
3995
3996	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3997	}
3998	#endif /* IN_RING3 */
3999
4000	/**
4001	* Flushes a pool page.
4002	*
4003	* This moves the page to the free list after removing all user references to it.
4004	* In GC this will cause a CR3 reload if the page is traced back to an active root page.
4005	*
4006	* @returns VBox status code.
4007	* @retval VINF_SUCCESS on success.
4008	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
4009	* @param pPool The pool.
4010	* @param HCPhys The HC physical address of the shadow page.
4011	*/
4012	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4013	{
4014	int rc = VINF_SUCCESS;
4015	STAM_PROFILE_START(&pPool->StatFlushPage, f);
4016	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4017	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4018
4019	/*
4020	* Quietly reject any attempts at flushing any of the special root pages.
4021	*/
4022	if (pPage->idx < PGMPOOL_IDX_FIRST)
4023	{
4024	Log(("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4025	return VINF_SUCCESS;
4026	}
4027
4028	/*
4029	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4030	*/
4031	if (pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage))
4032	{
4033	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4034	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4035	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4036	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
4037	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4038	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4039	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4040	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4041	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
4042	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(pPool->CTX_SUFF(pVM)), pPage->Core.Key, pPage->enmKind));
4043	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4044	return VINF_SUCCESS;
4045	}
4046
4047	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4048	/* Start a subset so we won't run out of mapping space. */
4049	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
4050	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
4051	#endif
4052
4053	/*
4054	* Mark the page as being in need of a ASMMemZeroPage().
4055	*/
4056	pPage->fZeroed = false;
4057
4058	#ifdef PGMPOOL_WITH_USER_TRACKING
4059	/*
4060	* Clear the page.
4061	*/
4062	pgmPoolTrackClearPageUsers(pPool, pPage);
4063	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
4064	pgmPoolTrackDeref(pPool, pPage);
4065	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
4066	#endif
4067
4068	#ifdef PGMPOOL_WITH_CACHE
4069	/*
4070	* Flush it from the cache.
4071	*/
4072	pgmPoolCacheFlushPage(pPool, pPage);
4073	#endif /* PGMPOOL_WITH_CACHE */
4074
4075	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4076	/* Heavy stuff done. */
4077	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
4078	#endif
4079
4080	#ifdef PGMPOOL_WITH_MONITORING
4081	/*
4082	* Deregistering the monitoring.
4083	*/
4084	if (pPage->fMonitored)
4085	rc = pgmPoolMonitorFlush(pPool, pPage);
4086	#endif
4087
4088	/*
4089	* Free the page.
4090	*/
4091	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
4092	pPage->iNext = pPool->iFreeHead;
4093	pPool->iFreeHead = pPage->idx;
4094	pPage->enmKind = PGMPOOLKIND_FREE;
4095	pPage->GCPhys = NIL_RTGCPHYS;
4096	pPage->fReusedFlushPending = false;
4097
4098	pPool->cUsedPages--;
4099	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
4100	return rc;
4101	}
4102
4103
4104	/**
4105	* Frees a usage of a pool page.
4106	*
4107	* The caller is responsible to updating the user table so that it no longer
4108	* references the shadow page.
4109	*
4110	* @param pPool The pool.
4111	* @param HCPhys The HC physical address of the shadow page.
4112	* @param iUser The shadow page pool index of the user table.
4113	* @param iUserTable The index into the user table (shadowed).
4114	*/
4115	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
4116	{
4117	STAM_PROFILE_START(&pPool->StatFree, a);
4118	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%#x iUserTable=%#x\n",
4119	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
4120	Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
4121	#ifdef PGMPOOL_WITH_USER_TRACKING
4122	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
4123	#endif
4124	#ifdef PGMPOOL_WITH_CACHE
4125	if (!pPage->fCached)
4126	#endif
4127	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED can be ignored here. */
4128	STAM_PROFILE_STOP(&pPool->StatFree, a);
4129	}
4130
4131
4132	/**
4133	* Makes one or more free page free.
4134	*
4135	* @returns VBox status code.
4136	* @retval VINF_SUCCESS on success.
4137	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4138	*
4139	* @param pPool The pool.
4140	* @param enmKind Page table kind
4141	* @param iUser The user of the page.
4142	*/
4143	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
4144	{
4145	LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
4146
4147	/*
4148	* If the pool isn't full grown yet, expand it.
4149	*/
4150	if ( pPool->cCurPages < pPool->cMaxPages
4151	#if defined(IN_RC)
4152	/* Hack alert: we can't deal with jumps to ring 3 when called from MapCR3 and allocating pages for PAE PDs. */
4153	&& enmKind != PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4154	&& (enmKind < PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD \|\| enmKind > PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD)
4155	#endif
4156	)
4157	{
4158	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
4159	#ifdef IN_RING3
4160	int rc = PGMR3PoolGrow(pPool->pVMR3);
4161	#else
4162	int rc = CTXALLMID(VMM, CallHost)(pPool->CTX_SUFF(pVM), VMMCALLHOST_PGM_POOL_GROW, 0);
4163	#endif
4164	if (RT_FAILURE(rc))
4165	return rc;
4166	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
4167	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
4168	return VINF_SUCCESS;
4169	}
4170
4171	#ifdef PGMPOOL_WITH_CACHE
4172	/*
4173	* Free one cached page.
4174	*/
4175	return pgmPoolCacheFreeOne(pPool, iUser);
4176	#else
4177	/*
4178	* Flush the pool.
4179	*
4180	* If we have tracking enabled, it should be possible to come up with
4181	* a cheap replacement strategy...
4182	*/
4183	/* @todo incompatible with long mode paging (cr3 root will be flushed) */
4184	Assert(!CPUMIsGuestInLongMode(pVM));
4185	pgmPoolFlushAllInt(pPool);
4186	return VERR_PGM_POOL_FLUSHED;
4187	#endif
4188	}
4189
4190
4191	/**
4192	* Allocates a page from the pool.
4193	*
4194	* This page may actually be a cached page and not in need of any processing
4195	* on the callers part.
4196	*
4197	* @returns VBox status code.
4198	* @retval VINF_SUCCESS if a NEW page was allocated.
4199	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4200	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4201	* @param pVM The VM handle.
4202	* @param GCPhys The GC physical address of the page we're gonna shadow.
4203	* For 4MB and 2MB PD entries, it's the first address the
4204	* shadow PT is covering.
4205	* @param enmKind The kind of mapping.
4206	* @param iUser The shadow page pool index of the user table.
4207	* @param iUserTable The index into the user table (shadowed).
4208	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4209	*/
4210	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
4211	{
4212	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4213	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4214	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%#x iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
4215	*ppPage = NULL;
4216	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4217	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
4218	* Assert(!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
4219
4220	#ifdef PGMPOOL_WITH_CACHE
4221	if (pPool->fCacheEnabled)
4222	{
4223	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, iUser, iUserTable, ppPage);
4224	if (RT_SUCCESS(rc2))
4225	{
4226	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4227	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
4228	return rc2;
4229	}
4230	}
4231	#endif
4232
4233	/*
4234	* Allocate a new one.
4235	*/
4236	int rc = VINF_SUCCESS;
4237	uint16_t iNew = pPool->iFreeHead;
4238	if (iNew == NIL_PGMPOOL_IDX)
4239	{
4240	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
4241	if (RT_FAILURE(rc))
4242	{
4243	if (rc != VERR_PGM_POOL_CLEARED)
4244	{
4245	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
4246	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4247	return rc;
4248	}
4249	Log(("pgmPoolMakeMoreFreePages failed with %Rrc -> return VERR_PGM_POOL_FLUSHED\n", rc));
4250	rc = VERR_PGM_POOL_FLUSHED;
4251	}
4252	iNew = pPool->iFreeHead;
4253	AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
4254	}
4255
4256	/* unlink the free head */
4257	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
4258	pPool->iFreeHead = pPage->iNext;
4259	pPage->iNext = NIL_PGMPOOL_IDX;
4260
4261	/*
4262	* Initialize it.
4263	*/
4264	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
4265	pPage->enmKind = enmKind;
4266	pPage->GCPhys = GCPhys;
4267	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
4268	pPage->fMonitored = false;
4269	pPage->fCached = false;
4270	pPage->fReusedFlushPending = false;
4271	#ifdef PGMPOOL_WITH_MONITORING
4272	pPage->cModifications = 0;
4273	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4274	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4275	#else
4276	pPage->fCR3Mix = false;
4277	#endif
4278	#ifdef PGMPOOL_WITH_USER_TRACKING
4279	pPage->cPresent = 0;
4280	pPage->iFirstPresent = ~0;
4281
4282	/*
4283	* Insert into the tracking and cache. If this fails, free the page.
4284	*/
4285	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
4286	if (RT_FAILURE(rc3))
4287	{
4288	if (rc3 != VERR_PGM_POOL_CLEARED)
4289	{
4290	pPool->cUsedPages--;
4291	pPage->enmKind = PGMPOOLKIND_FREE;
4292	pPage->GCPhys = NIL_RTGCPHYS;
4293	pPage->iNext = pPool->iFreeHead;
4294	pPool->iFreeHead = pPage->idx;
4295	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4296	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
4297	return rc3;
4298	}
4299	Log(("pgmPoolTrackInsert failed with %Rrc -> return VERR_PGM_POOL_FLUSHED\n", rc3));
4300	rc = VERR_PGM_POOL_FLUSHED;
4301	}
4302	#endif /* PGMPOOL_WITH_USER_TRACKING */
4303
4304	/*
4305	* Commit the allocation, clear the page and return.
4306	*/
4307	#ifdef VBOX_WITH_STATISTICS
4308	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
4309	pPool->cUsedPagesHigh = pPool->cUsedPages;
4310	#endif
4311
4312	if (!pPage->fZeroed)
4313	{
4314	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4315	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4316	ASMMemZeroPage(pv);
4317	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4318	}
4319
4320	*ppPage = pPage;
4321	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
4322	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
4323	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4324	return rc;
4325	}
4326
4327
4328	/**
4329	* Frees a usage of a pool page.
4330	*
4331	* @param pVM The VM handle.
4332	* @param HCPhys The HC physical address of the shadow page.
4333	* @param iUser The shadow page pool index of the user table.
4334	* @param iUserTable The index into the user table (shadowed).
4335	*/
4336	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
4337	{
4338	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
4339	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4340	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
4341	}
4342
4343
4344	/**
4345	* Gets a in-use page in the pool by it's physical address.
4346	*
4347	* @returns Pointer to the page.
4348	* @param pVM The VM handle.
4349	* @param HCPhys The HC physical address of the shadow page.
4350	* @remark This function will NEVER return NULL. It will assert if HCPhys is invalid.
4351	*/
4352	PPGMPOOLPAGE pgmPoolGetPageByHCPhys(PVM pVM, RTHCPHYS HCPhys)
4353	{
4354	/** @todo profile this! */
4355	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4356	PPGMPOOLPAGE pPage = pgmPoolGetPage(pPool, HCPhys);
4357	Log4(("pgmPoolGetPageByHCPhys: HCPhys=%RHp -> %p:{.idx=%d .GCPhys=%RGp .enmKind=%s}\n",
4358	HCPhys, pPage, pPage->idx, pPage->GCPhys, pgmPoolPoolKindToStr(pPage->enmKind)));
4359	return pPage;
4360	}
4361
4362	#ifdef IN_RING3
4363	/**
4364	* Flushes the entire cache.
4365	*
4366	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
4367	* and execute this CR3 flush.
4368	*
4369	* @param pPool The pool.
4370	*/
4371	void pgmPoolFlushAll(PVM pVM)
4372	{
4373	LogFlow(("pgmPoolFlushAll:\n"));
4374	pgmPoolFlushAllInt(pVM->pgm.s.CTX_SUFF(pPool));
4375	}
4376	#endif
4377
4378	#ifdef LOG_ENABLED
4379	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
4380	{
4381	switch(enmKind)
4382	{
4383	case PGMPOOLKIND_INVALID:
4384	return "PGMPOOLKIND_INVALID";
4385	case PGMPOOLKIND_FREE:
4386	return "PGMPOOLKIND_FREE";
4387	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4388	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
4389	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4390	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
4391	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4392	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
4393	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4394	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
4395	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4396	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
4397	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4398	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
4399	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4400	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
4401	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4402	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
4403	case PGMPOOLKIND_32BIT_PD:
4404	return "PGMPOOLKIND_32BIT_PD";
4405	case PGMPOOLKIND_32BIT_PD_PHYS:
4406	return "PGMPOOLKIND_32BIT_PD_PHYS";
4407	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4408	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
4409	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4410	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
4411	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4412	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
4413	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4414	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
4415	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4416	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
4417	case PGMPOOLKIND_PAE_PD_PHYS:
4418	return "PGMPOOLKIND_PAE_PD_PHYS";
4419	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4420	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
4421	case PGMPOOLKIND_PAE_PDPT:
4422	return "PGMPOOLKIND_PAE_PDPT";
4423	case PGMPOOLKIND_PAE_PDPT_PHYS:
4424	return "PGMPOOLKIND_PAE_PDPT_PHYS";
4425	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4426	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
4427	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4428	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
4429	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4430	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
4431	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4432	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
4433	case PGMPOOLKIND_64BIT_PML4:
4434	return "PGMPOOLKIND_64BIT_PML4";
4435	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4436	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
4437	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4438	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
4439	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4440	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
4441	case PGMPOOLKIND_ROOT_NESTED:
4442	return "PGMPOOLKIND_ROOT_NESTED";
4443	}
4444	return "Unknown kind!";
4445	}
4446	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 17641

Download in other formats: