PGMAllPool.cpp@ 16634

Last change on this file since 16634 was 16634, checked in by vboxsync, 16 years ago
VBOX_WITH_PGMPOOL_PAGING_ONLY: fixed invalidation of pae/32 bit pds
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 162.1 KB

Line
1	/* $Id: PGMAllPool.cpp 16634 2009-02-10 14:54:02Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_PGM_POOL
27	#include <VBox/pgm.h>
28	#include <VBox/mm.h>
29	#include <VBox/em.h>
30	#include <VBox/cpum.h>
31	#ifdef IN_RC
32	# include <VBox/patm.h>
33	#endif
34	#include "PGMInternal.h"
35	#include <VBox/vm.h>
36	#include <VBox/disopcode.h>
37	#include <VBox/hwacc_vmx.h>
38
39	#include <VBox/log.h>
40	#include <VBox/err.h>
41	#include <iprt/asm.h>
42
43
44	/*******************************************************************************
45	* Internal Functions *
46	*******************************************************************************/
47	__BEGIN_DECLS
48	static void pgmPoolFlushAllInt(PPGMPOOL pPool);
49	#ifdef PGMPOOL_WITH_USER_TRACKING
50	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
51	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
52	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
53	#endif
54	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
55	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint);
56	#endif
57	#ifdef PGMPOOL_WITH_CACHE
58	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
59	#endif
60	#ifdef PGMPOOL_WITH_MONITORING
61	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
62	#endif
63	#ifndef IN_RING3
64	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
65	#endif
66	__END_DECLS
67
68
69	/**
70	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
71	*
72	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
73	* @param enmKind The page kind.
74	*/
75	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
76	{
77	switch (enmKind)
78	{
79	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
80	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
81	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
82	return true;
83	default:
84	return false;
85	}
86	}
87
88
89	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
90	/**
91	* Maps a pool page into the current context.
92	*
93	* @returns Pointer to the mapping.
94	* @param pPGM Pointer to the PGM instance data.
95	* @param pPage The page to map.
96	*/
97	void *pgmPoolMapPageFallback(PPGM pPGM, PPGMPOOLPAGE pPage)
98	{
99	/* general pages are take care of by the inlined part, it
100	only ends up here in case of failure. */
101	AssertReleaseReturn(pPage->idx < PGMPOOL_IDX_FIRST, NULL);
102
103	/** @todo make sure HCPhys is valid for all indexes. */
104	/* special pages. */
105	# ifdef IN_RC
106	switch (pPage->idx)
107	{
108	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
109	case PGMPOOL_IDX_PD:
110	case PGMPOOL_IDX_PDPT:
111	case PGMPOOL_IDX_AMD64_CR3:
112	return pPGM->pShwRootRC;
113	# else
114	case PGMPOOL_IDX_PD:
115	return pPGM->pShw32BitPdRC;
116	case PGMPOOL_IDX_PAE_PD:
117	case PGMPOOL_IDX_PAE_PD_0:
118	return pPGM->apShwPaePDsRC[0];
119	case PGMPOOL_IDX_PAE_PD_1:
120	return pPGM->apShwPaePDsRC[1];
121	case PGMPOOL_IDX_PAE_PD_2:
122	return pPGM->apShwPaePDsRC[2];
123	case PGMPOOL_IDX_PAE_PD_3:
124	return pPGM->apShwPaePDsRC[3];
125	case PGMPOOL_IDX_PDPT:
126	return pPGM->pShwPaePdptRC;
127	# endif
128	default:
129	AssertReleaseMsgFailed(("Invalid index %d\n", pPage->idx));
130	return NULL;
131	}
132
133	# else /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
134	RTHCPHYS HCPhys;
135	switch (pPage->idx)
136	{
137	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
138	case PGMPOOL_IDX_PD:
139	case PGMPOOL_IDX_PDPT:
140	case PGMPOOL_IDX_AMD64_CR3:
141	HCPhys = pPGM->HCPhysShwCR3;
142	break;
143
144	case PGMPOOL_IDX_NESTED_ROOT:
145	HCPhys = pPGM->HCPhysShwNestedRoot;
146	break;
147	# else
148	case PGMPOOL_IDX_PD:
149	HCPhys = pPGM->HCPhysShw32BitPD;
150	break;
151	case PGMPOOL_IDX_PAE_PD_0:
152	HCPhys = pPGM->aHCPhysPaePDs[0];
153	break;
154	case PGMPOOL_IDX_PAE_PD_1:
155	HCPhys = pPGM->aHCPhysPaePDs[1];
156	break;
157	case PGMPOOL_IDX_PAE_PD_2:
158	HCPhys = pPGM->aHCPhysPaePDs[2];
159	break;
160	case PGMPOOL_IDX_PAE_PD_3:
161	HCPhys = pPGM->aHCPhysPaePDs[3];
162	break;
163	case PGMPOOL_IDX_PDPT:
164	HCPhys = pPGM->HCPhysShwPaePdpt;
165	break;
166	case PGMPOOL_IDX_NESTED_ROOT:
167	HCPhys = pPGM->HCPhysShwNestedRoot;
168	break;
169	case PGMPOOL_IDX_PAE_PD:
170	AssertReleaseMsgFailed(("PGMPOOL_IDX_PAE_PD is not usable in VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 context\n"));
171	return NULL;
172	# endif
173	default:
174	AssertReleaseMsgFailed(("Invalid index %d\n", pPage->idx));
175	return NULL;
176	}
177	AssertMsg(HCPhys && HCPhys != NIL_RTHCPHYS && !(PAGE_OFFSET_MASK & HCPhys), ("%RHp\n", HCPhys));
178
179	void *pv;
180	pgmR0DynMapHCPageInlined(pPGM, HCPhys, &pv);
181	return pv;
182	# endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
183	}
184	#endif /* IN_RC \|\| VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
185
186
187	#ifdef PGMPOOL_WITH_MONITORING
188	/**
189	* Determin the size of a write instruction.
190	* @returns number of bytes written.
191	* @param pDis The disassembler state.
192	*/
193	static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
194	{
195	/*
196	* This is very crude and possibly wrong for some opcodes,
197	* but since it's not really supposed to be called we can
198	* probably live with that.
199	*/
200	return DISGetParamSize(pDis, &pDis->param1);
201	}
202
203
204	/**
205	* Flushes a chain of pages sharing the same access monitor.
206	*
207	* @returns VBox status code suitable for scheduling.
208	* @param pPool The pool.
209	* @param pPage A page in the chain.
210	*/
211	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
212	{
213	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
214
215	/*
216	* Find the list head.
217	*/
218	uint16_t idx = pPage->idx;
219	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
220	{
221	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
222	{
223	idx = pPage->iMonitoredPrev;
224	Assert(idx != pPage->idx);
225	pPage = &pPool->aPages[idx];
226	}
227	}
228
229	/*
230	* Iterate the list flushing each shadow page.
231	*/
232	int rc = VINF_SUCCESS;
233	for (;;)
234	{
235	idx = pPage->iMonitoredNext;
236	Assert(idx != pPage->idx);
237	if (pPage->idx >= PGMPOOL_IDX_FIRST)
238	{
239	int rc2 = pgmPoolFlushPage(pPool, pPage);
240	if (rc2 == VERR_PGM_POOL_CLEARED && rc == VINF_SUCCESS)
241	rc = VINF_PGM_SYNC_CR3;
242	}
243	/* next */
244	if (idx == NIL_PGMPOOL_IDX)
245	break;
246	pPage = &pPool->aPages[idx];
247	}
248	return rc;
249	}
250
251
252	/**
253	* Wrapper for getting the current context pointer to the entry being modified.
254	*
255	* @returns Pointer to the current context mapping of the entry.
256	* @param pPool The pool.
257	* @param pvFault The fault virtual address.
258	* @param GCPhysFault The fault physical address.
259	* @param cbEntry The entry size.
260	*/
261	#ifdef IN_RING3
262	DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTHCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
263	#else
264	DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTGCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
265	#endif
266	{
267	#ifdef IN_RC
268	return (const void *)((RTGCUINTPTR)pvFault & ~(RTGCUINTPTR)(cbEntry - 1));
269
270	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
271	void *pvRet;
272	int rc = PGMDynMapGCPageOff(pPool->pVMR0, GCPhysFault & ~(RTGCPHYS)(cbEntry - 1), &pvRet);
273	AssertFatalRCSuccess(rc);
274	return pvRet;
275
276	#elif defined(IN_RING0)
277	void *pvRet;
278	int rc = pgmRamGCPhys2HCPtr(&pPool->pVMR0->pgm.s, GCPhysFault & ~(RTGCPHYS)(cbEntry - 1), &pvRet);
279	AssertFatalRCSuccess(rc);
280	return pvRet;
281
282	#elif defined(IN_RING3)
283	return (RTHCPTR)((uintptr_t)pvFault & ~(RTHCUINTPTR)(cbEntry - 1));
284	#else
285	# error "huh?"
286	#endif
287	}
288
289
290	/**
291	* Process shadow entries before they are changed by the guest.
292	*
293	* For PT entries we will clear them. For PD entries, we'll simply check
294	* for mapping conflicts and set the SyncCR3 FF if found.
295	*
296	* @param pPool The pool.
297	* @param pPage The head page.
298	* @param GCPhysFault The guest physical fault address.
299	* @param uAddress In R0 and GC this is the guest context fault address (flat).
300	* In R3 this is the host context 'fault' address.
301	* @param pCpu The disassembler state for figuring out the write size.
302	* This need not be specified if the caller knows we won't do cross entry accesses.
303	*/
304	#ifdef IN_RING3
305	void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTHCPTR pvAddress, PDISCPUSTATE pCpu)
306	#else
307	void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTGCPTR pvAddress, PDISCPUSTATE pCpu)
308	#endif
309	{
310	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
311	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
312	const unsigned cbWrite = (pCpu) ? pgmPoolDisasWriteSize(pCpu) : 0;
313
314	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp kind=%d cbWrite=%d\n", pvAddress, GCPhysFault, pPage->enmKind, cbWrite));
315
316	for (;;)
317	{
318	union
319	{
320	void *pv;
321	PX86PT pPT;
322	PX86PTPAE pPTPae;
323	PX86PD pPD;
324	PX86PDPAE pPDPae;
325	PX86PDPT pPDPT;
326	PX86PML4 pPML4;
327	} uShw;
328
329	switch (pPage->enmKind)
330	{
331	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
332	{
333	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
334	const unsigned iShw = off / sizeof(X86PTE);
335	if (uShw.pPT->a[iShw].n.u1Present)
336	{
337	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
338	PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
339	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
340	pgmPoolTracDerefGCPhysHint(pPool, pPage,
341	uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
342	pGstPte->u & X86_PTE_PG_MASK);
343	# endif
344	uShw.pPT->a[iShw].u = 0;
345	}
346	break;
347	}
348
349	/* page/2 sized */
350	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
351	{
352	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
353	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
354	{
355	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
356	if (uShw.pPTPae->a[iShw].n.u1Present)
357	{
358	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
359	PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
360	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
361	pgmPoolTracDerefGCPhysHint(pPool, pPage,
362	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
363	pGstPte->u & X86_PTE_PG_MASK);
364	# endif
365	uShw.pPTPae->a[iShw].u = 0;
366	}
367	}
368	break;
369	}
370
371	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
372	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
373	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
374	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
375	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
376	{
377	unsigned iGst = off / sizeof(X86PDE);
378	unsigned iShwPdpt = iGst / 256;
379	unsigned iShw = (iGst % 256) * 2;
380	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
381
382	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: idx = %d\n", iShwPdpt));
383	if (iShwPdpt == pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
384	{
385	for (unsigned i=0;i<2;i++)
386	{
387	if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
388	{
389	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
390	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
391	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
392	}
393	else
394	if (uShw.pPDPae->a[iShw+i].n.u1Present)
395	{
396	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
397	pgmPoolFree(pPool->CTX_SUFF(pVM),
398	uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
399	pPage->idx,
400	iShw + i);
401	uShw.pPDPae->a[iShw+i].u = 0;
402	}
403
404	/* paranoia / a bit assumptive. */
405	if ( pCpu
406	&& (off & 3)
407	&& (off & 3) + cbWrite > 4)
408	{
409	const unsigned iShw2 = iShw + 2 + i;
410	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
411	{
412	if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
413	{
414	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
415	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
416	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
417	}
418	else
419	if (uShw.pPDPae->a[iShw2].n.u1Present)
420	{
421	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
422	pgmPoolFree(pPool->CTX_SUFF(pVM),
423	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
424	pPage->idx,
425	iShw2);
426	uShw.pPDPae->a[iShw2].u = 0;
427	}
428	}
429	}
430	}
431	}
432	break;
433	}
434	# endif
435
436
437	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
438	{
439	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
440	const unsigned iShw = off / sizeof(X86PTEPAE);
441	if (uShw.pPTPae->a[iShw].n.u1Present)
442	{
443	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
444	PCX86PTEPAE pGstPte = (PCX86PTEPAE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
445	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PAE_PG_MASK));
446	pgmPoolTracDerefGCPhysHint(pPool, pPage,
447	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
448	pGstPte->u & X86_PTE_PAE_PG_MASK);
449	# endif
450	uShw.pPTPae->a[iShw].u = 0;
451	}
452
453	/* paranoia / a bit assumptive. */
454	if ( pCpu
455	&& (off & 7)
456	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
457	{
458	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
459	AssertReturnVoid(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
460
461	if (uShw.pPTPae->a[iShw2].n.u1Present)
462	{
463	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
464	PCX86PTEPAE pGstPte = (PCX86PTEPAE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
465	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PAE_PG_MASK));
466	pgmPoolTracDerefGCPhysHint(pPool, pPage,
467	uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK,
468	pGstPte->u & X86_PTE_PAE_PG_MASK);
469	# endif
470	uShw.pPTPae->a[iShw2].u = 0;
471	}
472	}
473
474	break;
475	}
476
477	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
478	case PGMPOOLKIND_32BIT_PD:
479	# else
480	case PGMPOOLKIND_ROOT_32BIT_PD:
481	# endif
482	{
483	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
484	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
485	if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
486	{
487	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
488	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
489	STAM_COUNTER_INC(&(pPool->CTX_SUFF(pVM)->pgm.s.StatRZGuestCR3WriteConflict));
490	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
491	}
492	/* paranoia / a bit assumptive. */
493	else if ( pCpu
494	&& (off & 3)
495	&& (off & 3) + cbWrite > sizeof(X86PTE))
496	{
497	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
498	if ( iShw2 != iShw
499	&& iShw2 < RT_ELEMENTS(uShw.pPD->a)
500	&& uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
501	{
502	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
503	STAM_COUNTER_INC(&(pPool->CTX_SUFF(pVM)->pgm.s.StatRZGuestCR3WriteConflict));
504	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
505	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
506	}
507	}
508	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
509	if ( uShw.pPD->a[iShw].n.u1Present
510	&& !VM_FF_ISSET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3))
511	{
512	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
513	# ifdef IN_RC /* TLB load - we're pushing things a bit... */
514	ASMProbeReadByte(pvAddress);
515	# endif
516	pgmPoolFree(pPool->CTX_SUFF(pVM), uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
517	uShw.pPD->a[iShw].u = 0;
518	}
519	#endif
520	break;
521	}
522
523	# ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
524	case PGMPOOLKIND_ROOT_PAE_PD:
525	{
526	unsigned iGst = off / sizeof(X86PDE); // ASSUMING 32-bit guest paging!
527	unsigned iShwPdpt = iGst / 256;
528	unsigned iShw = (iGst % 256) * 2;
529	Assert(pPage->idx == PGMPOOL_IDX_PAE_PD);
530	PPGMPOOLPAGE pPage2 = pPage + 1 + iShwPdpt;
531	Assert(pPage2->idx == PGMPOOL_IDX_PAE_PD_0 + iShwPdpt);
532	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage2);
533	for (unsigned i = 0; i < 2; i++, iShw++)
534	{
535	if ((uShw.pPDPae->a[iShw].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
536	{
537	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
538	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
539	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw));
540	}
541	/* paranoia / a bit assumptive. */
542	else if ( pCpu
543	&& (off & 3)
544	&& (off & 3) + cbWrite > 4)
545	{
546	const unsigned iShw2 = iShw + 2;
547	if ( iShw2 < RT_ELEMENTS(uShw.pPDPae->a) /** @todo was completely wrong, it's better now after #1865 but still wrong from cross PD. */
548	&& (uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
549	{
550	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
551	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
552	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
553	}
554	}
555	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
556	if ( uShw.pPDPae->a[iShw].n.u1Present
557	&& !VM_FF_ISSET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3))
558	{
559	LogFlow(("pgmPoolMonitorChainChanging: iShwPdpt=%#x iShw=%#x: %RX64 -> freeing it!\n", iShwPdpt, iShw, uShw.pPDPae->a[iShw].u));
560	# ifdef IN_RC /* TLB load - we're pushing things a bit... */
561	ASMProbeReadByte(pvAddress);
562	# endif
563	pgmPoolFree(pPool->CTX_SUFF(pVM), uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK, pPage->idx, iShw + iShwPdpt * X86_PG_PAE_ENTRIES);
564	uShw.pPDPae->a[iShw].u = 0;
565	}
566	#endif
567	}
568	break;
569	}
570	# endif /* !VBOX_WITH_PGMPOOL_PAGING_ONLY */
571
572	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
573	{
574	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
575	const unsigned iShw = off / sizeof(X86PDEPAE);
576	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
577	{
578	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
579	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
580	STAM_COUNTER_INC(&(pPool->CTX_SUFF(pVM)->pgm.s.StatRZGuestCR3WriteConflict));
581	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
582	}
583	#ifdef PGMPOOL_INVALIDATE_UPPER_SHADOW_TABLE_ENTRIES
584	/*
585	* Causes trouble when the guest uses a PDE to refer to the whole page table level
586	* structure. (Invalidate here; faults later on when it tries to change the page
587	* table entries -> recheck; probably only applies to the RC case.)
588	*/
589	else
590	{
591	if (uShw.pPDPae->a[iShw].n.u1Present)
592	{
593	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
594	pgmPoolFree(pPool->CTX_SUFF(pVM),
595	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
596	/* Note: hardcoded PAE implementation dependency */
597	(pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD) ? PGMPOOL_IDX_PAE_PD : pPage->idx,
598	(pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD) ? iShw + (pPage->idx - PGMPOOL_IDX_PAE_PD_0) * X86_PG_PAE_ENTRIES : iShw);
599	uShw.pPDPae->a[iShw].u = 0;
600	}
601	}
602	#endif
603	/* paranoia / a bit assumptive. */
604	if ( pCpu
605	&& (off & 7)
606	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
607	{
608	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
609	AssertReturnVoid(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
610
611	if ( iShw2 != iShw
612	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
613	{
614	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
615	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
616	STAM_COUNTER_INC(&(pPool->CTX_SUFF(pVM)->pgm.s.StatRZGuestCR3WriteConflict));
617	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
618	}
619	#ifdef PGMPOOL_INVALIDATE_UPPER_SHADOW_TABLE_ENTRIES
620	else if (uShw.pPDPae->a[iShw2].n.u1Present)
621	{
622	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
623	pgmPoolFree(pPool->CTX_SUFF(pVM),
624	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
625	/* Note: hardcoded PAE implementation dependency */
626	(pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD) ? PGMPOOL_IDX_PAE_PD : pPage->idx,
627	(pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD) ? iShw2 + (pPage->idx - PGMPOOL_IDX_PAE_PD_0) * X86_PG_PAE_ENTRIES : iShw2);
628	uShw.pPDPae->a[iShw2].u = 0;
629	}
630	#endif
631	}
632	break;
633	}
634
635	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
636	case PGMPOOLKIND_PAE_PDPT:
637	# else
638	case PGMPOOLKIND_ROOT_PDPT:
639	# endif
640	{
641	/*
642	* Hopefully this doesn't happen very often:
643	* - touching unused parts of the page
644	* - messing with the bits of pd pointers without changing the physical address
645	*/
646	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
647	const unsigned iShw = off / sizeof(X86PDPE);
648	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
649	{
650	if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
651	{
652	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
653	STAM_COUNTER_INC(&(pPool->CTX_SUFF(pVM)->pgm.s.StatRZGuestCR3WriteConflict));
654	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
655	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
656	}
657	/* paranoia / a bit assumptive. */
658	else if ( pCpu
659	&& (off & 7)
660	&& (off & 7) + cbWrite > sizeof(X86PDPE))
661	{
662	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
663	if ( iShw2 != iShw
664	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES
665	&& uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
666	{
667	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
668	STAM_COUNTER_INC(&(pPool->CTX_SUFF(pVM)->pgm.s.StatRZGuestCR3WriteConflict));
669	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
670	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
671	}
672	}
673	}
674	break;
675	}
676
677	#ifndef IN_RC
678	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
679	{
680	Assert(pPage->enmKind == PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD);
681
682	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
683	const unsigned iShw = off / sizeof(X86PDEPAE);
684	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
685	{
686	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
687	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
688	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
689	}
690	else
691	{
692	if (uShw.pPDPae->a[iShw].n.u1Present)
693	{
694	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
695	pgmPoolFree(pPool->CTX_SUFF(pVM),
696	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
697	pPage->idx,
698	iShw);
699	uShw.pPDPae->a[iShw].u = 0;
700	}
701	}
702	/* paranoia / a bit assumptive. */
703	if ( pCpu
704	&& (off & 7)
705	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
706	{
707	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
708	AssertReturnVoid(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
709
710	if ( iShw2 != iShw
711	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
712	{
713	Assert(pgmMapAreMappingsEnabled(&pPool->CTX_SUFF(pVM)->pgm.s));
714	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
715	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
716	}
717	else
718	if (uShw.pPDPae->a[iShw2].n.u1Present)
719	{
720	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
721	pgmPoolFree(pPool->CTX_SUFF(pVM),
722	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
723	pPage->idx,
724	iShw2);
725	uShw.pPDPae->a[iShw2].u = 0;
726	}
727	}
728	break;
729	}
730
731	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
732	{
733	/*
734	* Hopefully this doesn't happen very often:
735	* - messing with the bits of pd pointers without changing the physical address
736	*/
737	if (!VM_FF_ISSET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3))
738	{
739	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
740	const unsigned iShw = off / sizeof(X86PDPE);
741	if (uShw.pPDPT->a[iShw].n.u1Present)
742	{
743	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
744	pgmPoolFree(pPool->CTX_SUFF(pVM), uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
745	uShw.pPDPT->a[iShw].u = 0;
746	}
747	/* paranoia / a bit assumptive. */
748	if ( pCpu
749	&& (off & 7)
750	&& (off & 7) + cbWrite > sizeof(X86PDPE))
751	{
752	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
753	if (uShw.pPDPT->a[iShw2].n.u1Present)
754	{
755	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
756	pgmPoolFree(pPool->CTX_SUFF(pVM), uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
757	uShw.pPDPT->a[iShw2].u = 0;
758	}
759	}
760	}
761	break;
762	}
763
764	case PGMPOOLKIND_64BIT_PML4:
765	{
766	/*
767	* Hopefully this doesn't happen very often:
768	* - messing with the bits of pd pointers without changing the physical address
769	*/
770	if (!VM_FF_ISSET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3))
771	{
772	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
773	const unsigned iShw = off / sizeof(X86PDPE);
774	if (uShw.pPML4->a[iShw].n.u1Present)
775	{
776	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
777	pgmPoolFree(pPool->CTX_SUFF(pVM), uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
778	uShw.pPML4->a[iShw].u = 0;
779	}
780	/* paranoia / a bit assumptive. */
781	if ( pCpu
782	&& (off & 7)
783	&& (off & 7) + cbWrite > sizeof(X86PDPE))
784	{
785	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
786	if (uShw.pPML4->a[iShw2].n.u1Present)
787	{
788	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
789	pgmPoolFree(pPool->CTX_SUFF(pVM), uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
790	uShw.pPML4->a[iShw2].u = 0;
791	}
792	}
793	}
794	break;
795	}
796	#endif /* IN_RING0 */
797
798	default:
799	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
800	}
801
802	/* next */
803	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
804	return;
805	pPage = &pPool->aPages[pPage->iMonitoredNext];
806	}
807	}
808
809
810	# ifndef IN_RING3
811	/**
812	* Checks if a access could be a fork operation in progress.
813	*
814	* Meaning, that the guest is setuping up the parent process for Copy-On-Write.
815	*
816	* @returns true if it's likly that we're forking, otherwise false.
817	* @param pPool The pool.
818	* @param pCpu The disassembled instruction.
819	* @param offFault The access offset.
820	*/
821	DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pCpu, unsigned offFault)
822	{
823	/*
824	* i386 linux is using btr to clear X86_PTE_RW.
825	* The functions involved are (2.6.16 source inspection):
826	* clear_bit
827	* ptep_set_wrprotect
828	* copy_one_pte
829	* copy_pte_range
830	* copy_pmd_range
831	* copy_pud_range
832	* copy_page_range
833	* dup_mmap
834	* dup_mm
835	* copy_mm
836	* copy_process
837	* do_fork
838	*/
839	if ( pCpu->pCurInstr->opcode == OP_BTR
840	&& !(offFault & 4)
841	/** @todo Validate that the bit index is X86_PTE_RW. */
842	)
843	{
844	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,Fork));
845	return true;
846	}
847	return false;
848	}
849
850
851	/**
852	* Determine whether the page is likely to have been reused.
853	*
854	* @returns true if we consider the page as being reused for a different purpose.
855	* @returns false if we consider it to still be a paging page.
856	* @param pVM VM Handle.
857	* @param pPage The page in question.
858	* @param pRegFrame Trap register frame.
859	* @param pCpu The disassembly info for the faulting instruction.
860	* @param pvFault The fault address.
861	*
862	* @remark The REP prefix check is left to the caller because of STOSD/W.
863	*/
864	DECLINLINE(bool) pgmPoolMonitorIsReused(PVM pVM, PPGMPOOLPAGE pPage, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pCpu, RTGCPTR pvFault)
865	{
866	#ifndef IN_RC
867	/** @todo could make this general, faulting close to rsp should be safe reuse heuristic. */
868	if ( HWACCMHasPendingIrq(pVM)
869	&& (pRegFrame->rsp - pvFault) < 32)
870	{
871	/* Fault caused by stack writes while trying to inject an interrupt event. */
872	Log(("pgmPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
873	return true;
874	}
875	#else
876	NOREF(pVM); NOREF(pvFault);
877	#endif
878
879	switch (pCpu->pCurInstr->opcode)
880	{
881	/* call implies the actual push of the return address faulted */
882	case OP_CALL:
883	Log4(("pgmPoolMonitorIsReused: CALL\n"));
884	return true;
885	case OP_PUSH:
886	Log4(("pgmPoolMonitorIsReused: PUSH\n"));
887	return true;
888	case OP_PUSHF:
889	Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
890	return true;
891	case OP_PUSHA:
892	Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
893	return true;
894	case OP_FXSAVE:
895	Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
896	return true;
897	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
898	Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
899	return true;
900	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
901	Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
902	return true;
903	case OP_MOVSWD:
904	case OP_STOSWD:
905	if ( pCpu->prefix == (PREFIX_REP\|PREFIX_REX)
906	&& pRegFrame->rcx >= 0x40
907	)
908	{
909	Assert(pCpu->mode == CPUMODE_64BIT);
910
911	Log(("pgmPoolMonitorIsReused: OP_STOSQ\n"));
912	return true;
913	}
914	return false;
915	}
916	if ( (pCpu->param1.flags & USE_REG_GEN32)
917	&& (pCpu->param1.base.reg_gen == USE_REG_ESP))
918	{
919	Log4(("pgmPoolMonitorIsReused: ESP\n"));
920	return true;
921	}
922
923	//if (pPage->fCR3Mix)
924	// return false;
925	return false;
926	}
927
928
929	/**
930	* Flushes the page being accessed.
931	*
932	* @returns VBox status code suitable for scheduling.
933	* @param pVM The VM handle.
934	* @param pPool The pool.
935	* @param pPage The pool page (head).
936	* @param pCpu The disassembly of the write instruction.
937	* @param pRegFrame The trap register frame.
938	* @param GCPhysFault The fault address as guest physical address.
939	* @param pvFault The fault address.
940	*/
941	static int pgmPoolAccessHandlerFlush(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
942	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
943	{
944	/*
945	* First, do the flushing.
946	*/
947	int rc = pgmPoolMonitorChainFlush(pPool, pPage);
948
949	/*
950	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
951	*/
952	uint32_t cbWritten;
953	int rc2 = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cbWritten);
954	if (RT_SUCCESS(rc2))
955	pRegFrame->rip += pCpu->opsize;
956	else if (rc2 == VERR_EM_INTERPRETER)
957	{
958	#ifdef IN_RC
959	if (PATMIsPatchGCAddr(pVM, (RTRCPTR)pRegFrame->eip))
960	{
961	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
962	pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
963	rc = VINF_SUCCESS;
964	STAM_COUNTER_INC(&pPool->StatMonitorRZIntrFailPatch2);
965	}
966	else
967	#endif
968	{
969	rc = VINF_EM_RAW_EMULATE_INSTR;
970	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
971	}
972	}
973	else
974	rc = rc2;
975
976	/* See use in pgmPoolAccessHandlerSimple(). */
977	PGM_INVL_GUEST_TLBS();
978
979	LogFlow(("pgmPoolAccessHandlerPT: returns %Rrc (flushed)\n", rc));
980	return rc;
981
982	}
983
984
985	/**
986	* Handles the STOSD write accesses.
987	*
988	* @returns VBox status code suitable for scheduling.
989	* @param pVM The VM handle.
990	* @param pPool The pool.
991	* @param pPage The pool page (head).
992	* @param pCpu The disassembly of the write instruction.
993	* @param pRegFrame The trap register frame.
994	* @param GCPhysFault The fault address as guest physical address.
995	* @param pvFault The fault address.
996	*/
997	DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
998	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
999	{
1000	Assert(pCpu->mode == CPUMODE_32BIT);
1001
1002	/*
1003	* Increment the modification counter and insert it into the list
1004	* of modified pages the first time.
1005	*/
1006	if (!pPage->cModifications++)
1007	pgmPoolMonitorModifiedInsert(pPool, pPage);
1008
1009	/*
1010	* Execute REP STOSD.
1011	*
1012	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
1013	* write situation, meaning that it's safe to write here.
1014	*/
1015	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1016	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
1017	#endif
1018	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
1019	while (pRegFrame->ecx)
1020	{
1021	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1022	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1023	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1024	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1025	#else
1026	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1027	#endif
1028	#ifdef IN_RC
1029	(uint32_t )pu32 = pRegFrame->eax;
1030	#else
1031	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->eax, 4);
1032	#endif
1033	pu32 += 4;
1034	GCPhysFault += 4;
1035	pRegFrame->edi += 4;
1036	pRegFrame->ecx--;
1037	}
1038	pRegFrame->rip += pCpu->opsize;
1039
1040	/* See use in pgmPoolAccessHandlerSimple(). */
1041	PGM_INVL_GUEST_TLBS();
1042
1043	LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
1044	return VINF_SUCCESS;
1045	}
1046
1047
1048	/**
1049	* Handles the simple write accesses.
1050	*
1051	* @returns VBox status code suitable for scheduling.
1052	* @param pVM The VM handle.
1053	* @param pPool The pool.
1054	* @param pPage The pool page (head).
1055	* @param pCpu The disassembly of the write instruction.
1056	* @param pRegFrame The trap register frame.
1057	* @param GCPhysFault The fault address as guest physical address.
1058	* @param pvFault The fault address.
1059	*/
1060	DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
1061	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
1062	{
1063	/*
1064	* Increment the modification counter and insert it into the list
1065	* of modified pages the first time.
1066	*/
1067	if (!pPage->cModifications++)
1068	pgmPoolMonitorModifiedInsert(pPool, pPage);
1069
1070	/*
1071	* Clear all the pages. ASSUMES that pvFault is readable.
1072	*/
1073	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1074	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
1075	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1076	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pvFault, pCpu);
1077	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1078	#else
1079	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pvFault, pCpu);
1080	#endif
1081
1082	/*
1083	* Interpret the instruction.
1084	*/
1085	uint32_t cb;
1086	int rc = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cb);
1087	if (RT_SUCCESS(rc))
1088	pRegFrame->rip += pCpu->opsize;
1089	else if (rc == VERR_EM_INTERPRETER)
1090	{
1091	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for %04x:%RGv - opcode=%d\n",
1092	pRegFrame->cs, (RTGCPTR)pRegFrame->rip, pCpu->pCurInstr->opcode));
1093	rc = VINF_EM_RAW_EMULATE_INSTR;
1094	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
1095	}
1096
1097	/*
1098	* Quick hack, with logging enabled we're getting stale
1099	* code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
1100	* Flushing here is BAD and expensive, I think EMInterpretDisasOne will
1101	* have to be fixed to support this. But that'll have to wait till next week.
1102	*
1103	* An alternative is to keep track of the changed PTEs together with the
1104	* GCPhys from the guest PT. This may proove expensive though.
1105	*
1106	* At the moment, it's VITAL that it's done AFTER the instruction interpreting
1107	* because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
1108	*/
1109	PGM_INVL_GUEST_TLBS();
1110
1111	LogFlow(("pgmPoolAccessHandlerSimple: returns %Rrc cb=%d\n", rc, cb));
1112	return rc;
1113	}
1114
1115
1116	/**
1117	* \#PF Handler callback for PT write accesses.
1118	*
1119	* @returns VBox status code (appropriate for GC return).
1120	* @param pVM VM Handle.
1121	* @param uErrorCode CPU Error code.
1122	* @param pRegFrame Trap register frame.
1123	* NULL on DMA and other non CPU access.
1124	* @param pvFault The fault address (cr2).
1125	* @param GCPhysFault The GC physical address corresponding to pvFault.
1126	* @param pvUser User argument.
1127	*/
1128	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1129	{
1130	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1131	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1132	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1133	LogFlow(("pgmPoolAccessHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1134
1135	/*
1136	* We should ALWAYS have the list head as user parameter. This
1137	* is because we use that page to record the changes.
1138	*/
1139	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1140
1141	/*
1142	* Disassemble the faulting instruction.
1143	*/
1144	DISCPUSTATE Cpu;
1145	int rc = EMInterpretDisasOne(pVM, pRegFrame, &Cpu, NULL);
1146	AssertRCReturn(rc, rc);
1147
1148	/*
1149	* Check if it's worth dealing with.
1150	*/
1151	bool fReused = false;
1152	if ( ( pPage->cModifications < 48 /** @todo #define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1153	\|\| pPage->fCR3Mix)
1154	&& !(fReused = pgmPoolMonitorIsReused(pVM, pPage, pRegFrame, &Cpu, pvFault))
1155	&& !pgmPoolMonitorIsForking(pPool, &Cpu, GCPhysFault & PAGE_OFFSET_MASK))
1156	{
1157	/*
1158	* Simple instructions, no REP prefix.
1159	*/
1160	if (!(Cpu.prefix & (PREFIX_REP \| PREFIX_REPNE)))
1161	{
1162	rc = pgmPoolAccessHandlerSimple(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1163	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1164	return rc;
1165	}
1166
1167	/*
1168	* Windows is frequently doing small memset() operations (netio test 4k+).
1169	* We have to deal with these or we'll kill the cache and performance.
1170	*/
1171	if ( Cpu.pCurInstr->opcode == OP_STOSWD
1172	&& CPUMGetGuestCPL(pVM, pRegFrame) == 0
1173	&& pRegFrame->ecx <= 0x20
1174	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1175	&& !((uintptr_t)pvFault & 3)
1176	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1177	&& Cpu.mode == CPUMODE_32BIT
1178	&& Cpu.opmode == CPUMODE_32BIT
1179	&& Cpu.addrmode == CPUMODE_32BIT
1180	&& Cpu.prefix == PREFIX_REP
1181	&& !pRegFrame->eflags.Bits.u1DF
1182	)
1183	{
1184	rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1185	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,RepStosd), a);
1186	return rc;
1187	}
1188
1189	/* REP prefix, don't bother. */
1190	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,RepPrefix));
1191	Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1192	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, Cpu.pCurInstr->opcode, Cpu.prefix));
1193	}
1194
1195	/*
1196	* Not worth it, so flush it.
1197	*
1198	* If we considered it to be reused, don't to back to ring-3
1199	* to emulate failed instructions since we usually cannot
1200	* interpret then. This may be a bit risky, in which case
1201	* the reuse detection must be fixed.
1202	*/
1203	rc = pgmPoolAccessHandlerFlush(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1204	if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
1205	rc = VINF_SUCCESS;
1206	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1207	return rc;
1208	}
1209
1210	# endif /* !IN_RING3 */
1211	#endif /* PGMPOOL_WITH_MONITORING */
1212
1213	#ifdef PGMPOOL_WITH_CACHE
1214
1215	/**
1216	* Inserts a page into the GCPhys hash table.
1217	*
1218	* @param pPool The pool.
1219	* @param pPage The page.
1220	*/
1221	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1222	{
1223	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1224	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1225	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1226	pPage->iNext = pPool->aiHash[iHash];
1227	pPool->aiHash[iHash] = pPage->idx;
1228	}
1229
1230
1231	/**
1232	* Removes a page from the GCPhys hash table.
1233	*
1234	* @param pPool The pool.
1235	* @param pPage The page.
1236	*/
1237	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1238	{
1239	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1240	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1241	if (pPool->aiHash[iHash] == pPage->idx)
1242	pPool->aiHash[iHash] = pPage->iNext;
1243	else
1244	{
1245	uint16_t iPrev = pPool->aiHash[iHash];
1246	for (;;)
1247	{
1248	const int16_t i = pPool->aPages[iPrev].iNext;
1249	if (i == pPage->idx)
1250	{
1251	pPool->aPages[iPrev].iNext = pPage->iNext;
1252	break;
1253	}
1254	if (i == NIL_PGMPOOL_IDX)
1255	{
1256	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%#x\n", pPage->GCPhys, pPage->idx));
1257	break;
1258	}
1259	iPrev = i;
1260	}
1261	}
1262	pPage->iNext = NIL_PGMPOOL_IDX;
1263	}
1264
1265
1266	/**
1267	* Frees up one cache page.
1268	*
1269	* @returns VBox status code.
1270	* @retval VINF_SUCCESS on success.
1271	* @retval VERR_PGM_POOL_CLEARED if the deregistration of a physical handler will cause a light weight pool flush.
1272	* @param pPool The pool.
1273	* @param iUser The user index.
1274	*/
1275	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1276	{
1277	#ifndef IN_RC
1278	const PVM pVM = pPool->CTX_SUFF(pVM);
1279	#endif
1280	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1281	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1282
1283	/*
1284	* Select one page from the tail of the age list.
1285	*/
1286	uint16_t iToFree = pPool->iAgeTail;
1287	if (iToFree == iUser)
1288	iToFree = pPool->aPages[iToFree].iAgePrev;
1289	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
1290	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
1291	{
1292	uint16_t i = pPool->aPages[iToFree].iAgePrev;
1293	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
1294	{
1295	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
1296	continue;
1297	iToFree = i;
1298	break;
1299	}
1300	}
1301	*/
1302
1303	Assert(iToFree != iUser);
1304	AssertRelease(iToFree != NIL_PGMPOOL_IDX);
1305
1306	PPGMPOOLPAGE pPage = &pPool->aPages[iToFree];
1307
1308	/*
1309	* Reject any attempts at flushing the currently active shadow CR3 mapping
1310	*/
1311	if (PGMGetHyperCR3(pPool->CTX_SUFF(pVM)) == pPage->Core.Key)
1312	{
1313	/* Refresh the cr3 mapping by putting it at the head of the age list. */
1314	pgmPoolCacheUsed(pPool, pPage);
1315	return pgmPoolCacheFreeOne(pPool, iUser);
1316	}
1317
1318	int rc = pgmPoolFlushPage(pPool, pPage);
1319	if (rc == VINF_SUCCESS)
1320	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
1321	return rc;
1322	}
1323
1324
1325	/**
1326	* Checks if a kind mismatch is really a page being reused
1327	* or if it's just normal remappings.
1328	*
1329	* @returns true if reused and the cached page (enmKind1) should be flushed
1330	* @returns false if not reused.
1331	* @param enmKind1 The kind of the cached page.
1332	* @param enmKind2 The kind of the requested page.
1333	*/
1334	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
1335	{
1336	switch (enmKind1)
1337	{
1338	/*
1339	* Never reuse them. There is no remapping in non-paging mode.
1340	*/
1341	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1342	case PGMPOOLKIND_32BIT_PD_PHYS:
1343	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1344	case PGMPOOLKIND_PAE_PD_PHYS:
1345	case PGMPOOLKIND_PAE_PDPT_PHYS:
1346	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1347	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1348	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1349	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1350	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1351	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1352	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
1353	return false;
1354	#else
1355	return true;
1356	#endif
1357
1358	/*
1359	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1360	*/
1361	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1362	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1363	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1364	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1365	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1366	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1367	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1368	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1369	switch (enmKind2)
1370	{
1371	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1372	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1373	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1374	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1375	case PGMPOOLKIND_64BIT_PML4:
1376	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1377	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1378	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1379	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1380	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1381	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1382	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1383	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1384	return true;
1385	default:
1386	return false;
1387	}
1388
1389	/*
1390	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1391	*/
1392	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1393	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1394	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1395	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1396	case PGMPOOLKIND_64BIT_PML4:
1397	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1398	switch (enmKind2)
1399	{
1400	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1401	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1402	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1403	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1404	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1405	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1406	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1407	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1408	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1409	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1410	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1411	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1412	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1413	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1414	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1415	return true;
1416	default:
1417	return false;
1418	}
1419
1420	/*
1421	* These cannot be flushed, and it's common to reuse the PDs as PTs.
1422	*/
1423	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
1424	case PGMPOOLKIND_ROOT_32BIT_PD:
1425	case PGMPOOLKIND_ROOT_PAE_PD:
1426	case PGMPOOLKIND_ROOT_PDPT:
1427	#endif
1428	case PGMPOOLKIND_ROOT_NESTED:
1429	return false;
1430
1431	default:
1432	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1433	}
1434	}
1435
1436
1437	/**
1438	* Attempts to satisfy a pgmPoolAlloc request from the cache.
1439	*
1440	* @returns VBox status code.
1441	* @retval VINF_PGM_CACHED_PAGE on success.
1442	* @retval VERR_FILE_NOT_FOUND if not found.
1443	* @param pPool The pool.
1444	* @param GCPhys The GC physical address of the page we're gonna shadow.
1445	* @param enmKind The kind of mapping.
1446	* @param iUser The shadow page pool index of the user table.
1447	* @param iUserTable The index into the user table (shadowed).
1448	* @param ppPage Where to store the pointer to the page.
1449	*/
1450	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
1451	{
1452	#ifndef IN_RC
1453	const PVM pVM = pPool->CTX_SUFF(pVM);
1454	#endif
1455	/*
1456	* Look up the GCPhys in the hash.
1457	*/
1458	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1459	Log3(("pgmPoolCacheAlloc: %RGp kind %d iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, enmKind, iUser, iUserTable, i));
1460	if (i != NIL_PGMPOOL_IDX)
1461	{
1462	do
1463	{
1464	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1465	Log3(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
1466	if (pPage->GCPhys == GCPhys)
1467	{
1468	if ((PGMPOOLKIND)pPage->enmKind == enmKind)
1469	{
1470	int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1471	if (RT_SUCCESS(rc))
1472	{
1473	*ppPage = pPage;
1474	STAM_COUNTER_INC(&pPool->StatCacheHits);
1475	return VINF_PGM_CACHED_PAGE;
1476	}
1477	return rc;
1478	}
1479
1480	/*
1481	* The kind is different. In some cases we should now flush the page
1482	* as it has been reused, but in most cases this is normal remapping
1483	* of PDs as PT or big pages using the GCPhys field in a slightly
1484	* different way than the other kinds.
1485	*/
1486	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1487	{
1488	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1489	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED will be returned by pgmPoolTracInsert. */
1490	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
1491	break;
1492	}
1493	}
1494
1495	/* next */
1496	i = pPage->iNext;
1497	} while (i != NIL_PGMPOOL_IDX);
1498	}
1499
1500	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%d\n", GCPhys, enmKind));
1501	STAM_COUNTER_INC(&pPool->StatCacheMisses);
1502	return VERR_FILE_NOT_FOUND;
1503	}
1504
1505
1506	/**
1507	* Inserts a page into the cache.
1508	*
1509	* @param pPool The pool.
1510	* @param pPage The cached page.
1511	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1512	*/
1513	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1514	{
1515	/*
1516	* Insert into the GCPhys hash if the page is fit for that.
1517	*/
1518	Assert(!pPage->fCached);
1519	if (fCanBeCached)
1520	{
1521	pPage->fCached = true;
1522	pgmPoolHashInsert(pPool, pPage);
1523	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1524	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1525	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1526	}
1527	else
1528	{
1529	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1530	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1531	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1532	}
1533
1534	/*
1535	* Insert at the head of the age list.
1536	*/
1537	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1538	pPage->iAgeNext = pPool->iAgeHead;
1539	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1540	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1541	else
1542	pPool->iAgeTail = pPage->idx;
1543	pPool->iAgeHead = pPage->idx;
1544	}
1545
1546
1547	/**
1548	* Flushes a cached page.
1549	*
1550	* @param pPool The pool.
1551	* @param pPage The cached page.
1552	*/
1553	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1554	{
1555	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
1556
1557	/*
1558	* Remove the page from the hash.
1559	*/
1560	if (pPage->fCached)
1561	{
1562	pPage->fCached = false;
1563	pgmPoolHashRemove(pPool, pPage);
1564	}
1565	else
1566	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1567
1568	/*
1569	* Remove it from the age list.
1570	*/
1571	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1572	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1573	else
1574	pPool->iAgeTail = pPage->iAgePrev;
1575	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1576	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1577	else
1578	pPool->iAgeHead = pPage->iAgeNext;
1579	pPage->iAgeNext = NIL_PGMPOOL_IDX;
1580	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1581	}
1582
1583	#endif /* PGMPOOL_WITH_CACHE */
1584	#ifdef PGMPOOL_WITH_MONITORING
1585
1586	/**
1587	* Looks for pages sharing the monitor.
1588	*
1589	* @returns Pointer to the head page.
1590	* @returns NULL if not found.
1591	* @param pPool The Pool
1592	* @param pNewPage The page which is going to be monitored.
1593	*/
1594	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1595	{
1596	#ifdef PGMPOOL_WITH_CACHE
1597	/*
1598	* Look up the GCPhys in the hash.
1599	*/
1600	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1601	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1602	if (i == NIL_PGMPOOL_IDX)
1603	return NULL;
1604	do
1605	{
1606	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1607	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1608	&& pPage != pNewPage)
1609	{
1610	switch (pPage->enmKind)
1611	{
1612	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1613	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1614	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1615	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1616	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1617	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1618	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1619	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1620	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1621	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1622	case PGMPOOLKIND_64BIT_PML4:
1623	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1624	case PGMPOOLKIND_32BIT_PD:
1625	case PGMPOOLKIND_PAE_PDPT:
1626	#else
1627	case PGMPOOLKIND_ROOT_32BIT_PD:
1628	case PGMPOOLKIND_ROOT_PAE_PD:
1629	case PGMPOOLKIND_ROOT_PDPT:
1630	#endif
1631	{
1632	/* find the head */
1633	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1634	{
1635	Assert(pPage->iMonitoredPrev != pPage->idx);
1636	pPage = &pPool->aPages[pPage->iMonitoredPrev];
1637	}
1638	return pPage;
1639	}
1640
1641	/* ignore, no monitoring. */
1642	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1643	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1644	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1645	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1646	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1647	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1648	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1649	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1650	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1651	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1652	case PGMPOOLKIND_ROOT_NESTED:
1653	case PGMPOOLKIND_PAE_PD_PHYS:
1654	case PGMPOOLKIND_PAE_PDPT_PHYS:
1655	case PGMPOOLKIND_32BIT_PD_PHYS:
1656	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1657	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
1658	#endif
1659	break;
1660	default:
1661	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
1662	}
1663	}
1664
1665	/* next */
1666	i = pPage->iNext;
1667	} while (i != NIL_PGMPOOL_IDX);
1668	#endif
1669	return NULL;
1670	}
1671
1672
1673	/**
1674	* Enabled write monitoring of a guest page.
1675	*
1676	* @returns VBox status code.
1677	* @retval VINF_SUCCESS on success.
1678	* @retval VERR_PGM_POOL_CLEARED if the registration of the physical handler will cause a light weight pool flush.
1679	* @param pPool The pool.
1680	* @param pPage The cached page.
1681	*/
1682	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1683	{
1684	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
1685
1686	/*
1687	* Filter out the relevant kinds.
1688	*/
1689	switch (pPage->enmKind)
1690	{
1691	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1692	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1693	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1694	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1695	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1696	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1697	case PGMPOOLKIND_64BIT_PML4:
1698	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1699	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1700	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1701	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1702	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1703	case PGMPOOLKIND_32BIT_PD:
1704	case PGMPOOLKIND_PAE_PDPT:
1705	#else
1706	case PGMPOOLKIND_ROOT_PDPT:
1707	#endif
1708	break;
1709
1710	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1711	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1712	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1713	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1714	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1715	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1716	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1717	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1718	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1719	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1720	case PGMPOOLKIND_ROOT_NESTED:
1721	/* Nothing to monitor here. */
1722	return VINF_SUCCESS;
1723
1724	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1725	case PGMPOOLKIND_32BIT_PD_PHYS:
1726	case PGMPOOLKIND_PAE_PDPT_PHYS:
1727	case PGMPOOLKIND_PAE_PD_PHYS:
1728	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
1729	/* Nothing to monitor here. */
1730	return VINF_SUCCESS;
1731	#else
1732	case PGMPOOLKIND_ROOT_32BIT_PD:
1733	case PGMPOOLKIND_ROOT_PAE_PD:
1734	#endif
1735	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1736	break;
1737	#else
1738	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1739	#endif
1740	default:
1741	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1742	}
1743
1744	/*
1745	* Install handler.
1746	*/
1747	int rc;
1748	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
1749	if (pPageHead)
1750	{
1751	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
1752	Assert(pPageHead->iMonitoredPrev != pPage->idx);
1753	pPage->iMonitoredPrev = pPageHead->idx;
1754	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
1755	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
1756	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
1757	pPageHead->iMonitoredNext = pPage->idx;
1758	rc = VINF_SUCCESS;
1759	}
1760	else
1761	{
1762	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1763	PVM pVM = pPool->CTX_SUFF(pVM);
1764	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1765	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
1766	GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
1767	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
1768	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
1769	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
1770	pPool->pszAccessHandler);
1771	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
1772	* the heap size should suffice. */
1773	AssertFatalRC(rc);
1774	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1775	rc = VERR_PGM_POOL_CLEARED;
1776	}
1777	pPage->fMonitored = true;
1778	return rc;
1779	}
1780
1781
1782	/**
1783	* Disables write monitoring of a guest page.
1784	*
1785	* @returns VBox status code.
1786	* @retval VINF_SUCCESS on success.
1787	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1788	* @param pPool The pool.
1789	* @param pPage The cached page.
1790	*/
1791	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1792	{
1793	/*
1794	* Filter out the relevant kinds.
1795	*/
1796	switch (pPage->enmKind)
1797	{
1798	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1799	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1800	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1801	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1802	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1803	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1804	case PGMPOOLKIND_64BIT_PML4:
1805	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1806	case PGMPOOLKIND_32BIT_PD:
1807	case PGMPOOLKIND_PAE_PDPT:
1808	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1809	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1810	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1811	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1812	#else
1813	case PGMPOOLKIND_ROOT_PDPT:
1814	#endif
1815	break;
1816
1817	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1818	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1819	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1820	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1821	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1822	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1823	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1824	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1825	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1826	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1827	case PGMPOOLKIND_ROOT_NESTED:
1828	case PGMPOOLKIND_PAE_PD_PHYS:
1829	case PGMPOOLKIND_PAE_PDPT_PHYS:
1830	case PGMPOOLKIND_32BIT_PD_PHYS:
1831	/* Nothing to monitor here. */
1832	return VINF_SUCCESS;
1833
1834	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
1835	case PGMPOOLKIND_ROOT_32BIT_PD:
1836	case PGMPOOLKIND_ROOT_PAE_PD:
1837	#endif
1838	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1839	break;
1840	#endif
1841	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
1842	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1843	#endif
1844	default:
1845	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1846	}
1847
1848	/*
1849	* Remove the page from the monitored list or uninstall it if last.
1850	*/
1851	const PVM pVM = pPool->CTX_SUFF(pVM);
1852	int rc;
1853	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1854	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1855	{
1856	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1857	{
1858	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
1859	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
1860	pNewHead->fCR3Mix = pPage->fCR3Mix;
1861	rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
1862	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
1863	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
1864	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pNewHead),
1865	pPool->pszAccessHandler);
1866	AssertFatalRCSuccess(rc);
1867	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1868	}
1869	else
1870	{
1871	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
1872	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
1873	{
1874	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
1875	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1876	}
1877	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
1878	rc = VINF_SUCCESS;
1879	}
1880	}
1881	else
1882	{
1883	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
1884	AssertFatalRC(rc);
1885	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1886	rc = VERR_PGM_POOL_CLEARED;
1887	}
1888	pPage->fMonitored = false;
1889
1890	/*
1891	* Remove it from the list of modified pages (if in it).
1892	*/
1893	pgmPoolMonitorModifiedRemove(pPool, pPage);
1894
1895	return rc;
1896	}
1897
1898	# ifdef PGMPOOL_WITH_MIXED_PT_CR3
1899
1900	/**
1901	* Set or clear the fCR3Mix attribute in a chain of monitored pages.
1902	*
1903	* @param pPool The Pool.
1904	* @param pPage A page in the chain.
1905	* @param fCR3Mix The new fCR3Mix value.
1906	*/
1907	static void pgmPoolMonitorChainChangeCR3Mix(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCR3Mix)
1908	{
1909	/* current */
1910	pPage->fCR3Mix = fCR3Mix;
1911
1912	/* before */
1913	int16_t idx = pPage->iMonitoredPrev;
1914	while (idx != NIL_PGMPOOL_IDX)
1915	{
1916	pPool->aPages[idx].fCR3Mix = fCR3Mix;
1917	idx = pPool->aPages[idx].iMonitoredPrev;
1918	}
1919
1920	/* after */
1921	idx = pPage->iMonitoredNext;
1922	while (idx != NIL_PGMPOOL_IDX)
1923	{
1924	pPool->aPages[idx].fCR3Mix = fCR3Mix;
1925	idx = pPool->aPages[idx].iMonitoredNext;
1926	}
1927	}
1928
1929
1930	/**
1931	* Installs or modifies monitoring of a CR3 page (special).
1932	*
1933	* We're pretending the CR3 page is shadowed by the pool so we can use the
1934	* generic mechanisms in detecting chained monitoring. (This also gives us a
1935	* tast of what code changes are required to really pool CR3 shadow pages.)
1936	*
1937	* @returns VBox status code.
1938	* @param pPool The pool.
1939	* @param idxRoot The CR3 (root) page index.
1940	* @param GCPhysCR3 The (new) CR3 value.
1941	*/
1942	int pgmPoolMonitorMonitorCR3(PPGMPOOL pPool, uint16_t idxRoot, RTGCPHYS GCPhysCR3)
1943	{
1944	Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
1945	PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
1946	LogFlow(("pgmPoolMonitorMonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%RGp, .fMonitored=%d} GCPhysCR3=%RGp\n",
1947	idxRoot, pPage, pPage->GCPhys, pPage->fMonitored, GCPhysCR3));
1948
1949	/*
1950	* The unlikely case where it already matches.
1951	*/
1952	if (pPage->GCPhys == GCPhysCR3)
1953	{
1954	Assert(pPage->fMonitored);
1955	return VINF_SUCCESS;
1956	}
1957
1958	/*
1959	* Flush the current monitoring and remove it from the hash.
1960	*/
1961	int rc = VINF_SUCCESS;
1962	if (pPage->fMonitored)
1963	{
1964	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
1965	rc = pgmPoolMonitorFlush(pPool, pPage);
1966	if (rc == VERR_PGM_POOL_CLEARED)
1967	rc = VINF_SUCCESS;
1968	else
1969	AssertFatalRC(rc);
1970	pgmPoolHashRemove(pPool, pPage);
1971	}
1972
1973	/*
1974	* Monitor the page at the new location and insert it into the hash.
1975	*/
1976	pPage->GCPhys = GCPhysCR3;
1977	int rc2 = pgmPoolMonitorInsert(pPool, pPage);
1978	if (rc2 != VERR_PGM_POOL_CLEARED)
1979	{
1980	AssertFatalRC(rc2);
1981	if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS)
1982	rc = rc2;
1983	}
1984	pgmPoolHashInsert(pPool, pPage);
1985	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, true);
1986	return rc;
1987	}
1988
1989
1990	/**
1991	* Removes the monitoring of a CR3 page (special).
1992	*
1993	* @returns VBox status code.
1994	* @param pPool The pool.
1995	* @param idxRoot The CR3 (root) page index.
1996	*/
1997	int pgmPoolMonitorUnmonitorCR3(PPGMPOOL pPool, uint16_t idxRoot)
1998	{
1999	Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
2000	PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
2001	LogFlow(("pgmPoolMonitorUnmonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%RGp, .fMonitored=%d}\n",
2002	idxRoot, pPage, pPage->GCPhys, pPage->fMonitored));
2003
2004	if (!pPage->fMonitored)
2005	return VINF_SUCCESS;
2006
2007	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
2008	int rc = pgmPoolMonitorFlush(pPool, pPage);
2009	if (rc != VERR_PGM_POOL_CLEARED)
2010	AssertFatalRC(rc);
2011	else
2012	rc = VINF_SUCCESS;
2013	pgmPoolHashRemove(pPool, pPage);
2014	Assert(!pPage->fMonitored);
2015	pPage->GCPhys = NIL_RTGCPHYS;
2016	return rc;
2017	}
2018
2019	# endif /* PGMPOOL_WITH_MIXED_PT_CR3 */
2020
2021	/**
2022	* Inserts the page into the list of modified pages.
2023	*
2024	* @param pPool The pool.
2025	* @param pPage The page.
2026	*/
2027	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2028	{
2029	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2030	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2031	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2032	&& pPool->iModifiedHead != pPage->idx,
2033	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2034	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2035	pPool->iModifiedHead, pPool->cModifiedPages));
2036
2037	pPage->iModifiedNext = pPool->iModifiedHead;
2038	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2039	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2040	pPool->iModifiedHead = pPage->idx;
2041	pPool->cModifiedPages++;
2042	#ifdef VBOX_WITH_STATISTICS
2043	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2044	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2045	#endif
2046	}
2047
2048
2049	/**
2050	* Removes the page from the list of modified pages and resets the
2051	* moficiation counter.
2052	*
2053	* @param pPool The pool.
2054	* @param pPage The page which is believed to be in the list of modified pages.
2055	*/
2056	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2057	{
2058	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2059	if (pPool->iModifiedHead == pPage->idx)
2060	{
2061	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2062	pPool->iModifiedHead = pPage->iModifiedNext;
2063	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2064	{
2065	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2066	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2067	}
2068	pPool->cModifiedPages--;
2069	}
2070	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2071	{
2072	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2073	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2074	{
2075	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2076	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2077	}
2078	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2079	pPool->cModifiedPages--;
2080	}
2081	else
2082	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2083	pPage->cModifications = 0;
2084	}
2085
2086
2087	/**
2088	* Zaps the list of modified pages, resetting their modification counters in the process.
2089	*
2090	* @param pVM The VM handle.
2091	*/
2092	void pgmPoolMonitorModifiedClearAll(PVM pVM)
2093	{
2094	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2095	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2096
2097	unsigned cPages = 0; NOREF(cPages);
2098	uint16_t idx = pPool->iModifiedHead;
2099	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2100	while (idx != NIL_PGMPOOL_IDX)
2101	{
2102	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2103	idx = pPage->iModifiedNext;
2104	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2105	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2106	pPage->cModifications = 0;
2107	Assert(++cPages);
2108	}
2109	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2110	pPool->cModifiedPages = 0;
2111	}
2112
2113
2114	#ifdef IN_RING3
2115	/**
2116	* Clear all shadow pages and clear all modification counters.
2117	*
2118	* @param pVM The VM handle.
2119	* @remark Should only be used when monitoring is available, thus placed in
2120	* the PGMPOOL_WITH_MONITORING #ifdef.
2121	*/
2122	void pgmPoolClearAll(PVM pVM)
2123	{
2124	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2125	STAM_PROFILE_START(&pPool->StatClearAll, c);
2126	LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
2127
2128	/*
2129	* Iterate all the pages until we've encountered all that in use.
2130	* This is simple but not quite optimal solution.
2131	*/
2132	unsigned cModifiedPages = 0; NOREF(cModifiedPages);
2133	unsigned cLeft = pPool->cUsedPages;
2134	unsigned iPage = pPool->cCurPages;
2135	while (--iPage >= PGMPOOL_IDX_FIRST)
2136	{
2137	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2138	if (pPage->GCPhys != NIL_RTGCPHYS)
2139	{
2140	switch (pPage->enmKind)
2141	{
2142	/*
2143	* We only care about shadow page tables.
2144	*/
2145	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2146	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2147	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2148	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2149	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2150	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2151	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2152	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2153	{
2154	#ifdef PGMPOOL_WITH_USER_TRACKING
2155	if (pPage->cPresent)
2156	#endif
2157	{
2158	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
2159	STAM_PROFILE_START(&pPool->StatZeroPage, z);
2160	ASMMemZeroPage(pvShw);
2161	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
2162	#ifdef PGMPOOL_WITH_USER_TRACKING
2163	pPage->cPresent = 0;
2164	pPage->iFirstPresent = ~0;
2165	#endif
2166	}
2167	}
2168	/* fall thru */
2169
2170	default:
2171	Assert(!pPage->cModifications \|\| ++cModifiedPages);
2172	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2173	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2174	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2175	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2176	pPage->cModifications = 0;
2177	break;
2178
2179	}
2180	if (!--cLeft)
2181	break;
2182	}
2183	}
2184
2185	/* swipe the special pages too. */
2186	for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
2187	{
2188	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2189	if (pPage->GCPhys != NIL_RTGCPHYS)
2190	{
2191	Assert(!pPage->cModifications \|\| ++cModifiedPages);
2192	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2193	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2194	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2195	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2196	pPage->cModifications = 0;
2197	}
2198	}
2199
2200	#ifndef DEBUG_michael
2201	AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
2202	#endif
2203	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2204	pPool->cModifiedPages = 0;
2205
2206	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2207	/*
2208	* Clear all the GCPhys links and rebuild the phys ext free list.
2209	*/
2210	for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
2211	pRam;
2212	pRam = pRam->CTX_SUFF(pNext))
2213	{
2214	unsigned iPage = pRam->cb >> PAGE_SHIFT;
2215	while (iPage-- > 0)
2216	pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2217	}
2218
2219	pPool->iPhysExtFreeHead = 0;
2220	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
2221	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
2222	for (unsigned i = 0; i < cMaxPhysExts; i++)
2223	{
2224	paPhysExts[i].iNext = i + 1;
2225	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
2226	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
2227	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
2228	}
2229	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2230	#endif
2231
2232
2233	pPool->cPresent = 0;
2234	STAM_PROFILE_STOP(&pPool->StatClearAll, c);
2235	}
2236	#endif /* IN_RING3 */
2237
2238
2239	/**
2240	* Handle SyncCR3 pool tasks
2241	*
2242	* @returns VBox status code.
2243	* @retval VINF_SUCCESS if successfully added.
2244	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2245	* @param pVM The VM handle.
2246	* @remark Should only be used when monitoring is available, thus placed in
2247	* the PGMPOOL_WITH_MONITORING #ifdef.
2248	*/
2249	int pgmPoolSyncCR3(PVM pVM)
2250	{
2251	/*
2252	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2253	* Occasionally we will have to clear all the shadow page tables because we wanted
2254	* to monitor a page which was mapped by too many shadowed page tables. This operation
2255	* sometimes refered to as a 'lightweight flush'.
2256	*/
2257	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2258	pgmPoolMonitorModifiedClearAll(pVM);
2259	else
2260	{
2261	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2262	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2263	pgmPoolClearAll(pVM);
2264	# else /* !IN_RING3 */
2265	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2266	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2267	return VINF_PGM_SYNC_CR3;
2268	# endif /* !IN_RING3 */
2269	}
2270	return VINF_SUCCESS;
2271	}
2272
2273	#endif /* PGMPOOL_WITH_MONITORING */
2274	#ifdef PGMPOOL_WITH_USER_TRACKING
2275
2276	/**
2277	* Frees up at least one user entry.
2278	*
2279	* @returns VBox status code.
2280	* @retval VINF_SUCCESS if successfully added.
2281	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2282	* @param pPool The pool.
2283	* @param iUser The user index.
2284	*/
2285	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2286	{
2287	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2288	#ifdef PGMPOOL_WITH_CACHE
2289	/*
2290	* Just free cached pages in a braindead fashion.
2291	*/
2292	/** @todo walk the age list backwards and free the first with usage. */
2293	int rc = VINF_SUCCESS;
2294	do
2295	{
2296	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2297	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2298	rc = rc2;
2299	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2300	return rc;
2301	#else
2302	/*
2303	* Lazy approach.
2304	*/
2305	/* @todo incompatible with long mode paging (cr3 root will be flushed) */
2306	Assert(!CPUMIsGuestInLongMode(pVM));
2307	pgmPoolFlushAllInt(pPool);
2308	return VERR_PGM_POOL_FLUSHED;
2309	#endif
2310	}
2311
2312
2313	/**
2314	* Inserts a page into the cache.
2315	*
2316	* This will create user node for the page, insert it into the GCPhys
2317	* hash, and insert it into the age list.
2318	*
2319	* @returns VBox status code.
2320	* @retval VINF_SUCCESS if successfully added.
2321	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2322	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
2323	* @param pPool The pool.
2324	* @param pPage The cached page.
2325	* @param GCPhys The GC physical address of the page we're gonna shadow.
2326	* @param iUser The user index.
2327	* @param iUserTable The user table index.
2328	*/
2329	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2330	{
2331	int rc = VINF_SUCCESS;
2332	PPGMPOOLUSER pUser = pPool->CTX_SUFF(paUsers);
2333
2334	LogFlow(("pgmPoolTrackInsert iUser %d iUserTable %d\n", iUser, iUserTable));
2335
2336	/*
2337	* Find free a user node.
2338	*/
2339	uint16_t i = pPool->iUserFreeHead;
2340	if (i == NIL_PGMPOOL_USER_INDEX)
2341	{
2342	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2343	if (RT_FAILURE(rc))
2344	return rc;
2345	i = pPool->iUserFreeHead;
2346	}
2347
2348	/*
2349	* Unlink the user node from the free list,
2350	* initialize and insert it into the user list.
2351	*/
2352	pPool->iUserFreeHead = pUser[i].iNext;
2353	pUser[i].iNext = NIL_PGMPOOL_USER_INDEX;
2354	pUser[i].iUser = iUser;
2355	pUser[i].iUserTable = iUserTable;
2356	pPage->iUserHead = i;
2357
2358	/*
2359	* Insert into cache and enable monitoring of the guest page if enabled.
2360	*
2361	* Until we implement caching of all levels, including the CR3 one, we'll
2362	* have to make sure we don't try monitor & cache any recursive reuse of
2363	* a monitored CR3 page. Because all windows versions are doing this we'll
2364	* have to be able to do combined access monitoring, CR3 + PT and
2365	* PD + PT (guest PAE).
2366	*
2367	* Update:
2368	* We're now cooperating with the CR3 monitor if an uncachable page is found.
2369	*/
2370	#if defined(PGMPOOL_WITH_MONITORING) \|\| defined(PGMPOOL_WITH_CACHE)
2371	# ifdef PGMPOOL_WITH_MIXED_PT_CR3
2372	const bool fCanBeMonitored = true;
2373	# else
2374	bool fCanBeMonitored = pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
2375	\|\| (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
2376	\|\| pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
2377	# endif
2378	# ifdef PGMPOOL_WITH_CACHE
2379	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2380	# endif
2381	if (fCanBeMonitored)
2382	{
2383	# ifdef PGMPOOL_WITH_MONITORING
2384	rc = pgmPoolMonitorInsert(pPool, pPage);
2385	if (rc == VERR_PGM_POOL_CLEARED)
2386	{
2387	/* 'Failed' - free the usage, and keep it in the cache (if enabled). */
2388	# ifndef PGMPOOL_WITH_CACHE
2389	pgmPoolMonitorFlush(pPool, pPage);
2390	rc = VERR_PGM_POOL_FLUSHED;
2391	# endif
2392	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2393	pUser[i].iNext = pPool->iUserFreeHead;
2394	pUser[i].iUser = NIL_PGMPOOL_IDX;
2395	pPool->iUserFreeHead = i;
2396	}
2397	}
2398	# endif
2399	#endif /* PGMPOOL_WITH_MONITORING */
2400	return rc;
2401	}
2402
2403
2404	# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
2405	/**
2406	* Adds a user reference to a page.
2407	*
2408	* This will
2409	* This will move the page to the head of the
2410	*
2411	* @returns VBox status code.
2412	* @retval VINF_SUCCESS if successfully added.
2413	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2414	* @param pPool The pool.
2415	* @param pPage The cached page.
2416	* @param iUser The user index.
2417	* @param iUserTable The user table.
2418	*/
2419	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2420	{
2421	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2422
2423	LogFlow(("pgmPoolTrackAddUser iUser %d iUserTable %d\n", iUser, iUserTable));
2424	# ifdef VBOX_STRICT
2425	/*
2426	* Check that the entry doesn't already exists.
2427	*/
2428	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2429	{
2430	uint16_t i = pPage->iUserHead;
2431	do
2432	{
2433	Assert(i < pPool->cMaxUsers);
2434	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2435	i = paUsers[i].iNext;
2436	} while (i != NIL_PGMPOOL_USER_INDEX);
2437	}
2438	# endif
2439
2440	/*
2441	* Allocate a user node.
2442	*/
2443	uint16_t i = pPool->iUserFreeHead;
2444	if (i == NIL_PGMPOOL_USER_INDEX)
2445	{
2446	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2447	if (RT_FAILURE(rc))
2448	return rc;
2449	i = pPool->iUserFreeHead;
2450	}
2451	pPool->iUserFreeHead = paUsers[i].iNext;
2452
2453	/*
2454	* Initialize the user node and insert it.
2455	*/
2456	paUsers[i].iNext = pPage->iUserHead;
2457	paUsers[i].iUser = iUser;
2458	paUsers[i].iUserTable = iUserTable;
2459	pPage->iUserHead = i;
2460
2461	# ifdef PGMPOOL_WITH_CACHE
2462	/*
2463	* Tell the cache to update its replacement stats for this page.
2464	*/
2465	pgmPoolCacheUsed(pPool, pPage);
2466	# endif
2467	return VINF_SUCCESS;
2468	}
2469	# endif /* PGMPOOL_WITH_CACHE */
2470
2471
2472	/**
2473	* Frees a user record associated with a page.
2474	*
2475	* This does not clear the entry in the user table, it simply replaces the
2476	* user record to the chain of free records.
2477	*
2478	* @param pPool The pool.
2479	* @param HCPhys The HC physical address of the shadow page.
2480	* @param iUser The shadow page pool index of the user table.
2481	* @param iUserTable The index into the user table (shadowed).
2482	*/
2483	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2484	{
2485	/*
2486	* Unlink and free the specified user entry.
2487	*/
2488	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2489
2490	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2491	uint16_t i = pPage->iUserHead;
2492	if ( i != NIL_PGMPOOL_USER_INDEX
2493	&& paUsers[i].iUser == iUser
2494	&& paUsers[i].iUserTable == iUserTable)
2495	{
2496	pPage->iUserHead = paUsers[i].iNext;
2497
2498	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2499	paUsers[i].iNext = pPool->iUserFreeHead;
2500	pPool->iUserFreeHead = i;
2501	return;
2502	}
2503
2504	/* General: Linear search. */
2505	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2506	while (i != NIL_PGMPOOL_USER_INDEX)
2507	{
2508	if ( paUsers[i].iUser == iUser
2509	&& paUsers[i].iUserTable == iUserTable)
2510	{
2511	if (iPrev != NIL_PGMPOOL_USER_INDEX)
2512	paUsers[iPrev].iNext = paUsers[i].iNext;
2513	else
2514	pPage->iUserHead = paUsers[i].iNext;
2515
2516	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2517	paUsers[i].iNext = pPool->iUserFreeHead;
2518	pPool->iUserFreeHead = i;
2519	return;
2520	}
2521	iPrev = i;
2522	i = paUsers[i].iNext;
2523	}
2524
2525	/* Fatal: didn't find it */
2526	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%RGp\n",
2527	iUser, iUserTable, pPage->GCPhys));
2528	}
2529
2530
2531	/**
2532	* Gets the entry size of a shadow table.
2533	*
2534	* @param enmKind The kind of page.
2535	*
2536	* @returns The size of the entry in bytes. That is, 4 or 8.
2537	* @returns If the kind is not for a table, an assertion is raised and 0 is
2538	* returned.
2539	*/
2540	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2541	{
2542	switch (enmKind)
2543	{
2544	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2545	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2546	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2547	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
2548	case PGMPOOLKIND_32BIT_PD:
2549	case PGMPOOLKIND_32BIT_PD_PHYS:
2550	#else
2551	case PGMPOOLKIND_ROOT_32BIT_PD:
2552	#endif
2553	return 4;
2554
2555	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2556	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2557	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2558	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2559	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2560	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2561	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2562	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2563	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2564	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2565	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2566	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2567	case PGMPOOLKIND_64BIT_PML4:
2568	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
2569	case PGMPOOLKIND_ROOT_PAE_PD:
2570	case PGMPOOLKIND_ROOT_PDPT:
2571	#endif
2572	case PGMPOOLKIND_PAE_PDPT:
2573	case PGMPOOLKIND_ROOT_NESTED:
2574	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2575	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2576	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2577	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2578	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2579	case PGMPOOLKIND_PAE_PD_PHYS:
2580	case PGMPOOLKIND_PAE_PDPT_PHYS:
2581	return 8;
2582
2583	default:
2584	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2585	}
2586	}
2587
2588
2589	/**
2590	* Gets the entry size of a guest table.
2591	*
2592	* @param enmKind The kind of page.
2593	*
2594	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
2595	* @returns If the kind is not for a table, an assertion is raised and 0 is
2596	* returned.
2597	*/
2598	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2599	{
2600	switch (enmKind)
2601	{
2602	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2603	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2604	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
2605	case PGMPOOLKIND_32BIT_PD:
2606	#else
2607	case PGMPOOLKIND_ROOT_32BIT_PD:
2608	#endif
2609	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2610	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2611	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2612	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2613	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2614	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2615	return 4;
2616
2617	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2618	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2619	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2620	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2621	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2622	case PGMPOOLKIND_64BIT_PML4:
2623	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
2624	case PGMPOOLKIND_PAE_PDPT:
2625	#else
2626	case PGMPOOLKIND_ROOT_PAE_PD:
2627	case PGMPOOLKIND_ROOT_PDPT:
2628	#endif
2629	return 8;
2630
2631	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2632	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2633	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2634	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2635	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2636	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2637	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2638	case PGMPOOLKIND_ROOT_NESTED:
2639	case PGMPOOLKIND_PAE_PD_PHYS:
2640	case PGMPOOLKIND_PAE_PDPT_PHYS:
2641	case PGMPOOLKIND_32BIT_PD_PHYS:
2642	/** @todo can we return 0? (nobody is calling this...) */
2643	AssertFailed();
2644	return 0;
2645
2646	default:
2647	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2648	}
2649	}
2650
2651	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2652
2653	/**
2654	* Scans one shadow page table for mappings of a physical page.
2655	*
2656	* @param pVM The VM handle.
2657	* @param pPhysPage The guest page in question.
2658	* @param iShw The shadow page table.
2659	* @param cRefs The number of references made in that PT.
2660	*/
2661	static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2662	{
2663	LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2664	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2665
2666	/*
2667	* Assert sanity.
2668	*/
2669	Assert(cRefs == 1);
2670	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2671	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2672
2673	/*
2674	* Then, clear the actual mappings to the page in the shadow PT.
2675	*/
2676	switch (pPage->enmKind)
2677	{
2678	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2679	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2680	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2681	{
2682	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2683	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2684	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2685	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2686	{
2687	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2688	pPT->a[i].u = 0;
2689	cRefs--;
2690	if (!cRefs)
2691	return;
2692	}
2693	#ifdef LOG_ENABLED
2694	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2695	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2696	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2697	{
2698	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2699	pPT->a[i].u = 0;
2700	}
2701	#endif
2702	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2703	break;
2704	}
2705
2706	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2707	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2708	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2709	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2710	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2711	{
2712	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2713	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2714	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2715	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2716	{
2717	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2718	pPT->a[i].u = 0;
2719	cRefs--;
2720	if (!cRefs)
2721	return;
2722	}
2723	#ifdef LOG_ENABLED
2724	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2725	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2726	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2727	{
2728	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2729	pPT->a[i].u = 0;
2730	}
2731	#endif
2732	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d u64=%RX64\n", cRefs, pPage->iFirstPresent, pPage->cPresent, u64));
2733	break;
2734	}
2735
2736	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2737	{
2738	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2739	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2740	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2741	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
2742	{
2743	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2744	pPT->a[i].u = 0;
2745	cRefs--;
2746	if (!cRefs)
2747	return;
2748	}
2749	#ifdef LOG_ENABLED
2750	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2751	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2752	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
2753	{
2754	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2755	pPT->a[i].u = 0;
2756	}
2757	#endif
2758	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2759	break;
2760	}
2761
2762	default:
2763	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
2764	}
2765	}
2766
2767
2768	/**
2769	* Scans one shadow page table for mappings of a physical page.
2770	*
2771	* @param pVM The VM handle.
2772	* @param pPhysPage The guest page in question.
2773	* @param iShw The shadow page table.
2774	* @param cRefs The number of references made in that PT.
2775	*/
2776	void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2777	{
2778	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2779	LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2780	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
2781	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
2782	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2783	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
2784	}
2785
2786
2787	/**
2788	* Flushes a list of shadow page tables mapping the same physical page.
2789	*
2790	* @param pVM The VM handle.
2791	* @param pPhysPage The guest page in question.
2792	* @param iPhysExt The physical cross reference extent list to flush.
2793	*/
2794	void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
2795	{
2796	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2797	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
2798	LogFlow(("pgmPoolTrackFlushGCPhysPTs: HCPhys=%RHp iPhysExt\n", pPhysPage->HCPhys, iPhysExt));
2799
2800	const uint16_t iPhysExtStart = iPhysExt;
2801	PPGMPOOLPHYSEXT pPhysExt;
2802	do
2803	{
2804	Assert(iPhysExt < pPool->cMaxPhysExts);
2805	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
2806	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
2807	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
2808	{
2809	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
2810	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2811	}
2812
2813	/* next */
2814	iPhysExt = pPhysExt->iNext;
2815	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2816
2817	/* insert the list into the free list and clear the ram range entry. */
2818	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2819	pPool->iPhysExtFreeHead = iPhysExtStart;
2820	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2821
2822	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
2823	}
2824
2825	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2826
2827	/**
2828	* Scans all shadow page tables for mappings of a physical page.
2829	*
2830	* This may be slow, but it's most likely more efficient than cleaning
2831	* out the entire page pool / cache.
2832	*
2833	* @returns VBox status code.
2834	* @retval VINF_SUCCESS if all references has been successfully cleared.
2835	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
2836	* a page pool cleaning.
2837	*
2838	* @param pVM The VM handle.
2839	* @param pPhysPage The guest page in question.
2840	*/
2841	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
2842	{
2843	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2844	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2845	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d HCPhys=%RHp\n",
2846	pPool->cUsedPages, pPool->cPresent, pPhysPage->HCPhys));
2847
2848	#if 1
2849	/*
2850	* There is a limit to what makes sense.
2851	*/
2852	if (pPool->cPresent > 1024)
2853	{
2854	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
2855	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2856	return VINF_PGM_GCPHYS_ALIASED;
2857	}
2858	#endif
2859
2860	/*
2861	* Iterate all the pages until we've encountered all that in use.
2862	* This is simple but not quite optimal solution.
2863	*/
2864	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2865	const uint32_t u32 = u64;
2866	unsigned cLeft = pPool->cUsedPages;
2867	unsigned iPage = pPool->cCurPages;
2868	while (--iPage >= PGMPOOL_IDX_FIRST)
2869	{
2870	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2871	if (pPage->GCPhys != NIL_RTGCPHYS)
2872	{
2873	switch (pPage->enmKind)
2874	{
2875	/*
2876	* We only care about shadow page tables.
2877	*/
2878	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2879	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2880	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2881	{
2882	unsigned cPresent = pPage->cPresent;
2883	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2884	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2885	if (pPT->a[i].n.u1Present)
2886	{
2887	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2888	{
2889	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
2890	pPT->a[i].u = 0;
2891	}
2892	if (!--cPresent)
2893	break;
2894	}
2895	break;
2896	}
2897
2898	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2899	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2900	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2901	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2902	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2903	{
2904	unsigned cPresent = pPage->cPresent;
2905	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2906	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2907	if (pPT->a[i].n.u1Present)
2908	{
2909	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2910	{
2911	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
2912	pPT->a[i].u = 0;
2913	}
2914	if (!--cPresent)
2915	break;
2916	}
2917	break;
2918	}
2919	}
2920	if (!--cLeft)
2921	break;
2922	}
2923	}
2924
2925	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2926	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2927	return VINF_SUCCESS;
2928	}
2929
2930
2931	/**
2932	* Clears the user entry in a user table.
2933	*
2934	* This is used to remove all references to a page when flushing it.
2935	*/
2936	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
2937	{
2938	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
2939	Assert(pUser->iUser < pPool->cCurPages);
2940	uint32_t iUserTable = pUser->iUserTable;
2941
2942	/*
2943	* Map the user page.
2944	*/
2945	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
2946	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2947	if (pUserPage->enmKind == PGMPOOLKIND_ROOT_PAE_PD)
2948	{
2949	/* Must translate the fake 2048 entry PD to a 512 PD one since the R0 mapping is not linear. */
2950	Assert(pUser->iUser == PGMPOOL_IDX_PAE_PD);
2951	uint32_t iPdpt = iUserTable / X86_PG_PAE_ENTRIES;
2952	iUserTable %= X86_PG_PAE_ENTRIES;
2953	pUserPage = &pPool->aPages[PGMPOOL_IDX_PAE_PD_0 + iPdpt];
2954	Assert(pUserPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD);
2955	}
2956	#endif
2957	union
2958	{
2959	uint64_t *pau64;
2960	uint32_t *pau32;
2961	} u;
2962	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
2963
2964	/* Safety precaution in case we change the paging for other modes too in the future. */
2965	Assert(PGMGetHyperCR3(pPool->CTX_SUFF(pVM)) != pPage->Core.Key);
2966
2967	#ifdef VBOX_STRICT
2968	/*
2969	* Some sanity checks.
2970	*/
2971	switch (pUserPage->enmKind)
2972	{
2973	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
2974	case PGMPOOLKIND_32BIT_PD:
2975	case PGMPOOLKIND_32BIT_PD_PHYS:
2976	Assert(iUserTable < X86_PG_ENTRIES);
2977	break;
2978	# else
2979	case PGMPOOLKIND_ROOT_32BIT_PD:
2980	Assert(iUserTable < X86_PG_ENTRIES);
2981	Assert(!(u.pau32[iUserTable] & PGM_PDFLAGS_MAPPING));
2982	break;
2983	# endif
2984	# if !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) && !defined(VBOX_WITH_PGMPOOL_PAGING_ONLY)
2985	case PGMPOOLKIND_ROOT_PAE_PD:
2986	Assert(iUserTable < 2048 && pUser->iUser == PGMPOOL_IDX_PAE_PD);
2987	AssertMsg(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING), ("%llx %d\n", u.pau64[iUserTable], iUserTable));
2988	break;
2989	# endif
2990	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
2991	case PGMPOOLKIND_PAE_PDPT:
2992	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2993	case PGMPOOLKIND_PAE_PDPT_PHYS:
2994	# else
2995	case PGMPOOLKIND_ROOT_PDPT:
2996	# endif
2997	Assert(iUserTable < 4);
2998	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
2999	break;
3000	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3001	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3002	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3003	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3004	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3005	case PGMPOOLKIND_PAE_PD_PHYS:
3006	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3007	break;
3008	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3009	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3010	Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
3011	break;
3012	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3013	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3014	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3015	break;
3016	case PGMPOOLKIND_64BIT_PML4:
3017	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3018	/* GCPhys >> PAGE_SHIFT is the index here */
3019	break;
3020	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3021	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3022	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3023	break;
3024
3025	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3026	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3027	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3028	break;
3029
3030	case PGMPOOLKIND_ROOT_NESTED:
3031	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3032	break;
3033
3034	default:
3035	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3036	break;
3037	}
3038	#endif /* VBOX_STRICT */
3039
3040	/*
3041	* Clear the entry in the user page.
3042	*/
3043	switch (pUserPage->enmKind)
3044	{
3045	/* 32-bit entries */
3046	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
3047	case PGMPOOLKIND_32BIT_PD:
3048	case PGMPOOLKIND_32BIT_PD_PHYS:
3049	#else
3050	case PGMPOOLKIND_ROOT_32BIT_PD:
3051	#endif
3052	u.pau32[iUserTable] = 0;
3053	break;
3054
3055	/* 64-bit entries */
3056	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3057	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3058	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3059	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3060	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3061	case PGMPOOLKIND_PAE_PD_PHYS:
3062	case PGMPOOLKIND_PAE_PDPT_PHYS:
3063	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3064	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3065	case PGMPOOLKIND_64BIT_PML4:
3066	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3067	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3068	# if !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) && !defined(VBOX_WITH_PGMPOOL_PAGING_ONLY)
3069	case PGMPOOLKIND_ROOT_PAE_PD:
3070	#endif
3071	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
3072	case PGMPOOLKIND_PAE_PDPT:
3073	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3074	#else
3075	case PGMPOOLKIND_ROOT_PDPT:
3076	#endif
3077	case PGMPOOLKIND_ROOT_NESTED:
3078	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3079	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3080	u.pau64[iUserTable] = 0;
3081	break;
3082
3083	default:
3084	AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
3085	}
3086	}
3087
3088
3089	/**
3090	* Clears all users of a page.
3091	*/
3092	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3093	{
3094	/*
3095	* Free all the user records.
3096	*/
3097	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3098	uint16_t i = pPage->iUserHead;
3099	while (i != NIL_PGMPOOL_USER_INDEX)
3100	{
3101	/* Clear enter in user table. */
3102	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
3103
3104	/* Free it. */
3105	const uint16_t iNext = paUsers[i].iNext;
3106	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3107	paUsers[i].iNext = pPool->iUserFreeHead;
3108	pPool->iUserFreeHead = i;
3109
3110	/* Next. */
3111	i = iNext;
3112	}
3113	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3114	}
3115
3116	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3117
3118	/**
3119	* Allocates a new physical cross reference extent.
3120	*
3121	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
3122	* @param pVM The VM handle.
3123	* @param piPhysExt Where to store the phys ext index.
3124	*/
3125	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
3126	{
3127	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3128	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3129	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3130	{
3131	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3132	return NULL;
3133	}
3134	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3135	pPool->iPhysExtFreeHead = pPhysExt->iNext;
3136	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3137	*piPhysExt = iPhysExt;
3138	return pPhysExt;
3139	}
3140
3141
3142	/**
3143	* Frees a physical cross reference extent.
3144	*
3145	* @param pVM The VM handle.
3146	* @param iPhysExt The extent to free.
3147	*/
3148	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
3149	{
3150	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3151	Assert(iPhysExt < pPool->cMaxPhysExts);
3152	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3153	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3154	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3155	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3156	pPool->iPhysExtFreeHead = iPhysExt;
3157	}
3158
3159
3160	/**
3161	* Frees a physical cross reference extent.
3162	*
3163	* @param pVM The VM handle.
3164	* @param iPhysExt The extent to free.
3165	*/
3166	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
3167	{
3168	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3169
3170	const uint16_t iPhysExtStart = iPhysExt;
3171	PPGMPOOLPHYSEXT pPhysExt;
3172	do
3173	{
3174	Assert(iPhysExt < pPool->cMaxPhysExts);
3175	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3176	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3177	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3178
3179	/* next */
3180	iPhysExt = pPhysExt->iNext;
3181	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3182
3183	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3184	pPool->iPhysExtFreeHead = iPhysExtStart;
3185	}
3186
3187
3188	/**
3189	* Insert a reference into a list of physical cross reference extents.
3190	*
3191	* @returns The new ram range flags (top 16-bits).
3192	*
3193	* @param pVM The VM handle.
3194	* @param iPhysExt The physical extent index of the list head.
3195	* @param iShwPT The shadow page table index.
3196	*
3197	*/
3198	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
3199	{
3200	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3201	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3202
3203	/* special common case. */
3204	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3205	{
3206	paPhysExts[iPhysExt].aidx[2] = iShwPT;
3207	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3208	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{,,%d}\n", iPhysExt, iShwPT));
3209	return iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
3210	}
3211
3212	/* general treatment. */
3213	const uint16_t iPhysExtStart = iPhysExt;
3214	unsigned cMax = 15;
3215	for (;;)
3216	{
3217	Assert(iPhysExt < pPool->cMaxPhysExts);
3218	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3219	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3220	{
3221	paPhysExts[iPhysExt].aidx[i] = iShwPT;
3222	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3223	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
3224	return iPhysExtStart \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
3225	}
3226	if (!--cMax)
3227	{
3228	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3229	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3230	LogFlow(("pgmPoolTrackPhysExtAddref: overflow (1) iShwPT=%d\n", iShwPT));
3231	return MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
3232	}
3233	}
3234
3235	/* add another extent to the list. */
3236	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3237	if (!pNew)
3238	{
3239	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3240	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3241	return MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
3242	}
3243	pNew->iNext = iPhysExtStart;
3244	pNew->aidx[0] = iShwPT;
3245	LogFlow(("pgmPoolTrackPhysExtAddref: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
3246	return iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
3247	}
3248
3249
3250	/**
3251	* Add a reference to guest physical page where extents are in use.
3252	*
3253	* @returns The new ram range flags (top 16-bits).
3254	*
3255	* @param pVM The VM handle.
3256	* @param u16 The ram range flags (top 16-bits).
3257	* @param iShwPT The shadow page table index.
3258	*/
3259	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
3260	{
3261	if ((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) != MM_RAM_FLAGS_CREFS_PHYSEXT)
3262	{
3263	/*
3264	* Convert to extent list.
3265	*/
3266	Assert((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) == 1);
3267	uint16_t iPhysExt;
3268	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3269	if (pPhysExt)
3270	{
3271	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT));
3272	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
3273	pPhysExt->aidx[0] = u16 & MM_RAM_FLAGS_IDX_MASK;
3274	pPhysExt->aidx[1] = iShwPT;
3275	u16 = iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
3276	}
3277	else
3278	u16 = MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
3279	}
3280	else if (u16 != (MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT))))
3281	{
3282	/*
3283	* Insert into the extent list.
3284	*/
3285	u16 = pgmPoolTrackPhysExtInsert(pVM, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT);
3286	}
3287	else
3288	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
3289	return u16;
3290	}
3291
3292
3293	/**
3294	* Clear references to guest physical memory.
3295	*
3296	* @param pPool The pool.
3297	* @param pPage The page.
3298	* @param pPhysPage Pointer to the aPages entry in the ram range.
3299	*/
3300	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
3301	{
3302	const unsigned cRefs = pPhysPage->HCPhys >> MM_RAM_FLAGS_CREFS_SHIFT; /** @todo PAGE FLAGS */
3303	AssertFatalMsg(cRefs == MM_RAM_FLAGS_CREFS_PHYSEXT, ("cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
3304
3305	uint16_t iPhysExt = (pPhysPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT) & MM_RAM_FLAGS_IDX_MASK;
3306	if (iPhysExt != MM_RAM_FLAGS_IDX_OVERFLOWED)
3307	{
3308	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
3309	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3310	do
3311	{
3312	Assert(iPhysExt < pPool->cMaxPhysExts);
3313
3314	/*
3315	* Look for the shadow page and check if it's all freed.
3316	*/
3317	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3318	{
3319	if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
3320	{
3321	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
3322
3323	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3324	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
3325	{
3326	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
3327	return;
3328	}
3329
3330	/* we can free the node. */
3331	PVM pVM = pPool->CTX_SUFF(pVM);
3332	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
3333	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
3334	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
3335	{
3336	/* lonely node */
3337	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3338	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d lonely\n", pPhysPage->HCPhys, pPage->idx));
3339	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
3340	}
3341	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
3342	{
3343	/* head */
3344	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d head\n", pPhysPage->HCPhys, pPage->idx));
3345	pPhysPage->HCPhys = (pPhysPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) /** @todo PAGE FLAGS */
3346	\| ((uint64_t)MM_RAM_FLAGS_CREFS_PHYSEXT << MM_RAM_FLAGS_CREFS_SHIFT)
3347	\| ((uint64_t)iPhysExtNext << MM_RAM_FLAGS_IDX_SHIFT);
3348	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3349	}
3350	else
3351	{
3352	/* in list */
3353	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
3354	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
3355	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3356	}
3357	iPhysExt = iPhysExtNext;
3358	return;
3359	}
3360	}
3361
3362	/* next */
3363	iPhysExtPrev = iPhysExt;
3364	iPhysExt = paPhysExts[iPhysExt].iNext;
3365	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3366
3367	AssertFatalMsgFailed(("not-found! cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
3368	}
3369	else /* nothing to do */
3370	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64\n", pPhysPage->HCPhys));
3371	}
3372
3373
3374	/**
3375	* Clear references to guest physical memory.
3376	*
3377	* This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
3378	* is assumed to be correct, so the linear search can be skipped and we can assert
3379	* at an earlier point.
3380	*
3381	* @param pPool The pool.
3382	* @param pPage The page.
3383	* @param HCPhys The host physical address corresponding to the guest page.
3384	* @param GCPhys The guest physical address corresponding to HCPhys.
3385	*/
3386	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
3387	{
3388	/*
3389	* Walk range list.
3390	*/
3391	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3392	while (pRam)
3393	{
3394	RTGCPHYS off = GCPhys - pRam->GCPhys;
3395	if (off < pRam->cb)
3396	{
3397	/* does it match? */
3398	const unsigned iPage = off >> PAGE_SHIFT;
3399	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3400	#ifdef LOG_ENABLED
3401	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
3402	Log(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
3403	#endif
3404	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3405	{
3406	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3407	return;
3408	}
3409	break;
3410	}
3411	pRam = pRam->CTX_SUFF(pNext);
3412	}
3413	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
3414	}
3415
3416
3417	/**
3418	* Clear references to guest physical memory.
3419	*
3420	* @param pPool The pool.
3421	* @param pPage The page.
3422	* @param HCPhys The host physical address corresponding to the guest page.
3423	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
3424	*/
3425	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
3426	{
3427	/*
3428	* Walk range list.
3429	*/
3430	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3431	while (pRam)
3432	{
3433	RTGCPHYS off = GCPhysHint - pRam->GCPhys;
3434	if (off < pRam->cb)
3435	{
3436	/* does it match? */
3437	const unsigned iPage = off >> PAGE_SHIFT;
3438	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3439	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3440	{
3441	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3442	return;
3443	}
3444	break;
3445	}
3446	pRam = pRam->CTX_SUFF(pNext);
3447	}
3448
3449	/*
3450	* Damn, the hint didn't work. We'll have to do an expensive linear search.
3451	*/
3452	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
3453	pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3454	while (pRam)
3455	{
3456	unsigned iPage = pRam->cb >> PAGE_SHIFT;
3457	while (iPage-- > 0)
3458	{
3459	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3460	{
3461	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
3462	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
3463	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3464	return;
3465	}
3466	}
3467	pRam = pRam->CTX_SUFF(pNext);
3468	}
3469
3470	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp\n", HCPhys, GCPhysHint));
3471	}
3472
3473
3474	/**
3475	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
3476	*
3477	* @param pPool The pool.
3478	* @param pPage The page.
3479	* @param pShwPT The shadow page table (mapping of the page).
3480	* @param pGstPT The guest page table.
3481	*/
3482	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
3483	{
3484	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
3485	if (pShwPT->a[i].n.u1Present)
3486	{
3487	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
3488	i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3489	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3490	if (!--pPage->cPresent)
3491	break;
3492	}
3493	}
3494
3495
3496	/**
3497	* Clear references to guest physical memory in a PAE / 32-bit page table.
3498	*
3499	* @param pPool The pool.
3500	* @param pPage The page.
3501	* @param pShwPT The shadow page table (mapping of the page).
3502	* @param pGstPT The guest page table (just a half one).
3503	*/
3504	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
3505	{
3506	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3507	if (pShwPT->a[i].n.u1Present)
3508	{
3509	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX32 hint=%RX32\n",
3510	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3511	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3512	}
3513	}
3514
3515
3516	/**
3517	* Clear references to guest physical memory in a PAE / PAE page table.
3518	*
3519	* @param pPool The pool.
3520	* @param pPage The page.
3521	* @param pShwPT The shadow page table (mapping of the page).
3522	* @param pGstPT The guest page table.
3523	*/
3524	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
3525	{
3526	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3527	if (pShwPT->a[i].n.u1Present)
3528	{
3529	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
3530	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
3531	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
3532	}
3533	}
3534
3535
3536	/**
3537	* Clear references to guest physical memory in a 32-bit / 4MB page table.
3538	*
3539	* @param pPool The pool.
3540	* @param pPage The page.
3541	* @param pShwPT The shadow page table (mapping of the page).
3542	*/
3543	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
3544	{
3545	RTGCPHYS GCPhys = pPage->GCPhys;
3546	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3547	if (pShwPT->a[i].n.u1Present)
3548	{
3549	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
3550	i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
3551	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
3552	}
3553	}
3554
3555
3556	/**
3557	* Clear references to guest physical memory in a PAE / 2/4MB page table.
3558	*
3559	* @param pPool The pool.
3560	* @param pPage The page.
3561	* @param pShwPT The shadow page table (mapping of the page).
3562	*/
3563	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
3564	{
3565	RTGCPHYS GCPhys = pPage->GCPhys;
3566	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3567	if (pShwPT->a[i].n.u1Present)
3568	{
3569	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
3570	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
3571	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
3572	}
3573	}
3574
3575	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3576
3577
3578	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
3579	/**
3580	* Clear references to shadowed pages in a 32 bits page directory.
3581	*
3582	* @param pPool The pool.
3583	* @param pPage The page.
3584	* @param pShwPD The shadow page directory (mapping of the page).
3585	*/
3586	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
3587	{
3588	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3589	{
3590	if ( pShwPD->a[i].n.u1Present
3591	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3592	)
3593	{
3594	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
3595	if (pSubPage)
3596	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3597	else
3598	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
3599	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3600	}
3601	}
3602	}
3603	#endif
3604
3605	/**
3606	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
3607	*
3608	* @param pPool The pool.
3609	* @param pPage The page.
3610	* @param pShwPD The shadow page directory (mapping of the page).
3611	*/
3612	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
3613	{
3614	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3615	{
3616	if ( pShwPD->a[i].n.u1Present
3617	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
3618	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3619	#endif
3620	)
3621	{
3622	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
3623	if (pSubPage)
3624	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3625	else
3626	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
3627	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3628	}
3629	}
3630	}
3631
3632
3633	/**
3634	* Clear references to shadowed pages in a 64-bit page directory pointer table.
3635	*
3636	* @param pPool The pool.
3637	* @param pPage The page.
3638	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3639	*/
3640	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3641	{
3642	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3643	{
3644	if ( pShwPDPT->a[i].n.u1Present
3645	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
3646	&& !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
3647	#endif
3648	)
3649	{
3650	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3651	if (pSubPage)
3652	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3653	else
3654	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3655	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3656	}
3657	}
3658	}
3659
3660
3661	/**
3662	* Clear references to shadowed pages in a 64-bit level 4 page table.
3663	*
3664	* @param pPool The pool.
3665	* @param pPage The page.
3666	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
3667	*/
3668	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
3669	{
3670	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
3671	{
3672	if (pShwPML4->a[i].n.u1Present)
3673	{
3674	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
3675	if (pSubPage)
3676	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3677	else
3678	AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
3679	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3680	}
3681	}
3682	}
3683
3684
3685	/**
3686	* Clear references to shadowed pages in an EPT page table.
3687	*
3688	* @param pPool The pool.
3689	* @param pPage The page.
3690	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
3691	*/
3692	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
3693	{
3694	RTGCPHYS GCPhys = pPage->GCPhys;
3695	for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3696	if (pShwPT->a[i].n.u1Present)
3697	{
3698	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
3699	i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
3700	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys);
3701	}
3702	}
3703
3704
3705	/**
3706	* Clear references to shadowed pages in an EPT page directory.
3707	*
3708	* @param pPool The pool.
3709	* @param pPage The page.
3710	* @param pShwPD The shadow page directory (mapping of the page).
3711	*/
3712	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
3713	{
3714	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3715	{
3716	if (pShwPD->a[i].n.u1Present)
3717	{
3718	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
3719	if (pSubPage)
3720	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3721	else
3722	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
3723	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3724	}
3725	}
3726	}
3727
3728
3729	/**
3730	* Clear references to shadowed pages in an EPT page directory pointer table.
3731	*
3732	* @param pPool The pool.
3733	* @param pPage The page.
3734	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3735	*/
3736	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
3737	{
3738	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3739	{
3740	if (pShwPDPT->a[i].n.u1Present)
3741	{
3742	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
3743	if (pSubPage)
3744	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3745	else
3746	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
3747	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3748	}
3749	}
3750	}
3751
3752
3753	/**
3754	* Clears all references made by this page.
3755	*
3756	* This includes other shadow pages and GC physical addresses.
3757	*
3758	* @param pPool The pool.
3759	* @param pPage The page.
3760	*/
3761	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3762	{
3763	/*
3764	* Map the shadow page and take action according to the page kind.
3765	*/
3766	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
3767	switch (pPage->enmKind)
3768	{
3769	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3770	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3771	{
3772	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3773	void *pvGst;
3774	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3775	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
3776	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3777	break;
3778	}
3779
3780	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3781	{
3782	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3783	void *pvGst;
3784	int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3785	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
3786	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3787	break;
3788	}
3789
3790	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3791	{
3792	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3793	void *pvGst;
3794	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3795	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
3796	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3797	break;
3798	}
3799
3800	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
3801	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3802	{
3803	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3804	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
3805	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3806	break;
3807	}
3808
3809	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
3810	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3811	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3812	{
3813	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3814	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
3815	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3816	break;
3817	}
3818
3819	#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3820	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3821	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3822	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3823	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3824	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3825	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3826	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3827	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3828	break;
3829	#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3830
3831	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3832	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3833	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3834	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3835	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3836	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3837	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3838	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
3839	break;
3840
3841	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
3842	case PGMPOOLKIND_32BIT_PD:
3843	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
3844	break;
3845
3846	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3847	case PGMPOOLKIND_PAE_PDPT:
3848	#endif
3849	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3850	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3851	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
3852	break;
3853
3854	case PGMPOOLKIND_64BIT_PML4:
3855	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
3856	break;
3857
3858	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3859	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
3860	break;
3861
3862	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3863	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
3864	break;
3865
3866	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3867	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
3868	break;
3869
3870	default:
3871	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
3872	}
3873
3874	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
3875	STAM_PROFILE_START(&pPool->StatZeroPage, z);
3876	ASMMemZeroPage(pvShw);
3877	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3878	pPage->fZeroed = true;
3879	}
3880
3881	#endif /* PGMPOOL_WITH_USER_TRACKING */
3882
3883	/**
3884	* Flushes all the special root pages as part of a pgmPoolFlushAllInt operation.
3885	*
3886	* @param pPool The pool.
3887	*/
3888	static void pgmPoolFlushAllSpecialRoots(PPGMPOOL pPool)
3889	{
3890	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3891	/* Start a subset so we won't run out of mapping space. */
3892	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
3893	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3894	#endif
3895
3896	/*
3897	* These special pages are all mapped into the indexes 1..PGMPOOL_IDX_FIRST.
3898	*/
3899	Assert(NIL_PGMPOOL_IDX == 0);
3900	for (unsigned i = 1; i < PGMPOOL_IDX_FIRST; i++)
3901	{
3902	/*
3903	* Get the page address.
3904	*/
3905	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3906	union
3907	{
3908	uint64_t *pau64;
3909	uint32_t *pau32;
3910	} u;
3911
3912	/*
3913	* Mark stuff not present.
3914	*/
3915	switch (pPage->enmKind)
3916	{
3917	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
3918	case PGMPOOLKIND_ROOT_32BIT_PD:
3919	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
3920	for (unsigned iPage = 0; iPage < X86_PG_ENTRIES; iPage++)
3921	if ((u.pau32[iPage] & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == X86_PDE_P)
3922	u.pau32[iPage] = 0;
3923	break;
3924
3925	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3926	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
3927	for (unsigned iPage = 0; iPage < X86_PG_PAE_ENTRIES; iPage++)
3928	if ((u.pau64[iPage] & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == X86_PDE_P)
3929	u.pau64[iPage] = 0;
3930	break;
3931
3932	case PGMPOOLKIND_ROOT_PDPT:
3933	/* Not root of shadowed pages currently, ignore it. */
3934	break;
3935	#endif
3936
3937	case PGMPOOLKIND_ROOT_NESTED:
3938	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
3939	ASMMemZero32(u.pau64, PAGE_SIZE);
3940	break;
3941	}
3942	}
3943
3944	/*
3945	* Paranoia (to be removed), flag a global CR3 sync.
3946	*/
3947	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
3948
3949	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3950	/* Pop the subset. */
3951	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3952	#endif
3953	}
3954
3955
3956	/**
3957	* Flushes the entire cache.
3958	*
3959	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3960	* and execute this CR3 flush.
3961	*
3962	* @param pPool The pool.
3963	*/
3964	static void pgmPoolFlushAllInt(PPGMPOOL pPool)
3965	{
3966	STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
3967	LogFlow(("pgmPoolFlushAllInt:\n"));
3968
3969	/*
3970	* If there are no pages in the pool, there is nothing to do.
3971	*/
3972	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
3973	{
3974	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3975	return;
3976	}
3977
3978	/*
3979	* Nuke the free list and reinsert all pages into it.
3980	*/
3981	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
3982	{
3983	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3984
3985	#ifdef IN_RING3
3986	Assert(pPage->Core.Key == MMPage2Phys(pPool->pVMR3, pPage->pvPageR3));
3987	#endif
3988	#ifdef PGMPOOL_WITH_MONITORING
3989	if (pPage->fMonitored)
3990	pgmPoolMonitorFlush(pPool, pPage);
3991	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3992	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3993	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3994	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3995	pPage->cModifications = 0;
3996	#endif
3997	pPage->GCPhys = NIL_RTGCPHYS;
3998	pPage->enmKind = PGMPOOLKIND_FREE;
3999	Assert(pPage->idx == i);
4000	pPage->iNext = i + 1;
4001	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
4002	pPage->fSeenNonGlobal = false;
4003	pPage->fMonitored= false;
4004	pPage->fCached = false;
4005	pPage->fReusedFlushPending = false;
4006	pPage->fCR3Mix = false;
4007	#ifdef PGMPOOL_WITH_USER_TRACKING
4008	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4009	#endif
4010	#ifdef PGMPOOL_WITH_CACHE
4011	pPage->iAgeNext = NIL_PGMPOOL_IDX;
4012	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4013	#endif
4014	}
4015	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
4016	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
4017	pPool->cUsedPages = 0;
4018
4019	#ifdef PGMPOOL_WITH_USER_TRACKING
4020	/*
4021	* Zap and reinitialize the user records.
4022	*/
4023	pPool->cPresent = 0;
4024	pPool->iUserFreeHead = 0;
4025	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4026	const unsigned cMaxUsers = pPool->cMaxUsers;
4027	for (unsigned i = 0; i < cMaxUsers; i++)
4028	{
4029	paUsers[i].iNext = i + 1;
4030	paUsers[i].iUser = NIL_PGMPOOL_IDX;
4031	paUsers[i].iUserTable = 0xfffffffe;
4032	}
4033	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
4034	#endif
4035
4036	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4037	/*
4038	* Clear all the GCPhys links and rebuild the phys ext free list.
4039	*/
4040	for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
4041	pRam;
4042	pRam = pRam->CTX_SUFF(pNext))
4043	{
4044	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4045	while (iPage-- > 0)
4046	pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
4047	}
4048
4049	pPool->iPhysExtFreeHead = 0;
4050	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4051	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
4052	for (unsigned i = 0; i < cMaxPhysExts; i++)
4053	{
4054	paPhysExts[i].iNext = i + 1;
4055	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
4056	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
4057	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
4058	}
4059	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4060	#endif
4061
4062	#ifdef PGMPOOL_WITH_MONITORING
4063	/*
4064	* Just zap the modified list.
4065	*/
4066	pPool->cModifiedPages = 0;
4067	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
4068	#endif
4069
4070	#ifdef PGMPOOL_WITH_CACHE
4071	/*
4072	* Clear the GCPhys hash and the age list.
4073	*/
4074	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
4075	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
4076	pPool->iAgeHead = NIL_PGMPOOL_IDX;
4077	pPool->iAgeTail = NIL_PGMPOOL_IDX;
4078	#endif
4079
4080	/*
4081	* Flush all the special root pages.
4082	* Reinsert active pages into the hash and ensure monitoring chains are correct.
4083	*/
4084	pgmPoolFlushAllSpecialRoots(pPool);
4085	for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
4086	{
4087	PPGMPOOLPAGE pPage = &pPool->aPages[i];
4088	pPage->iNext = NIL_PGMPOOL_IDX;
4089	#ifdef PGMPOOL_WITH_MONITORING
4090	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4091	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4092	pPage->cModifications = 0;
4093	/* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
4094	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4095	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4096	if (pPage->fMonitored)
4097	{
4098	PVM pVM = pPool->CTX_SUFF(pVM);
4099	int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
4100	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
4101	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
4102	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
4103	pPool->pszAccessHandler);
4104	AssertFatalRCSuccess(rc);
4105	# ifdef PGMPOOL_WITH_CACHE
4106	pgmPoolHashInsert(pPool, pPage);
4107	# endif
4108	}
4109	#endif
4110	#ifdef PGMPOOL_WITH_USER_TRACKING
4111	Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
4112	#endif
4113	#ifdef PGMPOOL_WITH_CACHE
4114	Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
4115	Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
4116	#endif
4117	}
4118
4119	/*
4120	* Finally, assert the FF.
4121	*/
4122	VM_FF_SET(pPool->CTX_SUFF(pVM), VM_FF_PGM_SYNC_CR3);
4123
4124	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4125	}
4126
4127
4128	/**
4129	* Flushes a pool page.
4130	*
4131	* This moves the page to the free list after removing all user references to it.
4132	* In GC this will cause a CR3 reload if the page is traced back to an active root page.
4133	*
4134	* @returns VBox status code.
4135	* @retval VINF_SUCCESS on success.
4136	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
4137	* @param pPool The pool.
4138	* @param HCPhys The HC physical address of the shadow page.
4139	*/
4140	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4141	{
4142	int rc = VINF_SUCCESS;
4143	STAM_PROFILE_START(&pPool->StatFlushPage, f);
4144	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%d, .GCPhys=%RGp}\n",
4145	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
4146
4147	/*
4148	* Quietly reject any attempts at flushing any of the special root pages.
4149	*/
4150	if (pPage->idx < PGMPOOL_IDX_FIRST)
4151	{
4152	Log(("pgmPoolFlushPage: special root page, rejected. enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
4153	return VINF_SUCCESS;
4154	}
4155
4156	/*
4157	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4158	*/
4159	if (PGMGetHyperCR3(pPool->CTX_SUFF(pVM)) == pPage->Core.Key)
4160	{
4161	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
4162	AssertMsg(pPage->enmKind == PGMPOOLKIND_64BIT_PML4,
4163	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(pPool->CTX_SUFF(pVM)), pPage->Core.Key, pPage->enmKind));
4164	#endif
4165	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
4166	return VINF_SUCCESS;
4167	}
4168
4169	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4170	/* Start a subset so we won't run out of mapping space. */
4171	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
4172	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
4173	#endif
4174
4175	/*
4176	* Mark the page as being in need of a ASMMemZeroPage().
4177	*/
4178	pPage->fZeroed = false;
4179
4180	#ifdef PGMPOOL_WITH_USER_TRACKING
4181	/*
4182	* Clear the page.
4183	*/
4184	pgmPoolTrackClearPageUsers(pPool, pPage);
4185	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
4186	pgmPoolTrackDeref(pPool, pPage);
4187	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
4188	#endif
4189
4190	#ifdef PGMPOOL_WITH_CACHE
4191	/*
4192	* Flush it from the cache.
4193	*/
4194	pgmPoolCacheFlushPage(pPool, pPage);
4195	#endif /* PGMPOOL_WITH_CACHE */
4196
4197	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4198	/* Heavy stuff done. */
4199	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
4200	#endif
4201
4202	#ifdef PGMPOOL_WITH_MONITORING
4203	/*
4204	* Deregistering the monitoring.
4205	*/
4206	if (pPage->fMonitored)
4207	rc = pgmPoolMonitorFlush(pPool, pPage);
4208	#endif
4209
4210	/*
4211	* Free the page.
4212	*/
4213	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
4214	pPage->iNext = pPool->iFreeHead;
4215	pPool->iFreeHead = pPage->idx;
4216	pPage->enmKind = PGMPOOLKIND_FREE;
4217	pPage->GCPhys = NIL_RTGCPHYS;
4218	pPage->fReusedFlushPending = false;
4219
4220	pPool->cUsedPages--;
4221	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
4222	return rc;
4223	}
4224
4225
4226	/**
4227	* Frees a usage of a pool page.
4228	*
4229	* The caller is responsible to updating the user table so that it no longer
4230	* references the shadow page.
4231	*
4232	* @param pPool The pool.
4233	* @param HCPhys The HC physical address of the shadow page.
4234	* @param iUser The shadow page pool index of the user table.
4235	* @param iUserTable The index into the user table (shadowed).
4236	*/
4237	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
4238	{
4239	STAM_PROFILE_START(&pPool->StatFree, a);
4240	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%d} iUser=%#x iUserTable=%#x\n",
4241	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, iUser, iUserTable));
4242	Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
4243	#ifdef PGMPOOL_WITH_USER_TRACKING
4244	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
4245	#endif
4246	#ifdef PGMPOOL_WITH_CACHE
4247	if (!pPage->fCached)
4248	#endif
4249	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED can be ignored here. */
4250	STAM_PROFILE_STOP(&pPool->StatFree, a);
4251	}
4252
4253
4254	/**
4255	* Makes one or more free page free.
4256	*
4257	* @returns VBox status code.
4258	* @retval VINF_SUCCESS on success.
4259	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4260	*
4261	* @param pPool The pool.
4262	* @param iUser The user of the page.
4263	*/
4264	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, uint16_t iUser)
4265	{
4266	LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
4267
4268	/*
4269	* If the pool isn't full grown yet, expand it.
4270	*/
4271	if (pPool->cCurPages < pPool->cMaxPages)
4272	{
4273	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
4274	#ifdef IN_RING3
4275	int rc = PGMR3PoolGrow(pPool->pVMR3);
4276	#else
4277	int rc = CTXALLMID(VMM, CallHost)(pPool->CTX_SUFF(pVM), VMMCALLHOST_PGM_POOL_GROW, 0);
4278	#endif
4279	if (RT_FAILURE(rc))
4280	return rc;
4281	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
4282	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
4283	return VINF_SUCCESS;
4284	}
4285
4286	#ifdef PGMPOOL_WITH_CACHE
4287	/*
4288	* Free one cached page.
4289	*/
4290	return pgmPoolCacheFreeOne(pPool, iUser);
4291	#else
4292	/*
4293	* Flush the pool.
4294	*
4295	* If we have tracking enabled, it should be possible to come up with
4296	* a cheap replacement strategy...
4297	*/
4298	/* @todo incompatible with long mode paging (cr3 root will be flushed) */
4299	Assert(!CPUMIsGuestInLongMode(pVM));
4300	pgmPoolFlushAllInt(pPool);
4301	return VERR_PGM_POOL_FLUSHED;
4302	#endif
4303	}
4304
4305
4306	/**
4307	* Allocates a page from the pool.
4308	*
4309	* This page may actually be a cached page and not in need of any processing
4310	* on the callers part.
4311	*
4312	* @returns VBox status code.
4313	* @retval VINF_SUCCESS if a NEW page was allocated.
4314	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4315	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4316	* @param pVM The VM handle.
4317	* @param GCPhys The GC physical address of the page we're gonna shadow.
4318	* For 4MB and 2MB PD entries, it's the first address the
4319	* shadow PT is covering.
4320	* @param enmKind The kind of mapping.
4321	* @param iUser The shadow page pool index of the user table.
4322	* @param iUserTable The index into the user table (shadowed).
4323	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4324	*/
4325	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
4326	{
4327	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4328	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4329	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%d iUser=%#x iUserTable=%#x\n", GCPhys, enmKind, iUser, iUserTable));
4330	*ppPage = NULL;
4331	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4332	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
4333	* Assert(!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
4334
4335	#ifdef PGMPOOL_WITH_CACHE
4336	if (pPool->fCacheEnabled)
4337	{
4338	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, iUser, iUserTable, ppPage);
4339	if (RT_SUCCESS(rc2))
4340	{
4341	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4342	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
4343	return rc2;
4344	}
4345	}
4346	#endif
4347
4348	/*
4349	* Allocate a new one.
4350	*/
4351	int rc = VINF_SUCCESS;
4352	uint16_t iNew = pPool->iFreeHead;
4353	if (iNew == NIL_PGMPOOL_IDX)
4354	{
4355	rc = pgmPoolMakeMoreFreePages(pPool, iUser);
4356	if (RT_FAILURE(rc))
4357	{
4358	if (rc != VERR_PGM_POOL_CLEARED)
4359	{
4360	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
4361	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4362	return rc;
4363	}
4364	Log(("pgmPoolMakeMoreFreePages failed with %Rrc -> return VERR_PGM_POOL_FLUSHED\n", rc));
4365	rc = VERR_PGM_POOL_FLUSHED;
4366	}
4367	iNew = pPool->iFreeHead;
4368	AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
4369	}
4370
4371	/* unlink the free head */
4372	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
4373	pPool->iFreeHead = pPage->iNext;
4374	pPage->iNext = NIL_PGMPOOL_IDX;
4375
4376	/*
4377	* Initialize it.
4378	*/
4379	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
4380	pPage->enmKind = enmKind;
4381	pPage->GCPhys = GCPhys;
4382	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
4383	pPage->fMonitored = false;
4384	pPage->fCached = false;
4385	pPage->fReusedFlushPending = false;
4386	pPage->fCR3Mix = false;
4387	#ifdef PGMPOOL_WITH_MONITORING
4388	pPage->cModifications = 0;
4389	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4390	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4391	#endif
4392	#ifdef PGMPOOL_WITH_USER_TRACKING
4393	pPage->cPresent = 0;
4394	pPage->iFirstPresent = ~0;
4395
4396	/*
4397	* Insert into the tracking and cache. If this fails, free the page.
4398	*/
4399	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
4400	if (RT_FAILURE(rc3))
4401	{
4402	if (rc3 != VERR_PGM_POOL_CLEARED)
4403	{
4404	pPool->cUsedPages--;
4405	pPage->enmKind = PGMPOOLKIND_FREE;
4406	pPage->GCPhys = NIL_RTGCPHYS;
4407	pPage->iNext = pPool->iFreeHead;
4408	pPool->iFreeHead = pPage->idx;
4409	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4410	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
4411	return rc3;
4412	}
4413	Log(("pgmPoolTrackInsert failed with %Rrc -> return VERR_PGM_POOL_FLUSHED\n", rc3));
4414	rc = VERR_PGM_POOL_FLUSHED;
4415	}
4416	#endif /* PGMPOOL_WITH_USER_TRACKING */
4417
4418	/*
4419	* Commit the allocation, clear the page and return.
4420	*/
4421	#ifdef VBOX_WITH_STATISTICS
4422	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
4423	pPool->cUsedPagesHigh = pPool->cUsedPages;
4424	#endif
4425
4426	if (!pPage->fZeroed)
4427	{
4428	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4429	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4430	ASMMemZeroPage(pv);
4431	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4432	}
4433
4434	*ppPage = pPage;
4435	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
4436	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
4437	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4438	return rc;
4439	}
4440
4441
4442	/**
4443	* Frees a usage of a pool page.
4444	*
4445	* @param pVM The VM handle.
4446	* @param HCPhys The HC physical address of the shadow page.
4447	* @param iUser The shadow page pool index of the user table.
4448	* @param iUserTable The index into the user table (shadowed).
4449	*/
4450	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
4451	{
4452	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
4453	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4454	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
4455	}
4456
4457
4458	/**
4459	* Gets a in-use page in the pool by it's physical address.
4460	*
4461	* @returns Pointer to the page.
4462	* @param pVM The VM handle.
4463	* @param HCPhys The HC physical address of the shadow page.
4464	* @remark This function will NEVER return NULL. It will assert if HCPhys is invalid.
4465	*/
4466	PPGMPOOLPAGE pgmPoolGetPageByHCPhys(PVM pVM, RTHCPHYS HCPhys)
4467	{
4468	/** @todo profile this! */
4469	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4470	PPGMPOOLPAGE pPage = pgmPoolGetPage(pPool, HCPhys);
4471	Log3(("pgmPoolGetPageByHCPhys: HCPhys=%RHp -> %p:{.idx=%d .GCPhys=%RGp .enmKind=%d}\n",
4472	HCPhys, pPage, pPage->idx, pPage->GCPhys, pPage->enmKind));
4473	return pPage;
4474	}
4475
4476
4477	/**
4478	* Flushes the entire cache.
4479	*
4480	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
4481	* and execute this CR3 flush.
4482	*
4483	* @param pPool The pool.
4484	*/
4485	void pgmPoolFlushAll(PVM pVM)
4486	{
4487	LogFlow(("pgmPoolFlushAll:\n"));
4488	pgmPoolFlushAllInt(pVM->pgm.s.CTX_SUFF(pPool));
4489	}
4490

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 16634

Download in other formats: