PGMAllPool.cpp@ 8110

Last change on this file since 8110 was 8101, checked in by vboxsync, 17 years ago
Got rid of incorrect GC case for failed instruction emulation. Could cause an infinite loop.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 125.1 KB

Line
1	/* $Id: PGMAllPool.cpp 8101 2008-04-17 14:23:24Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 innotek GmbH
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_POOL
23	#include <VBox/pgm.h>
24	#include <VBox/mm.h>
25	#include <VBox/em.h>
26	#include <VBox/cpum.h>
27	#ifdef IN_GC
28	# include <VBox/patm.h>
29	#endif
30	#include "PGMInternal.h"
31	#include <VBox/vm.h>
32	#include <VBox/disopcode.h>
33
34	#include <VBox/log.h>
35	#include <VBox/err.h>
36	#include <iprt/asm.h>
37
38
39	/*******************************************************************************
40	* Internal Functions *
41	*******************************************************************************/
42	__BEGIN_DECLS
43	static void pgmPoolFlushAllInt(PPGMPOOL pPool);
44	#ifdef PGMPOOL_WITH_USER_TRACKING
45	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
46	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
47	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
48	#endif
49	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
50	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint);
51	#endif
52	#ifdef PGMPOOL_WITH_CACHE
53	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable);
54	#endif
55	#ifdef PGMPOOL_WITH_MONITORING
56	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
57	#endif
58	#ifndef IN_RING3
59	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
60	#endif
61	__END_DECLS
62
63
64	/**
65	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
66	*
67	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
68	* @param enmKind The page kind.
69	*/
70	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
71	{
72	switch (enmKind)
73	{
74	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
75	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
76	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
77	return true;
78	default:
79	return false;
80	}
81	}
82
83
84	#ifdef IN_GC
85	/**
86	* Maps a pool page into the current context.
87	*
88	* @returns Pointer to the mapping.
89	* @param pVM The VM handle.
90	* @param pPage The page to map.
91	*/
92	void *pgmGCPoolMapPage(PVM pVM, PPGMPOOLPAGE pPage)
93	{
94	/* general pages. */
95	if (pPage->idx >= PGMPOOL_IDX_FIRST)
96	{
97	Assert(pPage->idx < pVM->pgm.s.pPoolGC->cCurPages);
98	void *pv;
99	int rc = PGMGCDynMapHCPage(pVM, pPage->Core.Key, &pv);
100	AssertReleaseRC(rc);
101	return pv;
102	}
103
104	/* special pages. */
105	switch (pPage->idx)
106	{
107	case PGMPOOL_IDX_PD:
108	return pVM->pgm.s.pGC32BitPD;
109	case PGMPOOL_IDX_PAE_PD:
110	case PGMPOOL_IDX_PAE_PD_0:
111	return pVM->pgm.s.apGCPaePDs[0];
112	case PGMPOOL_IDX_PAE_PD_1:
113	return pVM->pgm.s.apGCPaePDs[1];
114	case PGMPOOL_IDX_PAE_PD_2:
115	return pVM->pgm.s.apGCPaePDs[2];
116	case PGMPOOL_IDX_PAE_PD_3:
117	return pVM->pgm.s.apGCPaePDs[3];
118	case PGMPOOL_IDX_PDPT:
119	return pVM->pgm.s.pGCPaePDPT;
120	case PGMPOOL_IDX_PML4:
121	return pVM->pgm.s.pGCPaePML4;
122	default:
123	AssertReleaseMsgFailed(("Invalid index %d\n", pPage->idx));
124	return NULL;
125	}
126	}
127	#endif /* IN_GC */
128
129
130	#ifdef PGMPOOL_WITH_MONITORING
131	/**
132	* Determin the size of a write instruction.
133	* @returns number of bytes written.
134	* @param pDis The disassembler state.
135	*/
136	static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
137	{
138	/*
139	* This is very crude and possibly wrong for some opcodes,
140	* but since it's not really supposed to be called we can
141	* probably live with that.
142	*/
143	return DISGetParamSize(pDis, &pDis->param1);
144	}
145
146
147	/**
148	* Flushes a chain of pages sharing the same access monitor.
149	*
150	* @returns VBox status code suitable for scheduling.
151	* @param pPool The pool.
152	* @param pPage A page in the chain.
153	*/
154	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
155	{
156	LogFlow(("pgmPoolMonitorChainFlush: Flush page %VGp type=%d\n", pPage->GCPhys, pPage->enmKind));
157
158	/*
159	* Find the list head.
160	*/
161	uint16_t idx = pPage->idx;
162	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
163	{
164	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
165	{
166	idx = pPage->iMonitoredPrev;
167	Assert(idx != pPage->idx);
168	pPage = &pPool->aPages[idx];
169	}
170	}
171
172	/*
173	* Itereate the list flushing each shadow page.
174	*/
175	int rc = VINF_SUCCESS;
176	for (;;)
177	{
178	idx = pPage->iMonitoredNext;
179	Assert(idx != pPage->idx);
180	if (pPage->idx >= PGMPOOL_IDX_FIRST)
181	{
182	int rc2 = pgmPoolFlushPage(pPool, pPage);
183	if (rc2 == VERR_PGM_POOL_CLEARED && rc == VINF_SUCCESS)
184	rc = VINF_PGM_SYNC_CR3;
185	}
186	/* next */
187	if (idx == NIL_PGMPOOL_IDX)
188	break;
189	pPage = &pPool->aPages[idx];
190	}
191	return rc;
192	}
193
194
195	/**
196	* Wrapper for getting the current context pointer to the entry being modified.
197	*
198	* @returns Pointer to the current context mapping of the entry.
199	* @param pPool The pool.
200	* @param pvFault The fault virtual address.
201	* @param GCPhysFault The fault physical address.
202	* @param cbEntry The entry size.
203	*/
204	#ifdef IN_RING3
205	DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTHCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
206	#else
207	DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTGCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
208	#endif
209	{
210	#ifdef IN_GC
211	return (RTGCPTR)((RTGCUINTPTR)pvFault & ~(RTGCUINTPTR)(cbEntry - 1));
212
213	#elif defined(IN_RING0)
214	void *pvRet;
215	int rc = pgmRamGCPhys2HCPtr(&pPool->pVMHC->pgm.s, GCPhysFault & ~(RTGCPHYS)(cbEntry - 1), &pvRet);
216	AssertFatalRCSuccess(rc);
217	return pvRet;
218
219	#elif defined(IN_RING3)
220	return (RTHCPTR)((uintptr_t)pvFault & ~(RTHCUINTPTR)(cbEntry - 1));
221	#else
222	# error "huh?"
223	#endif
224	}
225
226
227	/**
228	* Process shadow entries before they are changed by the guest.
229	*
230	* For PT entries we will clear them. For PD entries, we'll simply check
231	* for mapping conflicts and set the SyncCR3 FF if found.
232	*
233	* @param pPool The pool.
234	* @param pPage The head page.
235	* @param GCPhysFault The guest physical fault address.
236	* @param uAddress In R0 and GC this is the guest context fault address (flat).
237	* In R3 this is the host context 'fault' address.
238	* @param pCpu The disassembler state for figuring out the write size.
239	* This need not be specified if the caller knows we won't do cross entry accesses.
240	*/
241	#ifdef IN_RING3
242	void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTHCPTR pvAddress, PDISCPUSTATE pCpu)
243	#else
244	void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTGCPTR pvAddress, PDISCPUSTATE pCpu)
245	#endif
246	{
247	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
248	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
249
250	LogFlow(("pgmPoolMonitorChainChanging: %VGv phys=%VGp kind=%d\n", pvAddress, GCPhysFault, pPage->enmKind));
251
252	for (;;)
253	{
254	union
255	{
256	void *pv;
257	PX86PT pPT;
258	PX86PTPAE pPTPae;
259	PX86PD pPD;
260	PX86PDPAE pPDPae;
261	PX86PDPT pPDPT;
262	} uShw;
263	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
264
265	switch (pPage->enmKind)
266	{
267	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
268	{
269	const unsigned iShw = off / sizeof(X86PTE);
270	if (uShw.pPT->a[iShw].n.u1Present)
271	{
272	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
273	PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
274	Log4(("pgmPoolMonitorChainChanging 32_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
275	pgmPoolTracDerefGCPhysHint(pPool, pPage,
276	uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
277	pGstPte->u & X86_PTE_PG_MASK);
278	# endif
279	uShw.pPT->a[iShw].u = 0;
280	}
281	break;
282	}
283
284	/* page/2 sized */
285	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
286	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
287	{
288	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
289	if (uShw.pPTPae->a[iShw].n.u1Present)
290	{
291	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
292	PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
293	Log4(("pgmPoolMonitorChainChanging pae_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
294	pgmPoolTracDerefGCPhysHint(pPool, pPage,
295	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
296	pGstPte->u & X86_PTE_PG_MASK);
297	# endif
298	uShw.pPTPae->a[iShw].u = 0;
299	}
300	}
301	break;
302
303	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
304	{
305	const unsigned iShw = off / sizeof(X86PTEPAE);
306	if (uShw.pPTPae->a[iShw].n.u1Present)
307	{
308	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
309	PCX86PTEPAE pGstPte = (PCX86PTEPAE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
310	Log4(("pgmPoolMonitorChainChanging pae_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PAE_PG_MASK));
311	pgmPoolTracDerefGCPhysHint(pPool, pPage,
312	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
313	pGstPte->u & X86_PTE_PAE_PG_MASK);
314	# endif
315	uShw.pPTPae->a[iShw].u = 0;
316	}
317	break;
318	}
319
320	case PGMPOOLKIND_ROOT_32BIT_PD:
321	{
322	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
323	if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
324	{
325	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
326	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
327	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
328	}
329	/* paranoia / a bit assumptive. */
330	else if ( pCpu
331	&& (off & 3)
332	&& (off & 3) + pgmPoolDisasWriteSize(pCpu) > 4)
333	{
334	const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PTE);
335	if ( iShw2 != iShw
336	&& iShw2 < ELEMENTS(uShw.pPD->a)
337	&& uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
338	{
339	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
340	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
341	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
342	}
343	}
344	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
345	if ( uShw.pPD->a[iShw].n.u1Present
346	&& !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
347	{
348	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
349	# ifdef IN_GC /* TLB load - we're pushing things a bit... */
350	ASMProbeReadByte(pvAddress);
351	# endif
352	pgmPoolFree(pPool->CTXSUFF(pVM), uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
353	uShw.pPD->a[iShw].u = 0;
354	}
355	#endif
356	break;
357	}
358
359	case PGMPOOLKIND_ROOT_PAE_PD:
360	{
361	unsigned iShw = (off / sizeof(X86PTE)) * 2; // ASSUMING 32-bit guest paging!
362	for (unsigned i = 0; i < 2; i++, iShw++)
363	{
364	if ((uShw.pPDPae->a[iShw].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
365	{
366	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
367	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
368	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
369	}
370	/* paranoia / a bit assumptive. */
371	else if ( pCpu
372	&& (off & 3)
373	&& (off & 3) + pgmPoolDisasWriteSize(pCpu) > 4)
374	{
375	const unsigned iShw2 = iShw + 2;
376	if ( iShw2 < ELEMENTS(uShw.pPDPae->a)
377	&& (uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
378	{
379	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
380	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
381	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
382	}
383	}
384	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
385	if ( uShw.pPDPae->a[iShw].n.u1Present
386	&& !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
387	{
388	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
389	# ifdef IN_GC /* TLB load - we're pushing things a bit... */
390	ASMProbeReadByte(pvAddress);
391	# endif
392	pgmPoolFree(pPool->CTXSUFF(pVM), uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
393	uShw.pPDPae->a[iShw].u = 0;
394	}
395	#endif
396	}
397	break;
398	}
399
400	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
401	{
402	const unsigned iShw = off / sizeof(X86PTEPAE);
403	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
404	{
405	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
406	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
407	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
408	}
409	/* paranoia / a bit assumptive. */
410	else if ( pCpu
411	&& (off & 7)
412	&& (off & 7) + pgmPoolDisasWriteSize(pCpu) > sizeof(X86PTEPAE))
413	{
414	const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PTEPAE);
415	if ( iShw2 != iShw
416	&& iShw2 < ELEMENTS(uShw.pPDPae->a)
417	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
418	{
419	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
420	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
421	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
422	}
423	}
424	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
425	if ( uShw.pPDPae->a[iShw].n.u1Present
426	&& !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
427	{
428	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
429	# ifdef IN_GC /* TLB load - we're pushing things a bit... */
430	ASMProbeReadByte(pvAddress);
431	# endif
432	pgmPoolFree(pPool->CTXSUFF(pVM), uShwpPDPae->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
433	uShw.pPDPae->a[iShw].u = 0;
434	}
435	#endif
436	break;
437	}
438
439	case PGMPOOLKIND_ROOT_PDPT:
440	{
441	/* Hopefully this doesn't happen very often:
442	* - touching unused parts of the page
443	* - messing with the bits of pd pointers without changing the physical address
444	*/
445	const unsigned iShw = off / sizeof(X86PDPE);
446	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
447	{
448	if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
449	{
450	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
451	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
452	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
453	}
454	/* paranoia / a bit assumptive. */
455	else if ( pCpu
456	&& (off & 7)
457	&& (off & 7) + pgmPoolDisasWriteSize(pCpu) > sizeof(X86PDPE))
458	{
459	const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PDPE);
460	if ( iShw2 != iShw
461	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES
462	&& uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
463	{
464	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
465	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
466	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
467	}
468	}
469	}
470	break;
471	}
472
473	default:
474	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
475	}
476
477	/* next */
478	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
479	return;
480	pPage = &pPool->aPages[pPage->iMonitoredNext];
481	}
482	}
483
484
485	# ifndef IN_RING3
486	/**
487	* Checks if a access could be a fork operation in progress.
488	*
489	* Meaning, that the guest is setuping up the parent process for Copy-On-Write.
490	*
491	* @returns true if it's likly that we're forking, otherwise false.
492	* @param pPool The pool.
493	* @param pCpu The disassembled instruction.
494	* @param offFault The access offset.
495	*/
496	DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pCpu, unsigned offFault)
497	{
498	/*
499	* i386 linux is using btr to clear X86_PTE_RW.
500	* The functions involved are (2.6.16 source inspection):
501	* clear_bit
502	* ptep_set_wrprotect
503	* copy_one_pte
504	* copy_pte_range
505	* copy_pmd_range
506	* copy_pud_range
507	* copy_page_range
508	* dup_mmap
509	* dup_mm
510	* copy_mm
511	* copy_process
512	* do_fork
513	*/
514	if ( pCpu->pCurInstr->opcode == OP_BTR
515	&& !(offFault & 4)
516	/** @todo Validate that the bit index is X86_PTE_RW. */
517	)
518	{
519	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,Fork));
520	return true;
521	}
522	return false;
523	}
524
525
526	/**
527	* Determin whether the page is likely to have been reused.
528	*
529	* @returns true if we consider the page as being reused for a different purpose.
530	* @returns false if we consider it to still be a paging page.
531	* @param pPage The page in question.
532	* @param pCpu The disassembly info for the faulting insturction.
533	* @param pvFault The fault address.
534	*
535	* @remark The REP prefix check is left to the caller because of STOSD/W.
536	*/
537	DECLINLINE(bool) pgmPoolMonitorIsReused(PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu, RTGCPTR pvFault)
538	{
539	switch (pCpu->pCurInstr->opcode)
540	{
541	case OP_PUSH:
542	Log4(("pgmPoolMonitorIsReused: PUSH\n"));
543	return true;
544	case OP_PUSHF:
545	Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
546	return true;
547	case OP_PUSHA:
548	Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
549	return true;
550	case OP_FXSAVE:
551	Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
552	return true;
553	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
554	Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
555	return true;
556	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
557	Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
558	return true;
559	}
560	if ( (pCpu->param1.flags & USE_REG_GEN32)
561	&& (pCpu->param1.base.reg_gen32 == USE_REG_ESP))
562	{
563	Log4(("pgmPoolMonitorIsReused: ESP\n"));
564	return true;
565	}
566
567	//if (pPage->fCR3Mix)
568	// return false;
569	return false;
570	}
571
572
573	/**
574	* Flushes the page being accessed.
575	*
576	* @returns VBox status code suitable for scheduling.
577	* @param pVM The VM handle.
578	* @param pPool The pool.
579	* @param pPage The pool page (head).
580	* @param pCpu The disassembly of the write instruction.
581	* @param pRegFrame The trap register frame.
582	* @param GCPhysFault The fault address as guest physical address.
583	* @param pvFault The fault address.
584	*/
585	static int pgmPoolAccessHandlerFlush(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
586	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
587	{
588	/*
589	* First, do the flushing.
590	*/
591	int rc = pgmPoolMonitorChainFlush(pPool, pPage);
592
593	/*
594	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
595	*/
596	uint32_t cbWritten;
597	int rc2 = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cbWritten);
598	if (VBOX_SUCCESS(rc2))
599	pRegFrame->eip += pCpu->opsize;
600	else if (rc2 == VERR_EM_INTERPRETER)
601	{
602	#ifdef IN_GC
603	if (PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip))
604	{
605	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
606	pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
607	rc = VINF_SUCCESS;
608	STAM_COUNTER_INC(&pPool->StatMonitorGCIntrFailPatch2);
609	}
610	else
611	#endif
612	{
613	rc = VINF_EM_RAW_EMULATE_INSTR;
614	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,EmulateInstr));
615	}
616	}
617	else
618	rc = rc2;
619
620	/* See use in pgmPoolAccessHandlerSimple(). */
621	PGM_INVL_GUEST_TLBS();
622
623	LogFlow(("pgmPoolAccessHandlerPT: returns %Vrc (flushed)\n", rc));
624	return rc;
625
626	}
627
628
629	/**
630	* Handles the STOSD write accesses.
631	*
632	* @returns VBox status code suitable for scheduling.
633	* @param pVM The VM handle.
634	* @param pPool The pool.
635	* @param pPage The pool page (head).
636	* @param pCpu The disassembly of the write instruction.
637	* @param pRegFrame The trap register frame.
638	* @param GCPhysFault The fault address as guest physical address.
639	* @param pvFault The fault address.
640	*/
641	DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
642	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
643	{
644	/*
645	* Increment the modification counter and insert it into the list
646	* of modified pages the first time.
647	*/
648	if (!pPage->cModifications++)
649	pgmPoolMonitorModifiedInsert(pPool, pPage);
650
651	/*
652	* Execute REP STOSD.
653	*
654	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
655	* write situation, meaning that it's safe to write here.
656	*/
657	#ifdef IN_GC
658	uint32_t pu32 = (uint32_t )pvFault;
659	#else
660	RTGCPTR pu32 = pvFault;
661	#endif
662	while (pRegFrame->ecx)
663	{
664	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pu32, NULL);
665	#ifdef IN_GC
666	*pu32++ = pRegFrame->eax;
667	#else
668	PGMPhysWriteGCPhys(pVM, GCPhysFault, &pRegFrame->eax, 4);
669	pu32 += 4;
670	#endif
671	GCPhysFault += 4;
672	pRegFrame->edi += 4;
673	pRegFrame->ecx--;
674	}
675	pRegFrame->eip += pCpu->opsize;
676
677	/* See use in pgmPoolAccessHandlerSimple(). */
678	PGM_INVL_GUEST_TLBS();
679
680	LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
681	return VINF_SUCCESS;
682	}
683
684
685	/**
686	* Handles the simple write accesses.
687	*
688	* @returns VBox status code suitable for scheduling.
689	* @param pVM The VM handle.
690	* @param pPool The pool.
691	* @param pPage The pool page (head).
692	* @param pCpu The disassembly of the write instruction.
693	* @param pRegFrame The trap register frame.
694	* @param GCPhysFault The fault address as guest physical address.
695	* @param pvFault The fault address.
696	*/
697	DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
698	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
699	{
700	/*
701	* Increment the modification counter and insert it into the list
702	* of modified pages the first time.
703	*/
704	if (!pPage->cModifications++)
705	pgmPoolMonitorModifiedInsert(pPool, pPage);
706
707	/*
708	* Clear all the pages. ASSUMES that pvFault is readable.
709	*/
710	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pvFault, pCpu);
711
712	/*
713	* Interpret the instruction.
714	*/
715	uint32_t cb;
716	int rc = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cb);
717	if (VBOX_SUCCESS(rc))
718	pRegFrame->eip += pCpu->opsize;
719	else if (rc == VERR_EM_INTERPRETER)
720	{
721	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv - opcode=%d\n",
722	pRegFrame->cs, (RTGCPTR)pRegFrame->eip, pCpu->pCurInstr->opcode));
723	rc = VINF_EM_RAW_EMULATE_INSTR;
724	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,EmulateInstr));
725	}
726
727	/*
728	* Quick hack, with logging enabled we're getting stale
729	* code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
730	* Flushing here is BAD and expensive, I think EMInterpretDisasOne will
731	* have to be fixed to support this. But that'll have to wait till next week.
732	*
733	* An alternative is to keep track of the changed PTEs together with the
734	* GCPhys from the guest PT. This may proove expensive though.
735	*
736	* At the moment, it's VITAL that it's done AFTER the instruction interpreting
737	* because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
738	*/
739	PGM_INVL_GUEST_TLBS();
740
741	LogFlow(("pgmPoolAccessHandlerSimple: returns %Vrc cb=%d\n", rc, cb));
742	return rc;
743	}
744
745
746	/**
747	* \#PF Handler callback for PT write accesses.
748	*
749	* @returns VBox status code (appropriate for GC return).
750	* @param pVM VM Handle.
751	* @param uErrorCode CPU Error code.
752	* @param pRegFrame Trap register frame.
753	* NULL on DMA and other non CPU access.
754	* @param pvFault The fault address (cr2).
755	* @param GCPhysFault The GC physical address corresponding to pvFault.
756	* @param pvUser User argument.
757	*/
758	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
759	{
760	STAM_PROFILE_START(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), a);
761	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
762	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
763	LogFlow(("pgmPoolAccessHandler: pvFault=%p pPage=%p:{.idx=%d} GCPhysFault=%VGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
764
765	/*
766	* We should ALWAYS have the list head as user parameter. This
767	* is because we use that page to record the changes.
768	*/
769	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
770
771	/*
772	* Disassemble the faulting instruction.
773	*/
774	DISCPUSTATE Cpu;
775	int rc = EMInterpretDisasOne(pVM, pRegFrame, &Cpu, NULL);
776	AssertRCReturn(rc, rc);
777
778	/*
779	* Check if it's worth dealing with.
780	*/
781	bool fReused = false;
782	if ( ( pPage->cModifications < 48 /** @todo #define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
783	\|\| pPage->fCR3Mix)
784	&& !(fReused = pgmPoolMonitorIsReused(pPage, &Cpu, pvFault))
785	&& !pgmPoolMonitorIsForking(pPool, &Cpu, GCPhysFault & PAGE_OFFSET_MASK))
786	{
787	/*
788	* Simple instructions, no REP prefix.
789	*/
790	if (!(Cpu.prefix & (PREFIX_REP \| PREFIX_REPNE)))
791	{
792	rc = pgmPoolAccessHandlerSimple(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
793	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,Handled), a);
794	return rc;
795	}
796
797	/*
798	* Windows is frequently doing small memset() operations (netio test 4k+).
799	* We have to deal with these or we'll kill the cache and performance.
800	*/
801	if ( Cpu.pCurInstr->opcode == OP_STOSWD
802	&& CPUMGetGuestCPL(pVM, pRegFrame) == 0
803	&& pRegFrame->ecx <= 0x20
804	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
805	&& !((uintptr_t)pvFault & 3)
806	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
807	&& Cpu.mode == CPUMODE_32BIT
808	&& Cpu.opmode == CPUMODE_32BIT
809	&& Cpu.addrmode == CPUMODE_32BIT
810	&& Cpu.prefix == PREFIX_REP
811	&& !pRegFrame->eflags.Bits.u1DF
812	)
813	{
814	rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
815	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,RepStosd), a);
816	return rc;
817	}
818
819	/* REP prefix, don't bother. */
820	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,RepPrefix));
821	Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x eip=%#x opcode=%d prefix=%#x\n",
822	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, pRegFrame->eip, Cpu.pCurInstr->opcode, Cpu.prefix));
823	}
824
825	/*
826	* Not worth it, so flush it.
827	*
828	* If we considered it to be reused, don't to back to ring-3
829	* to emulate failed instructions since we usually cannot
830	* interpret then. This may be a bit risky, in which case
831	* the reuse detection must be fixed.
832	*/
833	rc = pgmPoolAccessHandlerFlush(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
834	if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
835	rc = VINF_SUCCESS;
836	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,FlushPage), a);
837	return rc;
838	}
839
840	# endif /* !IN_RING3 */
841	#endif /* PGMPOOL_WITH_MONITORING */
842
843
844
845	#ifdef PGMPOOL_WITH_CACHE
846	/**
847	* Inserts a page into the GCPhys hash table.
848	*
849	* @param pPool The pool.
850	* @param pPage The page.
851	*/
852	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
853	{
854	Log3(("pgmPoolHashInsert: %VGp\n", pPage->GCPhys));
855	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
856	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
857	pPage->iNext = pPool->aiHash[iHash];
858	pPool->aiHash[iHash] = pPage->idx;
859	}
860
861
862	/**
863	* Removes a page from the GCPhys hash table.
864	*
865	* @param pPool The pool.
866	* @param pPage The page.
867	*/
868	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
869	{
870	Log3(("pgmPoolHashRemove: %VGp\n", pPage->GCPhys));
871	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
872	if (pPool->aiHash[iHash] == pPage->idx)
873	pPool->aiHash[iHash] = pPage->iNext;
874	else
875	{
876	uint16_t iPrev = pPool->aiHash[iHash];
877	for (;;)
878	{
879	const int16_t i = pPool->aPages[iPrev].iNext;
880	if (i == pPage->idx)
881	{
882	pPool->aPages[iPrev].iNext = pPage->iNext;
883	break;
884	}
885	if (i == NIL_PGMPOOL_IDX)
886	{
887	AssertReleaseMsgFailed(("GCPhys=%VGp idx=%#x\n", pPage->GCPhys, pPage->idx));
888	break;
889	}
890	iPrev = i;
891	}
892	}
893	pPage->iNext = NIL_PGMPOOL_IDX;
894	}
895
896
897	/**
898	* Frees up one cache page.
899	*
900	* @returns VBox status code.
901	* @retval VINF_SUCCESS on success.
902	* @retval VERR_PGM_POOL_CLEARED if the deregistration of a physical handler will cause a light weight pool flush.
903	* @param pPool The pool.
904	* @param iUser The user index.
905	*/
906	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
907	{
908	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
909	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
910
911	/*
912	* Select one page from the tail of the age list.
913	*/
914	uint16_t iToFree = pPool->iAgeTail;
915	if (iToFree == iUser)
916	iToFree = pPool->aPages[iToFree].iAgePrev;
917	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
918	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
919	{
920	uint16_t i = pPool->aPages[iToFree].iAgePrev;
921	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
922	{
923	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
924	continue;
925	iToFree = i;
926	break;
927	}
928	}
929	*/
930	Assert(iToFree != iUser);
931	AssertRelease(iToFree != NIL_PGMPOOL_IDX);
932
933	int rc = pgmPoolFlushPage(pPool, &pPool->aPages[iToFree]);
934	if (rc == VINF_SUCCESS)
935	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
936	return rc;
937	}
938
939
940	/**
941	* Checks if a kind mismatch is really a page being reused
942	* or if it's just normal remappings.
943	*
944	* @returns true if reused and the cached page (enmKind1) should be flushed
945	* @returns false if not reused.
946	* @param enmKind1 The kind of the cached page.
947	* @param enmKind2 The kind of the requested page.
948	*/
949	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
950	{
951	switch (enmKind1)
952	{
953	/*
954	* Never reuse them. There is no remapping in non-paging mode.
955	*/
956	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
957	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
958	return true;
959
960	/*
961	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
962	*/
963	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
964	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
965	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
966	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
967	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
968	switch (enmKind2)
969	{
970	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
971	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
972	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
973	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
974	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
975	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
976	return true;
977	default:
978	return false;
979	}
980
981	/*
982	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
983	*/
984	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
985	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
986	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
987	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
988	switch (enmKind2)
989	{
990	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
991	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
992	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
993	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
994	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
995	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
996	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
997	return true;
998	default:
999	return false;
1000	}
1001
1002	/*
1003	* These cannot be flushed, and it's common to reuse the PDs as PTs.
1004	*/
1005	case PGMPOOLKIND_ROOT_32BIT_PD:
1006	case PGMPOOLKIND_ROOT_PAE_PD:
1007	case PGMPOOLKIND_ROOT_PDPT:
1008	case PGMPOOLKIND_ROOT_PML4:
1009	return false;
1010
1011	default:
1012	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1013	}
1014	}
1015
1016
1017	/**
1018	* Attempts to satisfy a pgmPoolAlloc request from the cache.
1019	*
1020	* @returns VBox status code.
1021	* @retval VINF_PGM_CACHED_PAGE on success.
1022	* @retval VERR_FILE_NOT_FOUND if not found.
1023	* @param pPool The pool.
1024	* @param GCPhys The GC physical address of the page we're gonna shadow.
1025	* @param enmKind The kind of mapping.
1026	* @param iUser The shadow page pool index of the user table.
1027	* @param iUserTable The index into the user table (shadowed).
1028	* @param ppPage Where to store the pointer to the page.
1029	*/
1030	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint16_t iUserTable, PPPGMPOOLPAGE ppPage)
1031	{
1032	/*
1033	* Look up the GCPhys in the hash.
1034	*/
1035	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1036	Log3(("pgmPoolCacheAlloc: %VGp kind %d iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, enmKind, iUser, iUserTable, i));
1037	if (i != NIL_PGMPOOL_IDX)
1038	{
1039	do
1040	{
1041	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1042	Log3(("pgmPoolCacheAlloc: slot %d found page %VGp\n", i, pPage->GCPhys));
1043	if (pPage->GCPhys == GCPhys)
1044	{
1045	if ((PGMPOOLKIND)pPage->enmKind == enmKind)
1046	{
1047	int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1048	if (VBOX_SUCCESS(rc))
1049	{
1050	*ppPage = pPage;
1051	STAM_COUNTER_INC(&pPool->StatCacheHits);
1052	return VINF_PGM_CACHED_PAGE;
1053	}
1054	return rc;
1055	}
1056
1057	/*
1058	* The kind is different. In some cases we should now flush the page
1059	* as it has been reused, but in most cases this is normal remapping
1060	* of PDs as PT or big pages using the GCPhys field in a slightly
1061	* different way than the other kinds.
1062	*/
1063	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1064	{
1065	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1066	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED will be returned by pgmPoolTracInsert. */
1067	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
1068	break;
1069	}
1070	}
1071
1072	/* next */
1073	i = pPage->iNext;
1074	} while (i != NIL_PGMPOOL_IDX);
1075	}
1076
1077	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%d\n", GCPhys, enmKind));
1078	STAM_COUNTER_INC(&pPool->StatCacheMisses);
1079	return VERR_FILE_NOT_FOUND;
1080	}
1081
1082
1083	/**
1084	* Inserts a page into the cache.
1085	*
1086	* @param pPool The pool.
1087	* @param pPage The cached page.
1088	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1089	*/
1090	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1091	{
1092	/*
1093	* Insert into the GCPhys hash if the page is fit for that.
1094	*/
1095	Assert(!pPage->fCached);
1096	if (fCanBeCached)
1097	{
1098	pPage->fCached = true;
1099	pgmPoolHashInsert(pPool, pPage);
1100	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1101	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1102	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1103	}
1104	else
1105	{
1106	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1107	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1108	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1109	}
1110
1111	/*
1112	* Insert at the head of the age list.
1113	*/
1114	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1115	pPage->iAgeNext = pPool->iAgeHead;
1116	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1117	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1118	else
1119	pPool->iAgeTail = pPage->idx;
1120	pPool->iAgeHead = pPage->idx;
1121	}
1122
1123
1124	/**
1125	* Flushes a cached page.
1126	*
1127	* @param pPool The pool.
1128	* @param pPage The cached page.
1129	*/
1130	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1131	{
1132	Log3(("pgmPoolCacheFlushPage: %VGp\n", pPage->GCPhys));
1133
1134	/*
1135	* Remove the page from the hash.
1136	*/
1137	if (pPage->fCached)
1138	{
1139	pPage->fCached = false;
1140	pgmPoolHashRemove(pPool, pPage);
1141	}
1142	else
1143	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1144
1145	/*
1146	* Remove it from the age list.
1147	*/
1148	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1149	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1150	else
1151	pPool->iAgeTail = pPage->iAgePrev;
1152	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1153	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1154	else
1155	pPool->iAgeHead = pPage->iAgeNext;
1156	pPage->iAgeNext = NIL_PGMPOOL_IDX;
1157	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1158	}
1159	#endif /* PGMPOOL_WITH_CACHE */
1160
1161
1162	#ifdef PGMPOOL_WITH_MONITORING
1163	/**
1164	* Looks for pages sharing the monitor.
1165	*
1166	* @returns Pointer to the head page.
1167	* @returns NULL if not found.
1168	* @param pPool The Pool
1169	* @param pNewPage The page which is going to be monitored.
1170	*/
1171	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1172	{
1173	#ifdef PGMPOOL_WITH_CACHE
1174	/*
1175	* Look up the GCPhys in the hash.
1176	*/
1177	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1178	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1179	if (i == NIL_PGMPOOL_IDX)
1180	return NULL;
1181	do
1182	{
1183	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1184	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1185	&& pPage != pNewPage)
1186	{
1187	switch (pPage->enmKind)
1188	{
1189	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1190	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1191	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1192	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1193	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1194	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1195	case PGMPOOLKIND_ROOT_32BIT_PD:
1196	case PGMPOOLKIND_ROOT_PAE_PD:
1197	case PGMPOOLKIND_ROOT_PDPT:
1198	case PGMPOOLKIND_ROOT_PML4:
1199	{
1200	/* find the head */
1201	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1202	{
1203	Assert(pPage->iMonitoredPrev != pPage->idx);
1204	pPage = &pPool->aPages[pPage->iMonitoredPrev];
1205	}
1206	return pPage;
1207	}
1208
1209	/* ignore, no monitoring. */
1210	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1211	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1212	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1213	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1214	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1215	break;
1216	default:
1217	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
1218	}
1219	}
1220
1221	/* next */
1222	i = pPage->iNext;
1223	} while (i != NIL_PGMPOOL_IDX);
1224	#endif
1225	return NULL;
1226	}
1227
1228	/**
1229	* Enabled write monitoring of a guest page.
1230	*
1231	* @returns VBox status code.
1232	* @retval VINF_SUCCESS on success.
1233	* @retval VERR_PGM_POOL_CLEARED if the registration of the physical handler will cause a light weight pool flush.
1234	* @param pPool The pool.
1235	* @param pPage The cached page.
1236	*/
1237	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1238	{
1239	LogFlow(("pgmPoolMonitorInsert %VGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
1240
1241	/*
1242	* Filter out the relevant kinds.
1243	*/
1244	switch (pPage->enmKind)
1245	{
1246	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1247	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1248	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1249	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1250	case PGMPOOLKIND_ROOT_PDPT:
1251	break;
1252
1253	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1254	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1255	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1256	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1257	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1258	/* Nothing to monitor here. */
1259	return VINF_SUCCESS;
1260
1261	case PGMPOOLKIND_ROOT_32BIT_PD:
1262	case PGMPOOLKIND_ROOT_PAE_PD:
1263	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1264	break;
1265	#endif
1266	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1267	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1268	case PGMPOOLKIND_ROOT_PML4:
1269	default:
1270	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1271	}
1272
1273	/*
1274	* Install handler.
1275	*/
1276	int rc;
1277	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
1278	if (pPageHead)
1279	{
1280	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
1281	Assert(pPageHead->iMonitoredPrev != pPage->idx);
1282	pPage->iMonitoredPrev = pPageHead->idx;
1283	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
1284	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
1285	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
1286	pPageHead->iMonitoredNext = pPage->idx;
1287	rc = VINF_SUCCESS;
1288	}
1289	else
1290	{
1291	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1292	PVM pVM = pPool->CTXSUFF(pVM);
1293	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1294	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
1295	GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
1296	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
1297	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
1298	pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pPage),
1299	pPool->pszAccessHandler);
1300	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
1301	* the heap size should suffice. */
1302	AssertFatalRC(rc);
1303	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1304	rc = VERR_PGM_POOL_CLEARED;
1305	}
1306	pPage->fMonitored = true;
1307	return rc;
1308	}
1309
1310
1311	/**
1312	* Disables write monitoring of a guest page.
1313	*
1314	* @returns VBox status code.
1315	* @retval VINF_SUCCESS on success.
1316	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1317	* @param pPool The pool.
1318	* @param pPage The cached page.
1319	*/
1320	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1321	{
1322	/*
1323	* Filter out the relevant kinds.
1324	*/
1325	switch (pPage->enmKind)
1326	{
1327	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1328	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1329	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1330	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1331	case PGMPOOLKIND_ROOT_PDPT:
1332	break;
1333
1334	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1335	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1336	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1337	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1338	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1339	/* Nothing to monitor here. */
1340	return VINF_SUCCESS;
1341
1342	case PGMPOOLKIND_ROOT_32BIT_PD:
1343	case PGMPOOLKIND_ROOT_PAE_PD:
1344	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1345	break;
1346	#endif
1347	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1348	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1349	case PGMPOOLKIND_ROOT_PML4:
1350	default:
1351	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1352	}
1353
1354	/*
1355	* Remove the page from the monitored list or uninstall it if last.
1356	*/
1357	const PVM pVM = pPool->CTXSUFF(pVM);
1358	int rc;
1359	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1360	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1361	{
1362	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1363	{
1364	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
1365	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
1366	pNewHead->fCR3Mix = pPage->fCR3Mix;
1367	rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
1368	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
1369	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
1370	pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pNewHead),
1371	pPool->pszAccessHandler);
1372	AssertFatalRCSuccess(rc);
1373	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1374	}
1375	else
1376	{
1377	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
1378	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
1379	{
1380	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
1381	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1382	}
1383	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
1384	rc = VINF_SUCCESS;
1385	}
1386	}
1387	else
1388	{
1389	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
1390	AssertFatalRC(rc);
1391	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1392	rc = VERR_PGM_POOL_CLEARED;
1393	}
1394	pPage->fMonitored = false;
1395
1396	/*
1397	* Remove it from the list of modified pages (if in it).
1398	*/
1399	pgmPoolMonitorModifiedRemove(pPool, pPage);
1400
1401	return rc;
1402	}
1403
1404
1405	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1406	/**
1407	* Set or clear the fCR3Mix attribute in a chain of monitored pages.
1408	*
1409	* @param pPool The Pool.
1410	* @param pPage A page in the chain.
1411	* @param fCR3Mix The new fCR3Mix value.
1412	*/
1413	static void pgmPoolMonitorChainChangeCR3Mix(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCR3Mix)
1414	{
1415	/* current */
1416	pPage->fCR3Mix = fCR3Mix;
1417
1418	/* before */
1419	int16_t idx = pPage->iMonitoredPrev;
1420	while (idx != NIL_PGMPOOL_IDX)
1421	{
1422	pPool->aPages[idx].fCR3Mix = fCR3Mix;
1423	idx = pPool->aPages[idx].iMonitoredPrev;
1424	}
1425
1426	/* after */
1427	idx = pPage->iMonitoredNext;
1428	while (idx != NIL_PGMPOOL_IDX)
1429	{
1430	pPool->aPages[idx].fCR3Mix = fCR3Mix;
1431	idx = pPool->aPages[idx].iMonitoredNext;
1432	}
1433	}
1434
1435
1436	/**
1437	* Installs or modifies monitoring of a CR3 page (special).
1438	*
1439	* We're pretending the CR3 page is shadowed by the pool so we can use the
1440	* generic mechanisms in detecting chained monitoring. (This also gives us a
1441	* tast of what code changes are required to really pool CR3 shadow pages.)
1442	*
1443	* @returns VBox status code.
1444	* @param pPool The pool.
1445	* @param idxRoot The CR3 (root) page index.
1446	* @param GCPhysCR3 The (new) CR3 value.
1447	*/
1448	int pgmPoolMonitorMonitorCR3(PPGMPOOL pPool, uint16_t idxRoot, RTGCPHYS GCPhysCR3)
1449	{
1450	Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
1451	PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
1452	LogFlow(("pgmPoolMonitorMonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%VGp, .fMonitored=%d} GCPhysCR3=%VGp\n",
1453	idxRoot, pPage, pPage->GCPhys, pPage->fMonitored, GCPhysCR3));
1454
1455	/*
1456	* The unlikely case where it already matches.
1457	*/
1458	if (pPage->GCPhys == GCPhysCR3)
1459	{
1460	Assert(pPage->fMonitored);
1461	return VINF_SUCCESS;
1462	}
1463
1464	/*
1465	* Flush the current monitoring and remove it from the hash.
1466	*/
1467	int rc = VINF_SUCCESS;
1468	if (pPage->fMonitored)
1469	{
1470	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
1471	rc = pgmPoolMonitorFlush(pPool, pPage);
1472	if (rc == VERR_PGM_POOL_CLEARED)
1473	rc = VINF_SUCCESS;
1474	else
1475	AssertFatalRC(rc);
1476	pgmPoolHashRemove(pPool, pPage);
1477	}
1478
1479	/*
1480	* Monitor the page at the new location and insert it into the hash.
1481	*/
1482	pPage->GCPhys = GCPhysCR3;
1483	int rc2 = pgmPoolMonitorInsert(pPool, pPage);
1484	if (rc2 != VERR_PGM_POOL_CLEARED)
1485	{
1486	AssertFatalRC(rc2);
1487	if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS)
1488	rc = rc2;
1489	}
1490	pgmPoolHashInsert(pPool, pPage);
1491	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, true);
1492	return rc;
1493	}
1494
1495
1496	/**
1497	* Removes the monitoring of a CR3 page (special).
1498	*
1499	* @returns VBox status code.
1500	* @param pPool The pool.
1501	* @param idxRoot The CR3 (root) page index.
1502	*/
1503	int pgmPoolMonitorUnmonitorCR3(PPGMPOOL pPool, uint16_t idxRoot)
1504	{
1505	Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
1506	PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
1507	LogFlow(("pgmPoolMonitorUnmonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%VGp, .fMonitored=%d}\n",
1508	idxRoot, pPage, pPage->GCPhys, pPage->fMonitored));
1509
1510	if (!pPage->fMonitored)
1511	return VINF_SUCCESS;
1512
1513	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
1514	int rc = pgmPoolMonitorFlush(pPool, pPage);
1515	if (rc != VERR_PGM_POOL_CLEARED)
1516	AssertFatalRC(rc);
1517	else
1518	rc = VINF_SUCCESS;
1519	pgmPoolHashRemove(pPool, pPage);
1520	Assert(!pPage->fMonitored);
1521	pPage->GCPhys = NIL_RTGCPHYS;
1522	return rc;
1523	}
1524	#endif /* PGMPOOL_WITH_MIXED_PT_CR3 */
1525
1526
1527	/**
1528	* Inserts the page into the list of modified pages.
1529	*
1530	* @param pPool The pool.
1531	* @param pPage The page.
1532	*/
1533	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1534	{
1535	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
1536	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
1537	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
1538	&& pPool->iModifiedHead != pPage->idx,
1539	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
1540	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
1541	pPool->iModifiedHead, pPool->cModifiedPages));
1542
1543	pPage->iModifiedNext = pPool->iModifiedHead;
1544	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
1545	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
1546	pPool->iModifiedHead = pPage->idx;
1547	pPool->cModifiedPages++;
1548	#ifdef VBOX_WITH_STATISTICS
1549	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
1550	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
1551	#endif
1552	}
1553
1554
1555	/**
1556	* Removes the page from the list of modified pages and resets the
1557	* moficiation counter.
1558	*
1559	* @param pPool The pool.
1560	* @param pPage The page which is believed to be in the list of modified pages.
1561	*/
1562	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1563	{
1564	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
1565	if (pPool->iModifiedHead == pPage->idx)
1566	{
1567	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1568	pPool->iModifiedHead = pPage->iModifiedNext;
1569	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1570	{
1571	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
1572	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1573	}
1574	pPool->cModifiedPages--;
1575	}
1576	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
1577	{
1578	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
1579	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1580	{
1581	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
1582	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1583	}
1584	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1585	pPool->cModifiedPages--;
1586	}
1587	else
1588	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1589	pPage->cModifications = 0;
1590	}
1591
1592
1593	/**
1594	* Zaps the list of modified pages, resetting their modification counters in the process.
1595	*
1596	* @param pVM The VM handle.
1597	*/
1598	void pgmPoolMonitorModifiedClearAll(PVM pVM)
1599	{
1600	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1601	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
1602
1603	unsigned cPages = 0; NOREF(cPages);
1604	uint16_t idx = pPool->iModifiedHead;
1605	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1606	while (idx != NIL_PGMPOOL_IDX)
1607	{
1608	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
1609	idx = pPage->iModifiedNext;
1610	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1611	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1612	pPage->cModifications = 0;
1613	Assert(++cPages);
1614	}
1615	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
1616	pPool->cModifiedPages = 0;
1617	}
1618
1619
1620	/**
1621	* Clear all shadow pages and clear all modification counters.
1622	*
1623	* @param pVM The VM handle.
1624	* @remark Should only be used when monitoring is available, thus placed in
1625	* the PGMPOOL_WITH_MONITORING #ifdef.
1626	*/
1627	void pgmPoolClearAll(PVM pVM)
1628	{
1629	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1630	STAM_PROFILE_START(&pPool->StatClearAll, c);
1631	LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
1632
1633	/*
1634	* Iterate all the pages until we've encountered all that in use.
1635	* This is simple but not quite optimal solution.
1636	*/
1637	unsigned cModifiedPages = 0; NOREF(cModifiedPages);
1638	unsigned cLeft = pPool->cUsedPages;
1639	unsigned iPage = pPool->cCurPages;
1640	while (--iPage >= PGMPOOL_IDX_FIRST)
1641	{
1642	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1643	if (pPage->GCPhys != NIL_RTGCPHYS)
1644	{
1645	switch (pPage->enmKind)
1646	{
1647	/*
1648	* We only care about shadow page tables.
1649	*/
1650	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1651	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1652	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1653	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1654	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1655	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1656	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1657	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1658	{
1659	#ifdef PGMPOOL_WITH_USER_TRACKING
1660	if (pPage->cPresent)
1661	#endif
1662	{
1663	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
1664	STAM_PROFILE_START(&pPool->StatZeroPage, z);
1665	ASMMemZeroPage(pvShw);
1666	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
1667	#ifdef PGMPOOL_WITH_USER_TRACKING
1668	pPage->cPresent = 0;
1669	pPage->iFirstPresent = ~0;
1670	#endif
1671	}
1672	}
1673	/* fall thru */
1674
1675	default:
1676	Assert(!pPage->cModifications \|\| ++cModifiedPages);
1677	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1678	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1679	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1680	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1681	pPage->cModifications = 0;
1682	break;
1683
1684	}
1685	if (!--cLeft)
1686	break;
1687	}
1688	}
1689
1690	/* swipe the special pages too. */
1691	for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
1692	{
1693	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1694	if (pPage->GCPhys != NIL_RTGCPHYS)
1695	{
1696	Assert(!pPage->cModifications \|\| ++cModifiedPages);
1697	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1698	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1699	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1700	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1701	pPage->cModifications = 0;
1702	}
1703	}
1704
1705	AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
1706	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1707	pPool->cModifiedPages = 0;
1708
1709	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1710	/*
1711	* Clear all the GCPhys links and rebuild the phys ext free list.
1712	*/
1713	for (PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
1714	pRam;
1715	pRam = CTXALLSUFF(pRam->pNext))
1716	{
1717	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1718	while (iPage-- > 0)
1719	pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
1720	}
1721
1722	pPool->iPhysExtFreeHead = 0;
1723	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
1724	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
1725	for (unsigned i = 0; i < cMaxPhysExts; i++)
1726	{
1727	paPhysExts[i].iNext = i + 1;
1728	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
1729	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
1730	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
1731	}
1732	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
1733	#endif
1734
1735
1736	pPool->cPresent = 0;
1737	STAM_PROFILE_STOP(&pPool->StatClearAll, c);
1738	}
1739	#endif /* PGMPOOL_WITH_MONITORING */
1740
1741
1742	#ifdef PGMPOOL_WITH_USER_TRACKING
1743	/**
1744	* Frees up at least one user entry.
1745	*
1746	* @returns VBox status code.
1747	* @retval VINF_SUCCESS if successfully added.
1748	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1749	* @param pPool The pool.
1750	* @param iUser The user index.
1751	*/
1752	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
1753	{
1754	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
1755	#ifdef PGMPOOL_WITH_CACHE
1756	/*
1757	* Just free cached pages in a braindead fashion.
1758	*/
1759	/** @todo walk the age list backwards and free the first with usage. */
1760	int rc = VINF_SUCCESS;
1761	do
1762	{
1763	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
1764	if (VBOX_FAILURE(rc2) && rc == VINF_SUCCESS)
1765	rc = rc2;
1766	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
1767	return rc;
1768	#else
1769	/*
1770	* Lazy approach.
1771	*/
1772	pgmPoolFlushAllInt(pPool);
1773	return VERR_PGM_POOL_FLUSHED;
1774	#endif
1775	}
1776
1777
1778	/**
1779	* Inserts a page into the cache.
1780	*
1781	* This will create user node for the page, insert it into the GCPhys
1782	* hash, and insert it into the age list.
1783	*
1784	* @returns VBox status code.
1785	* @retval VINF_SUCCESS if successfully added.
1786	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1787	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1788	* @param pPool The pool.
1789	* @param pPage The cached page.
1790	* @param GCPhys The GC physical address of the page we're gonna shadow.
1791	* @param iUser The user index.
1792	* @param iUserTable The user table index.
1793	*/
1794	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint16_t iUserTable)
1795	{
1796	int rc = VINF_SUCCESS;
1797	PPGMPOOLUSER pUser = pPool->CTXSUFF(paUsers);
1798
1799	LogFlow(("pgmPoolTrackInsert iUser %d iUserTable %d\n", iUser, iUserTable));
1800
1801	/*
1802	* Find free a user node.
1803	*/
1804	uint16_t i = pPool->iUserFreeHead;
1805	if (i == NIL_PGMPOOL_USER_INDEX)
1806	{
1807	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
1808	if (VBOX_FAILURE(rc))
1809	return rc;
1810	i = pPool->iUserFreeHead;
1811	}
1812
1813	/*
1814	* Unlink the user node from the free list,
1815	* initialize and insert it into the user list.
1816	*/
1817	pPool->iUserFreeHead = pUser[i].iNext;
1818	pUser[i].iNext = NIL_PGMPOOL_USER_INDEX;
1819	pUser[i].iUser = iUser;
1820	pUser[i].iUserTable = iUserTable;
1821	pPage->iUserHead = i;
1822
1823	/*
1824	* Insert into cache and enable monitoring of the guest page if enabled.
1825	*
1826	* Until we implement caching of all levels, including the CR3 one, we'll
1827	* have to make sure we don't try monitor & cache any recursive reuse of
1828	* a monitored CR3 page. Because all windows versions are doing this we'll
1829	* have to be able to do combined access monitoring, CR3 + PT and
1830	* PD + PT (guest PAE).
1831	*
1832	* Update:
1833	* We're now cooperating with the CR3 monitor if an uncachable page is found.
1834	*/
1835	#if defined(PGMPOOL_WITH_MONITORING) \|\| defined(PGMPOOL_WITH_CACHE)
1836	# ifdef PGMPOOL_WITH_MIXED_PT_CR3
1837	const bool fCanBeMonitored = true;
1838	# else
1839	bool fCanBeMonitored = pPool->CTXSUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
1840	\|\| (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTXSUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
1841	\|\| pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
1842	# endif
1843	# ifdef PGMPOOL_WITH_CACHE
1844	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
1845	# endif
1846	if (fCanBeMonitored)
1847	{
1848	# ifdef PGMPOOL_WITH_MONITORING
1849	rc = pgmPoolMonitorInsert(pPool, pPage);
1850	if (rc == VERR_PGM_POOL_CLEARED)
1851	{
1852	/* 'Failed' - free the usage, and keep it in the cache (if enabled). */
1853	# ifndef PGMPOOL_WITH_CACHE
1854	pgmPoolMonitorFlush(pPool, pPage);
1855	rc = VERR_PGM_POOL_FLUSHED;
1856	# endif
1857	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
1858	pUser[i].iNext = pPool->iUserFreeHead;
1859	pUser[i].iUser = NIL_PGMPOOL_IDX;
1860	pPool->iUserFreeHead = i;
1861	}
1862	}
1863	# endif
1864	#endif /* PGMPOOL_WITH_MONITORING */
1865	return rc;
1866	}
1867
1868
1869	# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
1870	/**
1871	* Adds a user reference to a page.
1872	*
1873	* This will
1874	* This will move the page to the head of the
1875	*
1876	* @returns VBox status code.
1877	* @retval VINF_SUCCESS if successfully added.
1878	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1879	* @param pPool The pool.
1880	* @param pPage The cached page.
1881	* @param iUser The user index.
1882	* @param iUserTable The user table.
1883	*/
1884	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
1885	{
1886	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
1887
1888	LogFlow(("pgmPoolTrackAddUser iUser %d iUserTable %d\n", iUser, iUserTable));
1889	# ifdef VBOX_STRICT
1890	/*
1891	* Check that the entry doesn't already exists.
1892	*/
1893	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
1894	{
1895	uint16_t i = pPage->iUserHead;
1896	do
1897	{
1898	Assert(i < pPool->cMaxUsers);
1899	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
1900	i = paUsers[i].iNext;
1901	} while (i != NIL_PGMPOOL_USER_INDEX);
1902	}
1903	# endif
1904
1905	/*
1906	* Allocate a user node.
1907	*/
1908	uint16_t i = pPool->iUserFreeHead;
1909	if (i == NIL_PGMPOOL_USER_INDEX)
1910	{
1911	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
1912	if (VBOX_FAILURE(rc))
1913	return rc;
1914	i = pPool->iUserFreeHead;
1915	}
1916	pPool->iUserFreeHead = paUsers[i].iNext;
1917
1918	/*
1919	* Initialize the user node and insert it.
1920	*/
1921	paUsers[i].iNext = pPage->iUserHead;
1922	paUsers[i].iUser = iUser;
1923	paUsers[i].iUserTable = iUserTable;
1924	pPage->iUserHead = i;
1925
1926	# ifdef PGMPOOL_WITH_CACHE
1927	/*
1928	* Tell the cache to update its replacement stats for this page.
1929	*/
1930	pgmPoolCacheUsed(pPool, pPage);
1931	# endif
1932	return VINF_SUCCESS;
1933	}
1934	# endif /* PGMPOOL_WITH_CACHE */
1935
1936
1937	/**
1938	* Frees a user record associated with a page.
1939	*
1940	* This does not clear the entry in the user table, it simply replaces the
1941	* user record to the chain of free records.
1942	*
1943	* @param pPool The pool.
1944	* @param HCPhys The HC physical address of the shadow page.
1945	* @param iUser The shadow page pool index of the user table.
1946	* @param iUserTable The index into the user table (shadowed).
1947	*/
1948	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
1949	{
1950	/*
1951	* Unlink and free the specified user entry.
1952	*/
1953	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
1954
1955	/* Special: For PAE and 32-bit paging, there are usually no more than one user. */
1956	uint16_t i = pPage->iUserHead;
1957	if ( i != NIL_PGMPOOL_USER_INDEX
1958	&& paUsers[i].iUser == iUser
1959	&& paUsers[i].iUserTable == iUserTable)
1960	{
1961	pPage->iUserHead = paUsers[i].iNext;
1962
1963	paUsers[i].iUser = NIL_PGMPOOL_IDX;
1964	paUsers[i].iNext = pPool->iUserFreeHead;
1965	pPool->iUserFreeHead = i;
1966	return;
1967	}
1968
1969	/* General: Linear search. */
1970	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
1971	while (i != NIL_PGMPOOL_USER_INDEX)
1972	{
1973	if ( paUsers[i].iUser == iUser
1974	&& paUsers[i].iUserTable == iUserTable)
1975	{
1976	if (iPrev != NIL_PGMPOOL_USER_INDEX)
1977	paUsers[iPrev].iNext = paUsers[i].iNext;
1978	else
1979	pPage->iUserHead = paUsers[i].iNext;
1980
1981	paUsers[i].iUser = NIL_PGMPOOL_IDX;
1982	paUsers[i].iNext = pPool->iUserFreeHead;
1983	pPool->iUserFreeHead = i;
1984	return;
1985	}
1986	iPrev = i;
1987	i = paUsers[i].iNext;
1988	}
1989
1990	/* Fatal: didn't find it */
1991	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%VGp\n",
1992	iUser, iUserTable, pPage->GCPhys));
1993	}
1994
1995
1996	/**
1997	* Gets the entry size of a shadow table.
1998	*
1999	* @param enmKind The kind of page.
2000	*
2001	* @returns The size of the entry in bytes. That is, 4 or 8.
2002	* @returns If the kind is not for a table, an assertion is raised and 0 is
2003	* returned.
2004	*/
2005	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2006	{
2007	switch (enmKind)
2008	{
2009	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2010	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2011	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2012	case PGMPOOLKIND_ROOT_32BIT_PD:
2013	return 4;
2014
2015	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2016	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2017	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2018	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2019	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2020	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2021	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2022	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2023	case PGMPOOLKIND_ROOT_PAE_PD:
2024	case PGMPOOLKIND_ROOT_PDPT:
2025	case PGMPOOLKIND_ROOT_PML4:
2026	return 8;
2027
2028	default:
2029	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2030	}
2031	}
2032
2033
2034	/**
2035	* Gets the entry size of a guest table.
2036	*
2037	* @param enmKind The kind of page.
2038	*
2039	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
2040	* @returns If the kind is not for a table, an assertion is raised and 0 is
2041	* returned.
2042	*/
2043	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2044	{
2045	switch (enmKind)
2046	{
2047	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2048	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2049	case PGMPOOLKIND_ROOT_32BIT_PD:
2050	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2051	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2052	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2053	return 4;
2054
2055	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2056	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2057	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2058	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2059	case PGMPOOLKIND_ROOT_PAE_PD:
2060	case PGMPOOLKIND_ROOT_PDPT:
2061	case PGMPOOLKIND_ROOT_PML4:
2062	return 8;
2063
2064	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2065	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2066	/** @todo can we return 0? (nobody is calling this...) */
2067	return 0;
2068
2069	default:
2070	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2071	}
2072	}
2073
2074
2075	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2076	/**
2077	* Scans one shadow page table for mappings of a physical page.
2078	*
2079	* @param pVM The VM handle.
2080	* @param pPhysPage The guest page in question.
2081	* @param iShw The shadow page table.
2082	* @param cRefs The number of references made in that PT.
2083	*/
2084	static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2085	{
2086	LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2087	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2088
2089	/*
2090	* Assert sanity.
2091	*/
2092	Assert(cRefs == 1);
2093	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2094	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2095
2096	/*
2097	* Then, clear the actual mappings to the page in the shadow PT.
2098	*/
2099	switch (pPage->enmKind)
2100	{
2101	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2102	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2103	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2104	{
2105	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2106	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2107	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2108	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2109	{
2110	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2111	pPT->a[i].u = 0;
2112	cRefs--;
2113	if (!cRefs)
2114	return;
2115	}
2116	#if defined(DEBUG) && !defined(IN_RING0) ///@todo RTLogPrintf is missing in R0.
2117	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2118	for (unsigned i = 0; i < ELEMENTS(pPT->a); i++)
2119	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2120	{
2121	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2122	pPT->a[i].u = 0;
2123	}
2124	#endif
2125	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2126	break;
2127	}
2128
2129	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2130	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2131	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2132	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2133	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2134	{
2135	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2136	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2137	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2138	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2139	{
2140	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2141	pPT->a[i].u = 0;
2142	cRefs--;
2143	if (!cRefs)
2144	return;
2145	}
2146	#if defined(DEBUG) && !defined(IN_RING0) ///@todo RTLogPrintf is missing in R0.
2147	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2148	for (unsigned i = 0; i < ELEMENTS(pPT->a); i++)
2149	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2150	{
2151	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2152	pPT->a[i].u = 0;
2153	}
2154	#endif
2155	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2156	break;
2157	}
2158
2159	default:
2160	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
2161	}
2162	}
2163
2164
2165	/**
2166	* Scans one shadow page table for mappings of a physical page.
2167	*
2168	* @param pVM The VM handle.
2169	* @param pPhysPage The guest page in question.
2170	* @param iShw The shadow page table.
2171	* @param cRefs The number of references made in that PT.
2172	*/
2173	void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2174	{
2175	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool); NOREF(pPool);
2176	LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2177	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
2178	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
2179	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2180	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
2181	}
2182
2183
2184	/**
2185	* Flushes a list of shadow page tables mapping the same physical page.
2186	*
2187	* @param pVM The VM handle.
2188	* @param pPhysPage The guest page in question.
2189	* @param iPhysExt The physical cross reference extent list to flush.
2190	*/
2191	void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
2192	{
2193	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2194	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
2195	LogFlow(("pgmPoolTrackFlushGCPhysPTs: HCPhys=%RHp iPhysExt\n", pPhysPage->HCPhys, iPhysExt));
2196
2197	const uint16_t iPhysExtStart = iPhysExt;
2198	PPGMPOOLPHYSEXT pPhysExt;
2199	do
2200	{
2201	Assert(iPhysExt < pPool->cMaxPhysExts);
2202	pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2203	for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2204	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
2205	{
2206	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
2207	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2208	}
2209
2210	/* next */
2211	iPhysExt = pPhysExt->iNext;
2212	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2213
2214	/* insert the list into the free list and clear the ram range entry. */
2215	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2216	pPool->iPhysExtFreeHead = iPhysExtStart;
2217	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2218
2219	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
2220	}
2221	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2222
2223
2224	/**
2225	* Scans all shadow page tables for mappings of a physical page.
2226	*
2227	* This may be slow, but it's most likely more efficient than cleaning
2228	* out the entire page pool / cache.
2229	*
2230	* @returns VBox status code.
2231	* @retval VINF_SUCCESS if all references has been successfully cleared.
2232	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
2233	* a page pool cleaning.
2234	*
2235	* @param pVM The VM handle.
2236	* @param pPhysPage The guest page in question.
2237	*/
2238	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
2239	{
2240	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2241	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2242	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d HCPhys=%RHp\n",
2243	pPool->cUsedPages, pPool->cPresent, pPhysPage->HCPhys));
2244
2245	#if 1
2246	/*
2247	* There is a limit to what makes sense.
2248	*/
2249	if (pPool->cPresent > 1024)
2250	{
2251	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
2252	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2253	return VINF_PGM_GCPHYS_ALIASED;
2254	}
2255	#endif
2256
2257	/*
2258	* Iterate all the pages until we've encountered all that in use.
2259	* This is simple but not quite optimal solution.
2260	*/
2261	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2262	const uint32_t u32 = u64;
2263	unsigned cLeft = pPool->cUsedPages;
2264	unsigned iPage = pPool->cCurPages;
2265	while (--iPage >= PGMPOOL_IDX_FIRST)
2266	{
2267	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2268	if (pPage->GCPhys != NIL_RTGCPHYS)
2269	{
2270	switch (pPage->enmKind)
2271	{
2272	/*
2273	* We only care about shadow page tables.
2274	*/
2275	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2276	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2277	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2278	{
2279	unsigned cPresent = pPage->cPresent;
2280	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2281	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2282	if (pPT->a[i].n.u1Present)
2283	{
2284	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2285	{
2286	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
2287	pPT->a[i].u = 0;
2288	}
2289	if (!--cPresent)
2290	break;
2291	}
2292	break;
2293	}
2294
2295	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2296	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2297	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2298	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2299	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2300	{
2301	unsigned cPresent = pPage->cPresent;
2302	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2303	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2304	if (pPT->a[i].n.u1Present)
2305	{
2306	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2307	{
2308	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
2309	pPT->a[i].u = 0;
2310	}
2311	if (!--cPresent)
2312	break;
2313	}
2314	break;
2315	}
2316	}
2317	if (!--cLeft)
2318	break;
2319	}
2320	}
2321
2322	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2323	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2324	return VINF_SUCCESS;
2325	}
2326
2327
2328	/**
2329	* Clears the user entry in a user table.
2330	*
2331	* This is used to remove all references to a page when flushing it.
2332	*/
2333	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
2334	{
2335	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
2336	Assert(pUser->iUser < pPool->cCurPages);
2337
2338	/*
2339	* Map the user page.
2340	*/
2341	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
2342	union
2343	{
2344	uint64_t *pau64;
2345	uint32_t *pau32;
2346	} u;
2347	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pUserPage);
2348
2349	#ifdef VBOX_STRICT
2350	/*
2351	* Some sanity checks.
2352	*/
2353	switch (pUserPage->enmKind)
2354	{
2355	case PGMPOOLKIND_ROOT_32BIT_PD:
2356	Assert(!(u.pau32[pUser->iUser] & PGM_PDFLAGS_MAPPING));
2357	Assert(pUser->iUserTable < X86_PG_ENTRIES);
2358	break;
2359	case PGMPOOLKIND_ROOT_PAE_PD:
2360	Assert(!(u.pau64[pUser->iUser] & PGM_PDFLAGS_MAPPING));
2361	Assert(pUser->iUserTable < 2048 && pUser->iUser == PGMPOOL_IDX_PAE_PD);
2362	break;
2363	case PGMPOOLKIND_ROOT_PDPT:
2364	Assert(!(u.pau64[pUser->iUserTable] & PGM_PLXFLAGS_PERMANENT));
2365	Assert(pUser->iUserTable < 4);
2366	break;
2367	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2368	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2369	Assert(pUser->iUserTable < X86_PG_PAE_ENTRIES);
2370	break;
2371	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2372	case PGMPOOLKIND_ROOT_PML4:
2373	Assert(!(u.pau64[pUser->iUserTable] & PGM_PLXFLAGS_PERMANENT));
2374	Assert(pUser->iUserTable < X86_PG_PAE_ENTRIES);
2375	break;
2376	default:
2377	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
2378	break;
2379	}
2380	#endif /* VBOX_STRICT */
2381
2382	/*
2383	* Clear the entry in the user page.
2384	*/
2385	switch (pUserPage->enmKind)
2386	{
2387	/* 32-bit entries */
2388	case PGMPOOLKIND_ROOT_32BIT_PD:
2389	u.pau32[pUser->iUserTable] = 0;
2390	break;
2391
2392	/* 64-bit entries */
2393	case PGMPOOLKIND_ROOT_PAE_PD:
2394	case PGMPOOLKIND_ROOT_PDPT:
2395	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2396	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2397	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2398	case PGMPOOLKIND_ROOT_PML4:
2399	u.pau64[pUser->iUserTable] = 0;
2400	break;
2401
2402	default:
2403	AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
2404	}
2405	}
2406
2407
2408	/**
2409	* Clears all users of a page.
2410	*/
2411	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2412	{
2413	/*
2414	* Free all the user records.
2415	*/
2416	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
2417	uint16_t i = pPage->iUserHead;
2418	while (i != NIL_PGMPOOL_USER_INDEX)
2419	{
2420	/* Clear enter in user table. */
2421	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
2422
2423	/* Free it. */
2424	const uint16_t iNext = paUsers[i].iNext;
2425	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2426	paUsers[i].iNext = pPool->iUserFreeHead;
2427	pPool->iUserFreeHead = i;
2428
2429	/* Next. */
2430	i = iNext;
2431	}
2432	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2433	}
2434
2435
2436	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2437	/**
2438	* Allocates a new physical cross reference extent.
2439	*
2440	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
2441	* @param pVM The VM handle.
2442	* @param piPhysExt Where to store the phys ext index.
2443	*/
2444	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
2445	{
2446	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2447	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
2448	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
2449	{
2450	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
2451	return NULL;
2452	}
2453	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2454	pPool->iPhysExtFreeHead = pPhysExt->iNext;
2455	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2456	*piPhysExt = iPhysExt;
2457	return pPhysExt;
2458	}
2459
2460
2461	/**
2462	* Frees a physical cross reference extent.
2463	*
2464	* @param pVM The VM handle.
2465	* @param iPhysExt The extent to free.
2466	*/
2467	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
2468	{
2469	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2470	Assert(iPhysExt < pPool->cMaxPhysExts);
2471	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2472	for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2473	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2474	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2475	pPool->iPhysExtFreeHead = iPhysExt;
2476	}
2477
2478
2479	/**
2480	* Frees a physical cross reference extent.
2481	*
2482	* @param pVM The VM handle.
2483	* @param iPhysExt The extent to free.
2484	*/
2485	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
2486	{
2487	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2488
2489	const uint16_t iPhysExtStart = iPhysExt;
2490	PPGMPOOLPHYSEXT pPhysExt;
2491	do
2492	{
2493	Assert(iPhysExt < pPool->cMaxPhysExts);
2494	pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2495	for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2496	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2497
2498	/* next */
2499	iPhysExt = pPhysExt->iNext;
2500	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2501
2502	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2503	pPool->iPhysExtFreeHead = iPhysExtStart;
2504	}
2505
2506	/**
2507	* Insert a reference into a list of physical cross reference extents.
2508	*
2509	* @returns The new ram range flags (top 16-bits).
2510	*
2511	* @param pVM The VM handle.
2512	* @param iPhysExt The physical extent index of the list head.
2513	* @param iShwPT The shadow page table index.
2514	*
2515	*/
2516	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
2517	{
2518	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2519	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
2520
2521	/* special common case. */
2522	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
2523	{
2524	paPhysExts[iPhysExt].aidx[2] = iShwPT;
2525	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
2526	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{,,%d}\n", iPhysExt, iShwPT));
2527	return iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2528	}
2529
2530	/* general treatment. */
2531	const uint16_t iPhysExtStart = iPhysExt;
2532	unsigned cMax = 15;
2533	for (;;)
2534	{
2535	Assert(iPhysExt < pPool->cMaxPhysExts);
2536	for (unsigned i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2537	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
2538	{
2539	paPhysExts[iPhysExt].aidx[i] = iShwPT;
2540	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
2541	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
2542	return iPhysExtStart \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2543	}
2544	if (!--cMax)
2545	{
2546	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
2547	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
2548	LogFlow(("pgmPoolTrackPhysExtAddref: overflow (1) iShwPT=%d\n", iShwPT));
2549	return MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2550	}
2551	}
2552
2553	/* add another extent to the list. */
2554	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
2555	if (!pNew)
2556	{
2557	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
2558	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
2559	return MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2560	}
2561	pNew->iNext = iPhysExtStart;
2562	pNew->aidx[0] = iShwPT;
2563	LogFlow(("pgmPoolTrackPhysExtAddref: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
2564	return iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2565	}
2566
2567
2568	/**
2569	* Add a reference to guest physical page where extents are in use.
2570	*
2571	* @returns The new ram range flags (top 16-bits).
2572	*
2573	* @param pVM The VM handle.
2574	* @param u16 The ram range flags (top 16-bits).
2575	* @param iShwPT The shadow page table index.
2576	*/
2577	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
2578	{
2579	if ((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) != MM_RAM_FLAGS_CREFS_PHYSEXT)
2580	{
2581	/*
2582	* Convert to extent list.
2583	*/
2584	Assert((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) == 1);
2585	uint16_t iPhysExt;
2586	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
2587	if (pPhysExt)
2588	{
2589	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT));
2590	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
2591	pPhysExt->aidx[0] = u16 & MM_RAM_FLAGS_IDX_MASK;
2592	pPhysExt->aidx[1] = iShwPT;
2593	u16 = iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2594	}
2595	else
2596	u16 = MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2597	}
2598	else if (u16 != (MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT))))
2599	{
2600	/*
2601	* Insert into the extent list.
2602	*/
2603	u16 = pgmPoolTrackPhysExtInsert(pVM, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT);
2604	}
2605	else
2606	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
2607	return u16;
2608	}
2609
2610
2611	/**
2612	* Clear references to guest physical memory.
2613	*
2614	* @param pPool The pool.
2615	* @param pPage The page.
2616	* @param pPhysPage Pointer to the aPages entry in the ram range.
2617	*/
2618	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
2619	{
2620	const unsigned cRefs = pPhysPage->HCPhys >> MM_RAM_FLAGS_CREFS_SHIFT; /** @todo PAGE FLAGS */
2621	AssertFatalMsg(cRefs == MM_RAM_FLAGS_CREFS_PHYSEXT, ("cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
2622
2623	uint16_t iPhysExt = (pPhysPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT) & MM_RAM_FLAGS_IDX_MASK;
2624	if (iPhysExt != MM_RAM_FLAGS_IDX_OVERFLOWED)
2625	{
2626	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
2627	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
2628	do
2629	{
2630	Assert(iPhysExt < pPool->cMaxPhysExts);
2631
2632	/*
2633	* Look for the shadow page and check if it's all freed.
2634	*/
2635	for (unsigned i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2636	{
2637	if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
2638	{
2639	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
2640
2641	for (i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2642	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
2643	{
2644	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
2645	return;
2646	}
2647
2648	/* we can free the node. */
2649	PVM pVM = pPool->CTXSUFF(pVM);
2650	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
2651	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
2652	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
2653	{
2654	/* lonely node */
2655	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2656	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d lonely\n", pPhysPage->HCPhys, pPage->idx));
2657	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2658	}
2659	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
2660	{
2661	/* head */
2662	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d head\n", pPhysPage->HCPhys, pPage->idx));
2663	pPhysPage->HCPhys = (pPhysPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) /** @todo PAGE FLAGS */
2664	\| ((uint64_t)MM_RAM_FLAGS_CREFS_PHYSEXT << MM_RAM_FLAGS_CREFS_SHIFT)
2665	\| ((uint64_t)iPhysExtNext << MM_RAM_FLAGS_IDX_SHIFT);
2666	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2667	}
2668	else
2669	{
2670	/* in list */
2671	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
2672	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
2673	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2674	}
2675	iPhysExt = iPhysExtNext;
2676	return;
2677	}
2678	}
2679
2680	/* next */
2681	iPhysExtPrev = iPhysExt;
2682	iPhysExt = paPhysExts[iPhysExt].iNext;
2683	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2684
2685	AssertFatalMsgFailed(("not-found! cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
2686	}
2687	else /* nothing to do */
2688	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64\n", pPhysPage->HCPhys));
2689	}
2690
2691
2692
2693	/**
2694	* Clear references to guest physical memory.
2695	*
2696	* This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
2697	* is assumed to be correct, so the linear search can be skipped and we can assert
2698	* at an earlier point.
2699	*
2700	* @param pPool The pool.
2701	* @param pPage The page.
2702	* @param HCPhys The host physical address corresponding to the guest page.
2703	* @param GCPhys The guest physical address corresponding to HCPhys.
2704	*/
2705	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
2706	{
2707	/*
2708	* Walk range list.
2709	*/
2710	PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2711	while (pRam)
2712	{
2713	RTGCPHYS off = GCPhys - pRam->GCPhys;
2714	if (off < pRam->cb)
2715	{
2716	/* does it match? */
2717	const unsigned iPage = off >> PAGE_SHIFT;
2718	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
2719	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2720	{
2721	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2722	return;
2723	}
2724	break;
2725	}
2726	pRam = CTXALLSUFF(pRam->pNext);
2727	}
2728	AssertFatalMsgFailed(("HCPhys=%VHp GCPhys=%VGp\n", HCPhys, GCPhys));
2729	}
2730
2731
2732	/**
2733	* Clear references to guest physical memory.
2734	*
2735	* @param pPool The pool.
2736	* @param pPage The page.
2737	* @param HCPhys The host physical address corresponding to the guest page.
2738	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
2739	*/
2740	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
2741	{
2742	/*
2743	* Walk range list.
2744	*/
2745	PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2746	while (pRam)
2747	{
2748	RTGCPHYS off = GCPhysHint - pRam->GCPhys;
2749	if (off < pRam->cb)
2750	{
2751	/* does it match? */
2752	const unsigned iPage = off >> PAGE_SHIFT;
2753	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
2754	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2755	{
2756	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2757	return;
2758	}
2759	break;
2760	}
2761	pRam = CTXALLSUFF(pRam->pNext);
2762	}
2763
2764	/*
2765	* Damn, the hint didn't work. We'll have to do an expensive linear search.
2766	*/
2767	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
2768	pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2769	while (pRam)
2770	{
2771	unsigned iPage = pRam->cb >> PAGE_SHIFT;
2772	while (iPage-- > 0)
2773	{
2774	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2775	{
2776	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%VHp GCPhysHint=%VGp GCPhysReal=%VGp\n",
2777	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
2778	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2779	return;
2780	}
2781	}
2782	pRam = CTXALLSUFF(pRam->pNext);
2783	}
2784
2785	AssertFatalMsgFailed(("HCPhys=%VHp GCPhysHint=%VGp\n", HCPhys, GCPhysHint));
2786	}
2787
2788
2789	/**
2790	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
2791	*
2792	* @param pPool The pool.
2793	* @param pPage The page.
2794	* @param pShwPT The shadow page table (mapping of the page).
2795	* @param pGstPT The guest page table.
2796	*/
2797	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
2798	{
2799	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pShwPT->a); i++)
2800	if (pShwPT->a[i].n.u1Present)
2801	{
2802	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
2803	i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
2804	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
2805	if (!--pPage->cPresent)
2806	break;
2807	}
2808	}
2809
2810
2811	/**
2812	* Clear references to guest physical memory in a PAE / 32-bit page table.
2813	*
2814	* @param pPool The pool.
2815	* @param pPage The page.
2816	* @param pShwPT The shadow page table (mapping of the page).
2817	* @param pGstPT The guest page table (just a half one).
2818	*/
2819	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
2820	{
2821	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++)
2822	if (pShwPT->a[i].n.u1Present)
2823	{
2824	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX32 hint=%RX32\n",
2825	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
2826	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
2827	}
2828	}
2829
2830
2831	/**
2832	* Clear references to guest physical memory in a PAE / PAE page table.
2833	*
2834	* @param pPool The pool.
2835	* @param pPage The page.
2836	* @param pShwPT The shadow page table (mapping of the page).
2837	* @param pGstPT The guest page table.
2838	*/
2839	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
2840	{
2841	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++)
2842	if (pShwPT->a[i].n.u1Present)
2843	{
2844	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
2845	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
2846	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
2847	}
2848	}
2849
2850
2851	/**
2852	* Clear references to guest physical memory in a 32-bit / 4MB page table.
2853	*
2854	* @param pPool The pool.
2855	* @param pPage The page.
2856	* @param pShwPT The shadow page table (mapping of the page).
2857	*/
2858	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
2859	{
2860	RTGCPHYS GCPhys = pPage->GCPhys;
2861	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
2862	if (pShwPT->a[i].n.u1Present)
2863	{
2864	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
2865	i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
2866	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
2867	}
2868	}
2869
2870
2871	/**
2872	* Clear references to guest physical memory in a PAE / 2/4MB page table.
2873	*
2874	* @param pPool The pool.
2875	* @param pPage The page.
2876	* @param pShwPT The shadow page table (mapping of the page).
2877	*/
2878	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
2879	{
2880	RTGCPHYS GCPhys = pPage->GCPhys;
2881	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
2882	if (pShwPT->a[i].n.u1Present)
2883	{
2884	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX32 hint=%RX32\n",
2885	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
2886	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
2887	}
2888	}
2889	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2890
2891
2892	/**
2893	* Clear references to shadowed pages in a PAE page directory.
2894	*
2895	* @param pPool The pool.
2896	* @param pPage The page.
2897	* @param pShwPD The shadow page directory (mapping of the page).
2898	*/
2899	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
2900	{
2901	for (unsigned i = 0; i < ELEMENTS(pShwPD->a); i++)
2902	{
2903	if (pShwPD->a[i].n.u1Present)
2904	{
2905	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
2906	if (pSubPage)
2907	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
2908	else
2909	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
2910	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
2911	}
2912	}
2913	}
2914
2915
2916	/**
2917	* Clear references to shadowed pages in a 64-bit page directory pointer table.
2918	*
2919	* @param pPool The pool.
2920	* @param pPage The page.
2921	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
2922	*/
2923	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
2924	{
2925	for (unsigned i = 0; i < ELEMENTS(pShwPDPT->a); i++)
2926	{
2927	if (pShwPDPT->a[i].n.u1Present)
2928	{
2929	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
2930	if (pSubPage)
2931	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
2932	else
2933	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
2934	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
2935	}
2936	}
2937	}
2938
2939
2940	/**
2941	* Clears all references made by this page.
2942	*
2943	* This includes other shadow pages and GC physical addresses.
2944	*
2945	* @param pPool The pool.
2946	* @param pPage The page.
2947	*/
2948	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2949	{
2950	/*
2951	* Map the shadow page and take action according to the page kind.
2952	*/
2953	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
2954	switch (pPage->enmKind)
2955	{
2956	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2957	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2958	{
2959	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2960	void *pvGst;
2961	int rc = PGM_GCPHYS_2_PTR(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2962	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
2963	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2964	break;
2965	}
2966
2967	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2968	{
2969	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2970	void *pvGst;
2971	int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2972	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
2973	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2974	break;
2975	}
2976
2977	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2978	{
2979	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2980	void *pvGst;
2981	int rc = PGM_GCPHYS_2_PTR(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2982	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
2983	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2984	break;
2985	}
2986
2987	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
2988	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2989	{
2990	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2991	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
2992	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2993	break;
2994	}
2995
2996	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 4 MB page */
2997	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2998	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2999	{
3000	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3001	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
3002	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3003	break;
3004	}
3005
3006	#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3007	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3008	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3009	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3010	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3011	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3012	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3013	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3014	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3015	break;
3016	#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3017
3018	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
3019	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3020	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
3021	break;
3022
3023	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3024	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
3025	break;
3026
3027	default:
3028	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
3029	}
3030
3031	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
3032	STAM_PROFILE_START(&pPool->StatZeroPage, z);
3033	ASMMemZeroPage(pvShw);
3034	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3035	pPage->fZeroed = true;
3036	}
3037	#endif /* PGMPOOL_WITH_USER_TRACKING */
3038
3039
3040	/**
3041	* Flushes all the special root pages as part of a pgmPoolFlushAllInt operation.
3042	*
3043	* @param pPool The pool.
3044	*/
3045	static void pgmPoolFlushAllSpecialRoots(PPGMPOOL pPool)
3046	{
3047	/*
3048	* These special pages are all mapped into the indexes 1..PGMPOOL_IDX_FIRST.
3049	*/
3050	Assert(NIL_PGMPOOL_IDX == 0);
3051	for (unsigned i = 1; i < PGMPOOL_IDX_FIRST; i++)
3052	{
3053	/*
3054	* Get the page address.
3055	*/
3056	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3057	union
3058	{
3059	uint64_t *pau64;
3060	uint32_t *pau32;
3061	} u;
3062	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
3063
3064	/*
3065	* Mark stuff not present.
3066	*/
3067	switch (pPage->enmKind)
3068	{
3069	case PGMPOOLKIND_ROOT_32BIT_PD:
3070	for (unsigned iPage = 0; iPage < X86_PG_ENTRIES; iPage++)
3071	if ((u.pau32[iPage] & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == X86_PDE_P)
3072	u.pau32[iPage] = 0;
3073	break;
3074
3075	case PGMPOOLKIND_ROOT_PAE_PD:
3076	for (unsigned iPage = 0; iPage < X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES; iPage++)
3077	if ((u.pau64[iPage] & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == X86_PDE_P)
3078	u.pau64[iPage] = 0;
3079	break;
3080
3081	case PGMPOOLKIND_ROOT_PML4:
3082	for (unsigned iPage = 0; iPage < X86_PG_PAE_ENTRIES; iPage++)
3083	if ((u.pau64[iPage] & (PGM_PLXFLAGS_PERMANENT \| X86_PML4E_P)) == X86_PML4E_P)
3084	u.pau64[iPage] = 0;
3085	break;
3086
3087	case PGMPOOLKIND_ROOT_PDPT:
3088	/* Not root of shadowed pages currently, ignore it. */
3089	break;
3090	}
3091	}
3092
3093	/*
3094	* Paranoia (to be removed), flag a global CR3 sync.
3095	*/
3096	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
3097	}
3098
3099
3100	/**
3101	* Flushes the entire cache.
3102	*
3103	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3104	* and execute this CR3 flush.
3105	*
3106	* @param pPool The pool.
3107	*/
3108	static void pgmPoolFlushAllInt(PPGMPOOL pPool)
3109	{
3110	STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
3111	LogFlow(("pgmPoolFlushAllInt:\n"));
3112
3113	/*
3114	* If there are no pages in the pool, there is nothing to do.
3115	*/
3116	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
3117	{
3118	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3119	return;
3120	}
3121
3122	/*
3123	* Nuke the free list and reinsert all pages into it.
3124	*/
3125	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
3126	{
3127	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3128
3129	#ifdef IN_RING3
3130	Assert(pPage->Core.Key == MMPage2Phys(pPool->pVMHC, pPage->pvPageHC));
3131	#endif
3132	#ifdef PGMPOOL_WITH_MONITORING
3133	if (pPage->fMonitored)
3134	pgmPoolMonitorFlush(pPool, pPage);
3135	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3136	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3137	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3138	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3139	pPage->cModifications = 0;
3140	#endif
3141	pPage->GCPhys = NIL_RTGCPHYS;
3142	pPage->enmKind = PGMPOOLKIND_FREE;
3143	Assert(pPage->idx == i);
3144	pPage->iNext = i + 1;
3145	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
3146	pPage->fSeenNonGlobal = false;
3147	pPage->fMonitored= false;
3148	pPage->fCached = false;
3149	pPage->fReusedFlushPending = false;
3150	pPage->fCR3Mix = false;
3151	#ifdef PGMPOOL_WITH_USER_TRACKING
3152	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3153	#endif
3154	#ifdef PGMPOOL_WITH_CACHE
3155	pPage->iAgeNext = NIL_PGMPOOL_IDX;
3156	pPage->iAgePrev = NIL_PGMPOOL_IDX;
3157	#endif
3158	}
3159	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
3160	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
3161	pPool->cUsedPages = 0;
3162
3163	#ifdef PGMPOOL_WITH_USER_TRACKING
3164	/*
3165	* Zap and reinitialize the user records.
3166	*/
3167	pPool->cPresent = 0;
3168	pPool->iUserFreeHead = 0;
3169	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
3170	const unsigned cMaxUsers = pPool->cMaxUsers;
3171	for (unsigned i = 0; i < cMaxUsers; i++)
3172	{
3173	paUsers[i].iNext = i + 1;
3174	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3175	paUsers[i].iUserTable = 0xfffe;
3176	}
3177	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
3178	#endif
3179
3180	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3181	/*
3182	* Clear all the GCPhys links and rebuild the phys ext free list.
3183	*/
3184	for (PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
3185	pRam;
3186	pRam = CTXALLSUFF(pRam->pNext))
3187	{
3188	unsigned iPage = pRam->cb >> PAGE_SHIFT;
3189	while (iPage-- > 0)
3190	pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
3191	}
3192
3193	pPool->iPhysExtFreeHead = 0;
3194	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
3195	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
3196	for (unsigned i = 0; i < cMaxPhysExts; i++)
3197	{
3198	paPhysExts[i].iNext = i + 1;
3199	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
3200	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
3201	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
3202	}
3203	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3204	#endif
3205
3206	#ifdef PGMPOOL_WITH_MONITORING
3207	/*
3208	* Just zap the modified list.
3209	*/
3210	pPool->cModifiedPages = 0;
3211	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
3212	#endif
3213
3214	#ifdef PGMPOOL_WITH_CACHE
3215	/*
3216	* Clear the GCPhys hash and the age list.
3217	*/
3218	for (unsigned i = 0; i < ELEMENTS(pPool->aiHash); i++)
3219	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
3220	pPool->iAgeHead = NIL_PGMPOOL_IDX;
3221	pPool->iAgeTail = NIL_PGMPOOL_IDX;
3222	#endif
3223
3224	/*
3225	* Flush all the special root pages.
3226	* Reinsert active pages into the hash and ensure monitoring chains are correct.
3227	*/
3228	pgmPoolFlushAllSpecialRoots(pPool);
3229	for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
3230	{
3231	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3232	pPage->iNext = NIL_PGMPOOL_IDX;
3233	#ifdef PGMPOOL_WITH_MONITORING
3234	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3235	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3236	pPage->cModifications = 0;
3237	/* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
3238	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3239	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3240	if (pPage->fMonitored)
3241	{
3242	PVM pVM = pPool->CTXSUFF(pVM);
3243	int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
3244	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
3245	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
3246	pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pPage),
3247	pPool->pszAccessHandler);
3248	AssertFatalRCSuccess(rc);
3249	# ifdef PGMPOOL_WITH_CACHE
3250	pgmPoolHashInsert(pPool, pPage);
3251	# endif
3252	}
3253	#endif
3254	#ifdef PGMPOOL_WITH_USER_TRACKING
3255	Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
3256	#endif
3257	#ifdef PGMPOOL_WITH_CACHE
3258	Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
3259	Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
3260	#endif
3261	}
3262
3263	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3264	}
3265
3266
3267	/**
3268	* Flushes a pool page.
3269	*
3270	* This moves the page to the free list after removing all user references to it.
3271	* In GC this will cause a CR3 reload if the page is traced back to an active root page.
3272	*
3273	* @returns VBox status code.
3274	* @retval VINF_SUCCESS on success.
3275	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
3276	* @param pPool The pool.
3277	* @param HCPhys The HC physical address of the shadow page.
3278	*/
3279	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3280	{
3281	int rc = VINF_SUCCESS;
3282	STAM_PROFILE_START(&pPool->StatFlushPage, f);
3283	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%VHp, .idx=%d, .enmKind=%d, .GCPhys=%VGp}\n",
3284	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
3285
3286	/*
3287	* Quietly reject any attempts at flushing any of the special root pages.
3288	*/
3289	if (pPage->idx < PGMPOOL_IDX_FIRST)
3290	{
3291	Log(("pgmPoolFlushPage: specaial root page, rejected. enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
3292	return VINF_SUCCESS;
3293	}
3294
3295	/*
3296	* Mark the page as being in need of a ASMMemZeroPage().
3297	*/
3298	pPage->fZeroed = false;
3299
3300	#ifdef PGMPOOL_WITH_USER_TRACKING
3301	/*
3302	* Clear the page.
3303	*/
3304	pgmPoolTrackClearPageUsers(pPool, pPage);
3305	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
3306	pgmPoolTrackDeref(pPool, pPage);
3307	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
3308	#endif
3309
3310	#ifdef PGMPOOL_WITH_CACHE
3311	/*
3312	* Flush it from the cache.
3313	*/
3314	pgmPoolCacheFlushPage(pPool, pPage);
3315	#endif /* PGMPOOL_WITH_CACHE */
3316
3317	#ifdef PGMPOOL_WITH_MONITORING
3318	/*
3319	* Deregistering the monitoring.
3320	*/
3321	if (pPage->fMonitored)
3322	rc = pgmPoolMonitorFlush(pPool, pPage);
3323	#endif
3324
3325	/*
3326	* Free the page.
3327	*/
3328	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
3329	pPage->iNext = pPool->iFreeHead;
3330	pPool->iFreeHead = pPage->idx;
3331	pPage->enmKind = PGMPOOLKIND_FREE;
3332	pPage->GCPhys = NIL_RTGCPHYS;
3333	pPage->fReusedFlushPending = false;
3334
3335	pPool->cUsedPages--;
3336	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
3337	return rc;
3338	}
3339
3340
3341	/**
3342	* Frees a usage of a pool page.
3343	*
3344	* The caller is responsible to updating the user table so that it no longer
3345	* references the shadow page.
3346	*
3347	* @param pPool The pool.
3348	* @param HCPhys The HC physical address of the shadow page.
3349	* @param iUser The shadow page pool index of the user table.
3350	* @param iUserTable The index into the user table (shadowed).
3351	*/
3352	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
3353	{
3354	STAM_PROFILE_START(&pPool->StatFree, a);
3355	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%VHp, .idx=%d, enmKind=%d} iUser=%#x iUserTable=%#x\n",
3356	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, iUser, iUserTable));
3357	Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
3358	#ifdef PGMPOOL_WITH_USER_TRACKING
3359	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
3360	#endif
3361	#ifdef PGMPOOL_WITH_CACHE
3362	if (!pPage->fCached)
3363	#endif
3364	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED can be ignored here. */
3365	STAM_PROFILE_STOP(&pPool->StatFree, a);
3366	}
3367
3368
3369	/**
3370	* Makes one or more free page free.
3371	*
3372	* @returns VBox status code.
3373	* @retval VINF_SUCCESS on success.
3374	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3375	*
3376	* @param pPool The pool.
3377	* @param iUser The user of the page.
3378	*/
3379	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, uint16_t iUser)
3380	{
3381	LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
3382
3383	/*
3384	* If the pool isn't full grown yet, expand it.
3385	*/
3386	if (pPool->cCurPages < pPool->cMaxPages)
3387	{
3388	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
3389	#ifdef IN_RING3
3390	int rc = PGMR3PoolGrow(pPool->pVMHC);
3391	#else
3392	int rc = CTXALLMID(VMM, CallHost)(pPool->CTXSUFF(pVM), VMMCALLHOST_PGM_POOL_GROW, 0);
3393	#endif
3394	if (VBOX_FAILURE(rc))
3395	return rc;
3396	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
3397	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
3398	return VINF_SUCCESS;
3399	}
3400
3401	#ifdef PGMPOOL_WITH_CACHE
3402	/*
3403	* Free one cached page.
3404	*/
3405	return pgmPoolCacheFreeOne(pPool, iUser);
3406	#else
3407	/*
3408	* Flush the pool.
3409	* If we have tracking enabled, it should be possible to come up with
3410	* a cheap replacement strategy...
3411	*/
3412	pgmPoolFlushAllInt(pPool);
3413	return VERR_PGM_POOL_FLUSHED;
3414	#endif
3415	}
3416
3417
3418	/**
3419	* Allocates a page from the pool.
3420	*
3421	* This page may actually be a cached page and not in need of any processing
3422	* on the callers part.
3423	*
3424	* @returns VBox status code.
3425	* @retval VINF_SUCCESS if a NEW page was allocated.
3426	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
3427	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3428	* @param pVM The VM handle.
3429	* @param GCPhys The GC physical address of the page we're gonna shadow.
3430	* For 4MB and 2MB PD entries, it's the first address the
3431	* shadow PT is covering.
3432	* @param enmKind The kind of mapping.
3433	* @param iUser The shadow page pool index of the user table.
3434	* @param iUserTable The index into the user table (shadowed).
3435	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
3436	*/
3437	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint16_t iUserTable, PPPGMPOOLPAGE ppPage)
3438	{
3439	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3440	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
3441	LogFlow(("pgmPoolAlloc: GCPhys=%VGp enmKind=%d iUser=%#x iUserTable=%#x\n", GCPhys, enmKind, iUser, iUserTable));
3442	*ppPage = NULL;
3443
3444	#ifdef PGMPOOL_WITH_CACHE
3445	if (pPool->fCacheEnabled)
3446	{
3447	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, iUser, iUserTable, ppPage);
3448	if (VBOX_SUCCESS(rc2))
3449	{
3450	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3451	LogFlow(("pgmPoolAlloc: returns %Vrc ppPage=%p:{.Key=%VHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
3452	return rc2;
3453	}
3454	}
3455	#endif
3456
3457	/*
3458	* Allocate a new one.
3459	*/
3460	int rc = VINF_SUCCESS;
3461	uint16_t iNew = pPool->iFreeHead;
3462	if (iNew == NIL_PGMPOOL_IDX)
3463	{
3464	rc = pgmPoolMakeMoreFreePages(pPool, iUser);
3465	if (VBOX_FAILURE(rc))
3466	{
3467	if (rc != VERR_PGM_POOL_CLEARED)
3468	{
3469	Log(("pgmPoolAlloc: returns %Vrc (Free)\n", rc));
3470	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3471	return rc;
3472	}
3473	rc = VERR_PGM_POOL_FLUSHED;
3474	}
3475	iNew = pPool->iFreeHead;
3476	AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
3477	}
3478
3479	/* unlink the free head */
3480	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
3481	pPool->iFreeHead = pPage->iNext;
3482	pPage->iNext = NIL_PGMPOOL_IDX;
3483
3484	/*
3485	* Initialize it.
3486	*/
3487	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
3488	pPage->enmKind = enmKind;
3489	pPage->GCPhys = GCPhys;
3490	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
3491	pPage->fMonitored = false;
3492	pPage->fCached = false;
3493	pPage->fReusedFlushPending = false;
3494	pPage->fCR3Mix = false;
3495	#ifdef PGMPOOL_WITH_MONITORING
3496	pPage->cModifications = 0;
3497	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3498	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3499	#endif
3500	#ifdef PGMPOOL_WITH_USER_TRACKING
3501	pPage->cPresent = 0;
3502	pPage->iFirstPresent = ~0;
3503
3504	/*
3505	* Insert into the tracking and cache. If this fails, free the page.
3506	*/
3507	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
3508	if (VBOX_FAILURE(rc3))
3509	{
3510	if (rc3 != VERR_PGM_POOL_CLEARED)
3511	{
3512	pPool->cUsedPages--;
3513	pPage->enmKind = PGMPOOLKIND_FREE;
3514	pPage->GCPhys = NIL_RTGCPHYS;
3515	pPage->iNext = pPool->iFreeHead;
3516	pPool->iFreeHead = pPage->idx;
3517	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3518	Log(("pgmPoolAlloc: returns %Vrc (Insert)\n", rc3));
3519	return rc3;
3520	}
3521	rc = VERR_PGM_POOL_FLUSHED;
3522	}
3523	#endif /* PGMPOOL_WITH_USER_TRACKING */
3524
3525	/*
3526	* Commit the allocation, clear the page and return.
3527	*/
3528	#ifdef VBOX_WITH_STATISTICS
3529	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
3530	pPool->cUsedPagesHigh = pPool->cUsedPages;
3531	#endif
3532
3533	if (!pPage->fZeroed)
3534	{
3535	STAM_PROFILE_START(&pPool->StatZeroPage, z);
3536	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
3537	ASMMemZeroPage(pv);
3538	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3539	}
3540
3541	*ppPage = pPage;
3542	LogFlow(("pgmPoolAlloc: returns %Vrc *ppPage=%p:{.Key=%VHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
3543	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
3544	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3545	return rc;
3546	}
3547
3548
3549	/**
3550	* Frees a usage of a pool page.
3551	*
3552	* @param pVM The VM handle.
3553	* @param HCPhys The HC physical address of the shadow page.
3554	* @param iUser The shadow page pool index of the user table.
3555	* @param iUserTable The index into the user table (shadowed).
3556	*/
3557	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint16_t iUserTable)
3558	{
3559	LogFlow(("pgmPoolFree: HCPhys=%VHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
3560	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3561	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
3562	}
3563
3564
3565	/**
3566	* Gets a in-use page in the pool by it's physical address.
3567	*
3568	* @returns Pointer to the page.
3569	* @param pVM The VM handle.
3570	* @param HCPhys The HC physical address of the shadow page.
3571	* @remark This function will NEVER return NULL. It will assert if HCPhys is invalid.
3572	*/
3573	PPGMPOOLPAGE pgmPoolGetPageByHCPhys(PVM pVM, RTHCPHYS HCPhys)
3574	{
3575	/** @todo profile this! */
3576	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3577	PPGMPOOLPAGE pPage = pgmPoolGetPage(pPool, HCPhys);
3578	Log3(("pgmPoolGetPageByHCPhys: HCPhys=%VHp -> %p:{.idx=%d .GCPhys=%VGp .enmKind=%d}\n",
3579	HCPhys, pPage, pPage->idx, pPage->GCPhys, pPage->enmKind));
3580	return pPage;
3581	}
3582
3583
3584	/**
3585	* Flushes the entire cache.
3586	*
3587	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3588	* and execute this CR3 flush.
3589	*
3590	* @param pPool The pool.
3591	*/
3592	void pgmPoolFlushAll(PVM pVM)
3593	{
3594	LogFlow(("pgmPoolFlushAll:\n"));
3595	pgmPoolFlushAllInt(pVM->pgm.s.CTXSUFF(pPool));
3596	}
3597

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 8110

Download in other formats: