PGMAllPool.cpp@ 8090

Last change on this file since 8090 was 8090, checked in by vboxsync, 17 years ago
Missing PGMPOOLKIND_PAE_PT_FOR_PAE_2MB case in pgmPoolTrackDeref.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 125.5 KB

Line
1	/* $Id: PGMAllPool.cpp 8090 2008-04-17 12:18:06Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 innotek GmbH
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_POOL
23	#include <VBox/pgm.h>
24	#include <VBox/mm.h>
25	#include <VBox/em.h>
26	#include <VBox/cpum.h>
27	#ifdef IN_GC
28	# include <VBox/patm.h>
29	#endif
30	#include "PGMInternal.h"
31	#include <VBox/vm.h>
32	#include <VBox/disopcode.h>
33
34	#include <VBox/log.h>
35	#include <VBox/err.h>
36	#include <iprt/asm.h>
37
38
39	/*******************************************************************************
40	* Internal Functions *
41	*******************************************************************************/
42	__BEGIN_DECLS
43	static void pgmPoolFlushAllInt(PPGMPOOL pPool);
44	#ifdef PGMPOOL_WITH_USER_TRACKING
45	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
46	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
47	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
48	#endif
49	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
50	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint);
51	#endif
52	#ifdef PGMPOOL_WITH_CACHE
53	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable);
54	#endif
55	#ifdef PGMPOOL_WITH_MONITORING
56	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
57	#endif
58	#ifndef IN_RING3
59	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
60	#endif
61	__END_DECLS
62
63
64	/**
65	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
66	*
67	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
68	* @param enmKind The page kind.
69	*/
70	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
71	{
72	switch (enmKind)
73	{
74	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
75	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
76	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
77	return true;
78	default:
79	return false;
80	}
81	}
82
83
84	#ifdef IN_GC
85	/**
86	* Maps a pool page into the current context.
87	*
88	* @returns Pointer to the mapping.
89	* @param pVM The VM handle.
90	* @param pPage The page to map.
91	*/
92	void *pgmGCPoolMapPage(PVM pVM, PPGMPOOLPAGE pPage)
93	{
94	/* general pages. */
95	if (pPage->idx >= PGMPOOL_IDX_FIRST)
96	{
97	Assert(pPage->idx < pVM->pgm.s.pPoolGC->cCurPages);
98	void *pv;
99	int rc = PGMGCDynMapHCPage(pVM, pPage->Core.Key, &pv);
100	AssertReleaseRC(rc);
101	return pv;
102	}
103
104	/* special pages. */
105	switch (pPage->idx)
106	{
107	case PGMPOOL_IDX_PD:
108	return pVM->pgm.s.pGC32BitPD;
109	case PGMPOOL_IDX_PAE_PD:
110	case PGMPOOL_IDX_PAE_PD_0:
111	return pVM->pgm.s.apGCPaePDs[0];
112	case PGMPOOL_IDX_PAE_PD_1:
113	return pVM->pgm.s.apGCPaePDs[1];
114	case PGMPOOL_IDX_PAE_PD_2:
115	return pVM->pgm.s.apGCPaePDs[2];
116	case PGMPOOL_IDX_PAE_PD_3:
117	return pVM->pgm.s.apGCPaePDs[3];
118	case PGMPOOL_IDX_PDPT:
119	return pVM->pgm.s.pGCPaePDPT;
120	case PGMPOOL_IDX_PML4:
121	return pVM->pgm.s.pGCPaePML4;
122	default:
123	AssertReleaseMsgFailed(("Invalid index %d\n", pPage->idx));
124	return NULL;
125	}
126	}
127	#endif /* IN_GC */
128
129
130	#ifdef PGMPOOL_WITH_MONITORING
131	/**
132	* Determin the size of a write instruction.
133	* @returns number of bytes written.
134	* @param pDis The disassembler state.
135	*/
136	static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
137	{
138	/*
139	* This is very crude and possibly wrong for some opcodes,
140	* but since it's not really supposed to be called we can
141	* probably live with that.
142	*/
143	return DISGetParamSize(pDis, &pDis->param1);
144	}
145
146
147	/**
148	* Flushes a chain of pages sharing the same access monitor.
149	*
150	* @returns VBox status code suitable for scheduling.
151	* @param pPool The pool.
152	* @param pPage A page in the chain.
153	*/
154	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
155	{
156	/*
157	* Find the list head.
158	*/
159	uint16_t idx = pPage->idx;
160	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
161	{
162	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
163	{
164	idx = pPage->iMonitoredPrev;
165	Assert(idx != pPage->idx);
166	pPage = &pPool->aPages[idx];
167	}
168	}
169
170	/*
171	* Itereate the list flushing each shadow page.
172	*/
173	int rc = VINF_SUCCESS;
174	for (;;)
175	{
176	idx = pPage->iMonitoredNext;
177	Assert(idx != pPage->idx);
178	if (pPage->idx >= PGMPOOL_IDX_FIRST)
179	{
180	int rc2 = pgmPoolFlushPage(pPool, pPage);
181	if (rc2 == VERR_PGM_POOL_CLEARED && rc == VINF_SUCCESS)
182	rc = VINF_PGM_SYNC_CR3;
183	}
184	/* next */
185	if (idx == NIL_PGMPOOL_IDX)
186	break;
187	pPage = &pPool->aPages[idx];
188	}
189	return rc;
190	}
191
192
193	/**
194	* Wrapper for getting the current context pointer to the entry being modified.
195	*
196	* @returns Pointer to the current context mapping of the entry.
197	* @param pPool The pool.
198	* @param pvFault The fault virtual address.
199	* @param GCPhysFault The fault physical address.
200	* @param cbEntry The entry size.
201	*/
202	#ifdef IN_RING3
203	DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTHCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
204	#else
205	DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTGCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
206	#endif
207	{
208	#ifdef IN_GC
209	return (RTGCPTR)((RTGCUINTPTR)pvFault & ~(RTGCUINTPTR)(cbEntry - 1));
210
211	#elif defined(IN_RING0)
212	void *pvRet;
213	int rc = pgmRamGCPhys2HCPtr(&pPool->pVMHC->pgm.s, GCPhysFault & ~(RTGCPHYS)(cbEntry - 1), &pvRet);
214	AssertFatalRCSuccess(rc);
215	return pvRet;
216
217	#elif defined(IN_RING3)
218	return (RTHCPTR)((uintptr_t)pvFault & ~(RTHCUINTPTR)(cbEntry - 1));
219	#else
220	# error "huh?"
221	#endif
222	}
223
224
225	/**
226	* Process shadow entries before they are changed by the guest.
227	*
228	* For PT entries we will clear them. For PD entries, we'll simply check
229	* for mapping conflicts and set the SyncCR3 FF if found.
230	*
231	* @param pPool The pool.
232	* @param pPage The head page.
233	* @param GCPhysFault The guest physical fault address.
234	* @param uAddress In R0 and GC this is the guest context fault address (flat).
235	* In R3 this is the host context 'fault' address.
236	* @param pCpu The disassembler state for figuring out the write size.
237	* This need not be specified if the caller knows we won't do cross entry accesses.
238	*/
239	#ifdef IN_RING3
240	void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTHCPTR pvAddress, PDISCPUSTATE pCpu)
241	#else
242	void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTGCPTR pvAddress, PDISCPUSTATE pCpu)
243	#endif
244	{
245	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
246	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
247
248	LogFlow(("pgmPoolMonitorChainChanging: %VGv phys=%VGp kind=%d\n", pvAddress, GCPhysFault, pPage->enmKind));
249
250	for (;;)
251	{
252	union
253	{
254	void *pv;
255	PX86PT pPT;
256	PX86PTPAE pPTPae;
257	PX86PD pPD;
258	PX86PDPAE pPDPae;
259	PX86PDPT pPDPT;
260	} uShw;
261	uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
262
263	switch (pPage->enmKind)
264	{
265	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
266	{
267	const unsigned iShw = off / sizeof(X86PTE);
268	if (uShw.pPT->a[iShw].n.u1Present)
269	{
270	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
271	PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
272	Log4(("pgmPoolMonitorChainChanging 32_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
273	pgmPoolTracDerefGCPhysHint(pPool, pPage,
274	uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
275	pGstPte->u & X86_PTE_PG_MASK);
276	# endif
277	uShw.pPT->a[iShw].u = 0;
278	}
279	break;
280	}
281
282	/* page/2 sized */
283	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
284	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
285	{
286	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
287	if (uShw.pPTPae->a[iShw].n.u1Present)
288	{
289	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
290	PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
291	Log4(("pgmPoolMonitorChainChanging pae_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
292	pgmPoolTracDerefGCPhysHint(pPool, pPage,
293	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
294	pGstPte->u & X86_PTE_PG_MASK);
295	# endif
296	uShw.pPTPae->a[iShw].u = 0;
297	}
298	}
299	break;
300
301	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
302	{
303	const unsigned iShw = off / sizeof(X86PTEPAE);
304	if (uShw.pPTPae->a[iShw].n.u1Present)
305	{
306	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
307	PCX86PTEPAE pGstPte = (PCX86PTEPAE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
308	Log4(("pgmPoolMonitorChainChanging pae_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PAE_PG_MASK));
309	pgmPoolTracDerefGCPhysHint(pPool, pPage,
310	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
311	pGstPte->u & X86_PTE_PAE_PG_MASK);
312	# endif
313	uShw.pPTPae->a[iShw].u = 0;
314	}
315	break;
316	}
317
318	case PGMPOOLKIND_ROOT_32BIT_PD:
319	{
320	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
321	if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
322	{
323	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
324	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
325	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
326	}
327	/* paranoia / a bit assumptive. */
328	else if ( pCpu
329	&& (off & 3)
330	&& (off & 3) + pgmPoolDisasWriteSize(pCpu) > 4)
331	{
332	const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PTE);
333	if ( iShw2 != iShw
334	&& iShw2 < ELEMENTS(uShw.pPD->a)
335	&& uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
336	{
337	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
338	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
339	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
340	}
341	}
342	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
343	if ( uShw.pPD->a[iShw].n.u1Present
344	&& !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
345	{
346	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
347	# ifdef IN_GC /* TLB load - we're pushing things a bit... */
348	ASMProbeReadByte(pvAddress);
349	# endif
350	pgmPoolFree(pPool->CTXSUFF(pVM), uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
351	uShw.pPD->a[iShw].u = 0;
352	}
353	#endif
354	break;
355	}
356
357	case PGMPOOLKIND_ROOT_PAE_PD:
358	{
359	unsigned iShw = (off / sizeof(X86PTE)) * 2; // ASSUMING 32-bit guest paging!
360	for (unsigned i = 0; i < 2; i++, iShw++)
361	{
362	if ((uShw.pPDPae->a[iShw].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
363	{
364	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
365	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
366	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
367	}
368	/* paranoia / a bit assumptive. */
369	else if ( pCpu
370	&& (off & 3)
371	&& (off & 3) + pgmPoolDisasWriteSize(pCpu) > 4)
372	{
373	const unsigned iShw2 = iShw + 2;
374	if ( iShw2 < ELEMENTS(uShw.pPDPae->a)
375	&& (uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
376	{
377	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
378	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
379	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
380	}
381	}
382	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
383	if ( uShw.pPDPae->a[iShw].n.u1Present
384	&& !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
385	{
386	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
387	# ifdef IN_GC /* TLB load - we're pushing things a bit... */
388	ASMProbeReadByte(pvAddress);
389	# endif
390	pgmPoolFree(pPool->CTXSUFF(pVM), uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
391	uShw.pPDPae->a[iShw].u = 0;
392	}
393	#endif
394	}
395	break;
396	}
397
398	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
399	{
400	const unsigned iShw = off / sizeof(X86PTEPAE);
401	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
402	{
403	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
404	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
405	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
406	}
407	/* paranoia / a bit assumptive. */
408	else if ( pCpu
409	&& (off & 7)
410	&& (off & 7) + pgmPoolDisasWriteSize(pCpu) > sizeof(X86PTEPAE))
411	{
412	const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PTEPAE);
413	if ( iShw2 != iShw
414	&& iShw2 < ELEMENTS(uShw.pPDPae->a)
415	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
416	{
417	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
418	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
419	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
420	}
421	}
422	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
423	if ( uShw.pPDPae->a[iShw].n.u1Present
424	&& !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
425	{
426	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
427	# ifdef IN_GC /* TLB load - we're pushing things a bit... */
428	ASMProbeReadByte(pvAddress);
429	# endif
430	pgmPoolFree(pPool->CTXSUFF(pVM), uShwpPDPae->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
431	uShw.pPDPae->a[iShw].u = 0;
432	}
433	#endif
434	break;
435	}
436
437	case PGMPOOLKIND_ROOT_PDPT:
438	{
439	/* Hopefully this doesn't happen very often:
440	* - touching unused parts of the page
441	* - messing with the bits of pd pointers without changing the physical address
442	*/
443	const unsigned iShw = off / sizeof(X86PDPE);
444	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
445	{
446	if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
447	{
448	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
449	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
450	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
451	}
452	/* paranoia / a bit assumptive. */
453	else if ( pCpu
454	&& (off & 7)
455	&& (off & 7) + pgmPoolDisasWriteSize(pCpu) > sizeof(X86PDPE))
456	{
457	const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PDPE);
458	if ( iShw2 != iShw
459	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES
460	&& uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
461	{
462	Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
463	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
464	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
465	}
466	}
467	}
468	break;
469	}
470
471	default:
472	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
473	}
474
475	/* next */
476	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
477	return;
478	pPage = &pPool->aPages[pPage->iMonitoredNext];
479	}
480	}
481
482
483	# ifndef IN_RING3
484	/**
485	* Checks if a access could be a fork operation in progress.
486	*
487	* Meaning, that the guest is setuping up the parent process for Copy-On-Write.
488	*
489	* @returns true if it's likly that we're forking, otherwise false.
490	* @param pPool The pool.
491	* @param pCpu The disassembled instruction.
492	* @param offFault The access offset.
493	*/
494	DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pCpu, unsigned offFault)
495	{
496	/*
497	* i386 linux is using btr to clear X86_PTE_RW.
498	* The functions involved are (2.6.16 source inspection):
499	* clear_bit
500	* ptep_set_wrprotect
501	* copy_one_pte
502	* copy_pte_range
503	* copy_pmd_range
504	* copy_pud_range
505	* copy_page_range
506	* dup_mmap
507	* dup_mm
508	* copy_mm
509	* copy_process
510	* do_fork
511	*/
512	if ( pCpu->pCurInstr->opcode == OP_BTR
513	&& !(offFault & 4)
514	/** @todo Validate that the bit index is X86_PTE_RW. */
515	)
516	{
517	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,Fork));
518	return true;
519	}
520	return false;
521	}
522
523
524	/**
525	* Determin whether the page is likely to have been reused.
526	*
527	* @returns true if we consider the page as being reused for a different purpose.
528	* @returns false if we consider it to still be a paging page.
529	* @param pPage The page in question.
530	* @param pCpu The disassembly info for the faulting insturction.
531	* @param pvFault The fault address.
532	*
533	* @remark The REP prefix check is left to the caller because of STOSD/W.
534	*/
535	DECLINLINE(bool) pgmPoolMonitorIsReused(PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu, RTGCPTR pvFault)
536	{
537	switch (pCpu->pCurInstr->opcode)
538	{
539	case OP_PUSH:
540	Log4(("pgmPoolMonitorIsReused: PUSH\n"));
541	return true;
542	case OP_PUSHF:
543	Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
544	return true;
545	case OP_PUSHA:
546	Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
547	return true;
548	case OP_FXSAVE:
549	Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
550	return true;
551	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
552	Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
553	return true;
554	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
555	Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
556	return true;
557	}
558	if ( (pCpu->param1.flags & USE_REG_GEN32)
559	&& (pCpu->param1.base.reg_gen32 == USE_REG_ESP))
560	{
561	Log4(("pgmPoolMonitorIsReused: ESP\n"));
562	return true;
563	}
564
565	//if (pPage->fCR3Mix)
566	// return false;
567	return false;
568	}
569
570
571	/**
572	* Flushes the page being accessed.
573	*
574	* @returns VBox status code suitable for scheduling.
575	* @param pVM The VM handle.
576	* @param pPool The pool.
577	* @param pPage The pool page (head).
578	* @param pCpu The disassembly of the write instruction.
579	* @param pRegFrame The trap register frame.
580	* @param GCPhysFault The fault address as guest physical address.
581	* @param pvFault The fault address.
582	*/
583	static int pgmPoolAccessHandlerFlush(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
584	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
585	{
586	/*
587	* First, do the flushing.
588	*/
589	int rc = pgmPoolMonitorChainFlush(pPool, pPage);
590
591	/*
592	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
593	*/
594	uint32_t cbWritten;
595	int rc2 = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cbWritten);
596	if (VBOX_SUCCESS(rc2))
597	pRegFrame->eip += pCpu->opsize;
598	else if (rc2 == VERR_EM_INTERPRETER)
599	{
600	#ifdef IN_GC
601	if (PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip))
602	{
603	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
604	pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
605	rc = VINF_SUCCESS;
606	STAM_COUNTER_INC(&pPool->StatMonitorGCIntrFailPatch2);
607	}
608	else
609	#endif
610	{
611	rc = VINF_EM_RAW_EMULATE_INSTR;
612	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,EmulateInstr));
613	}
614	}
615	else
616	rc = rc2;
617
618	/* See use in pgmPoolAccessHandlerSimple(). */
619	PGM_INVL_GUEST_TLBS();
620
621	LogFlow(("pgmPoolAccessHandlerPT: returns %Vrc (flushed)\n", rc));
622	return rc;
623
624	}
625
626
627	/**
628	* Handles the STOSD write accesses.
629	*
630	* @returns VBox status code suitable for scheduling.
631	* @param pVM The VM handle.
632	* @param pPool The pool.
633	* @param pPage The pool page (head).
634	* @param pCpu The disassembly of the write instruction.
635	* @param pRegFrame The trap register frame.
636	* @param GCPhysFault The fault address as guest physical address.
637	* @param pvFault The fault address.
638	*/
639	DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
640	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
641	{
642	/*
643	* Increment the modification counter and insert it into the list
644	* of modified pages the first time.
645	*/
646	if (!pPage->cModifications++)
647	pgmPoolMonitorModifiedInsert(pPool, pPage);
648
649	/*
650	* Execute REP STOSD.
651	*
652	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
653	* write situation, meaning that it's safe to write here.
654	*/
655	#ifdef IN_GC
656	uint32_t pu32 = (uint32_t )pvFault;
657	#else
658	RTGCPTR pu32 = pvFault;
659	#endif
660	while (pRegFrame->ecx)
661	{
662	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pu32, NULL);
663	#ifdef IN_GC
664	*pu32++ = pRegFrame->eax;
665	#else
666	PGMPhysWriteGCPhys(pVM, GCPhysFault, &pRegFrame->eax, 4);
667	pu32 += 4;
668	#endif
669	GCPhysFault += 4;
670	pRegFrame->edi += 4;
671	pRegFrame->ecx--;
672	}
673	pRegFrame->eip += pCpu->opsize;
674
675	/* See use in pgmPoolAccessHandlerSimple(). */
676	PGM_INVL_GUEST_TLBS();
677
678	LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
679	return VINF_SUCCESS;
680	}
681
682
683	/**
684	* Handles the simple write accesses.
685	*
686	* @returns VBox status code suitable for scheduling.
687	* @param pVM The VM handle.
688	* @param pPool The pool.
689	* @param pPage The pool page (head).
690	* @param pCpu The disassembly of the write instruction.
691	* @param pRegFrame The trap register frame.
692	* @param GCPhysFault The fault address as guest physical address.
693	* @param pvFault The fault address.
694	*/
695	DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
696	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
697	{
698	/*
699	* Increment the modification counter and insert it into the list
700	* of modified pages the first time.
701	*/
702	if (!pPage->cModifications++)
703	pgmPoolMonitorModifiedInsert(pPool, pPage);
704
705	/*
706	* Clear all the pages. ASSUMES that pvFault is readable.
707	*/
708	pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pvFault, pCpu);
709
710	/*
711	* Interpret the instruction.
712	*/
713	uint32_t cb;
714	int rc = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cb);
715	if (VBOX_SUCCESS(rc))
716	pRegFrame->eip += pCpu->opsize;
717	else if (rc == VERR_EM_INTERPRETER)
718	{
719	# ifdef IN_GC
720	if (PATMIsPatchGCAddr(pVM, (RTGCPTR)(RTGCUINTPTR)pCpu->opaddr))
721	{
722	/* We're not able to handle this in ring-3, so fix the interpreter! */
723	/** @note Should be fine. There's no need to flush the whole thing. */
724	#ifndef DEBUG_sandervl
725	AssertMsgFailed(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv - opcode=%d\n",
726	pRegFrame->cs, (RTGCPTR)pRegFrame->eip, pCpu->pCurInstr->opcode));
727	#endif
728	STAM_COUNTER_INC(&pPool->StatMonitorGCIntrFailPatch1);
729	rc = pgmPoolMonitorChainFlush(pPool, pPage);
730	}
731	else
732	# endif
733	{
734	rc = VINF_EM_RAW_EMULATE_INSTR;
735	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,EmulateInstr));
736	}
737	}
738
739	/*
740	* Quick hack, with logging enabled we're getting stale
741	* code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
742	* Flushing here is BAD and expensive, I think EMInterpretDisasOne will
743	* have to be fixed to support this. But that'll have to wait till next week.
744	*
745	* An alternative is to keep track of the changed PTEs together with the
746	* GCPhys from the guest PT. This may proove expensive though.
747	*
748	* At the moment, it's VITAL that it's done AFTER the instruction interpreting
749	* because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
750	*/
751	PGM_INVL_GUEST_TLBS();
752
753	LogFlow(("pgmPoolAccessHandlerSimple: returns %Vrc cb=%d\n", rc, cb));
754	return rc;
755	}
756
757
758	/**
759	* \#PF Handler callback for PT write accesses.
760	*
761	* @returns VBox status code (appropriate for GC return).
762	* @param pVM VM Handle.
763	* @param uErrorCode CPU Error code.
764	* @param pRegFrame Trap register frame.
765	* NULL on DMA and other non CPU access.
766	* @param pvFault The fault address (cr2).
767	* @param GCPhysFault The GC physical address corresponding to pvFault.
768	* @param pvUser User argument.
769	*/
770	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
771	{
772	STAM_PROFILE_START(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), a);
773	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
774	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
775	LogFlow(("pgmPoolAccessHandler: pvFault=%p pPage=%p:{.idx=%d} GCPhysFault=%VGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
776
777	/*
778	* We should ALWAYS have the list head as user parameter. This
779	* is because we use that page to record the changes.
780	*/
781	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
782
783	/*
784	* Disassemble the faulting instruction.
785	*/
786	DISCPUSTATE Cpu;
787	int rc = EMInterpretDisasOne(pVM, pRegFrame, &Cpu, NULL);
788	AssertRCReturn(rc, rc);
789
790	/*
791	* Check if it's worth dealing with.
792	*/
793	bool fReused = false;
794	if ( ( pPage->cModifications < 48 /** @todo #define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
795	\|\| pPage->fCR3Mix)
796	&& !(fReused = pgmPoolMonitorIsReused(pPage, &Cpu, pvFault))
797	&& !pgmPoolMonitorIsForking(pPool, &Cpu, GCPhysFault & PAGE_OFFSET_MASK))
798	{
799	/*
800	* Simple instructions, no REP prefix.
801	*/
802	if (!(Cpu.prefix & (PREFIX_REP \| PREFIX_REPNE)))
803	{
804	rc = pgmPoolAccessHandlerSimple(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
805	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,Handled), a);
806	return rc;
807	}
808
809	/*
810	* Windows is frequently doing small memset() operations (netio test 4k+).
811	* We have to deal with these or we'll kill the cache and performance.
812	*/
813	if ( Cpu.pCurInstr->opcode == OP_STOSWD
814	&& CPUMGetGuestCPL(pVM, pRegFrame) == 0
815	&& pRegFrame->ecx <= 0x20
816	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
817	&& !((uintptr_t)pvFault & 3)
818	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
819	&& Cpu.mode == CPUMODE_32BIT
820	&& Cpu.opmode == CPUMODE_32BIT
821	&& Cpu.addrmode == CPUMODE_32BIT
822	&& Cpu.prefix == PREFIX_REP
823	&& !pRegFrame->eflags.Bits.u1DF
824	)
825	{
826	rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
827	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,RepStosd), a);
828	return rc;
829	}
830
831	/* REP prefix, don't bother. */
832	STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,RepPrefix));
833	Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x eip=%#x opcode=%d prefix=%#x\n",
834	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, pRegFrame->eip, Cpu.pCurInstr->opcode, Cpu.prefix));
835	}
836
837	/*
838	* Not worth it, so flush it.
839	*
840	* If we considered it to be reused, don't to back to ring-3
841	* to emulate failed instructions since we usually cannot
842	* interpret then. This may be a bit risky, in which case
843	* the reuse detection must be fixed.
844	*/
845	rc = pgmPoolAccessHandlerFlush(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
846	if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
847	rc = VINF_SUCCESS;
848	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,FlushPage), a);
849	return rc;
850	}
851
852	# endif /* !IN_RING3 */
853	#endif /* PGMPOOL_WITH_MONITORING */
854
855
856
857	#ifdef PGMPOOL_WITH_CACHE
858	/**
859	* Inserts a page into the GCPhys hash table.
860	*
861	* @param pPool The pool.
862	* @param pPage The page.
863	*/
864	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
865	{
866	Log3(("pgmPoolHashInsert: %VGp\n", pPage->GCPhys));
867	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
868	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
869	pPage->iNext = pPool->aiHash[iHash];
870	pPool->aiHash[iHash] = pPage->idx;
871	}
872
873
874	/**
875	* Removes a page from the GCPhys hash table.
876	*
877	* @param pPool The pool.
878	* @param pPage The page.
879	*/
880	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
881	{
882	Log3(("pgmPoolHashRemove: %VGp\n", pPage->GCPhys));
883	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
884	if (pPool->aiHash[iHash] == pPage->idx)
885	pPool->aiHash[iHash] = pPage->iNext;
886	else
887	{
888	uint16_t iPrev = pPool->aiHash[iHash];
889	for (;;)
890	{
891	const int16_t i = pPool->aPages[iPrev].iNext;
892	if (i == pPage->idx)
893	{
894	pPool->aPages[iPrev].iNext = pPage->iNext;
895	break;
896	}
897	if (i == NIL_PGMPOOL_IDX)
898	{
899	AssertReleaseMsgFailed(("GCPhys=%VGp idx=%#x\n", pPage->GCPhys, pPage->idx));
900	break;
901	}
902	iPrev = i;
903	}
904	}
905	pPage->iNext = NIL_PGMPOOL_IDX;
906	}
907
908
909	/**
910	* Frees up one cache page.
911	*
912	* @returns VBox status code.
913	* @retval VINF_SUCCESS on success.
914	* @retval VERR_PGM_POOL_CLEARED if the deregistration of a physical handler will cause a light weight pool flush.
915	* @param pPool The pool.
916	* @param iUser The user index.
917	*/
918	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
919	{
920	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
921	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
922
923	/*
924	* Select one page from the tail of the age list.
925	*/
926	uint16_t iToFree = pPool->iAgeTail;
927	if (iToFree == iUser)
928	iToFree = pPool->aPages[iToFree].iAgePrev;
929	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
930	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
931	{
932	uint16_t i = pPool->aPages[iToFree].iAgePrev;
933	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
934	{
935	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
936	continue;
937	iToFree = i;
938	break;
939	}
940	}
941	*/
942	Assert(iToFree != iUser);
943	AssertRelease(iToFree != NIL_PGMPOOL_IDX);
944
945	int rc = pgmPoolFlushPage(pPool, &pPool->aPages[iToFree]);
946	if (rc == VINF_SUCCESS)
947	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
948	return rc;
949	}
950
951
952	/**
953	* Checks if a kind mismatch is really a page being reused
954	* or if it's just normal remappings.
955	*
956	* @returns true if reused and the cached page (enmKind1) should be flushed
957	* @returns false if not reused.
958	* @param enmKind1 The kind of the cached page.
959	* @param enmKind2 The kind of the requested page.
960	*/
961	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
962	{
963	switch (enmKind1)
964	{
965	/*
966	* Never reuse them. There is no remapping in non-paging mode.
967	*/
968	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
969	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
970	return true;
971
972	/*
973	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
974	*/
975	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
976	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
977	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
978	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
979	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
980	switch (enmKind2)
981	{
982	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
983	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
984	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
985	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
986	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
987	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
988	return true;
989	default:
990	return false;
991	}
992
993	/*
994	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
995	*/
996	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
997	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
998	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
999	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1000	switch (enmKind2)
1001	{
1002	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1003	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1004	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1005	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1006	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1007	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1008	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1009	return true;
1010	default:
1011	return false;
1012	}
1013
1014	/*
1015	* These cannot be flushed, and it's common to reuse the PDs as PTs.
1016	*/
1017	case PGMPOOLKIND_ROOT_32BIT_PD:
1018	case PGMPOOLKIND_ROOT_PAE_PD:
1019	case PGMPOOLKIND_ROOT_PDPT:
1020	case PGMPOOLKIND_ROOT_PML4:
1021	return false;
1022
1023	default:
1024	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1025	}
1026	}
1027
1028
1029	/**
1030	* Attempts to satisfy a pgmPoolAlloc request from the cache.
1031	*
1032	* @returns VBox status code.
1033	* @retval VINF_PGM_CACHED_PAGE on success.
1034	* @retval VERR_FILE_NOT_FOUND if not found.
1035	* @param pPool The pool.
1036	* @param GCPhys The GC physical address of the page we're gonna shadow.
1037	* @param enmKind The kind of mapping.
1038	* @param iUser The shadow page pool index of the user table.
1039	* @param iUserTable The index into the user table (shadowed).
1040	* @param ppPage Where to store the pointer to the page.
1041	*/
1042	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint16_t iUserTable, PPPGMPOOLPAGE ppPage)
1043	{
1044	/*
1045	* Look up the GCPhys in the hash.
1046	*/
1047	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1048	Log3(("pgmPoolCacheAlloc: %VGp kind %d iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, enmKind, iUser, iUserTable, i));
1049	if (i != NIL_PGMPOOL_IDX)
1050	{
1051	do
1052	{
1053	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1054	Log3(("pgmPoolCacheAlloc: slot %d found page %VGp\n", i, pPage->GCPhys));
1055	if (pPage->GCPhys == GCPhys)
1056	{
1057	if ((PGMPOOLKIND)pPage->enmKind == enmKind)
1058	{
1059	int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1060	if (VBOX_SUCCESS(rc))
1061	{
1062	*ppPage = pPage;
1063	STAM_COUNTER_INC(&pPool->StatCacheHits);
1064	return VINF_PGM_CACHED_PAGE;
1065	}
1066	return rc;
1067	}
1068
1069	/*
1070	* The kind is different. In some cases we should now flush the page
1071	* as it has been reused, but in most cases this is normal remapping
1072	* of PDs as PT or big pages using the GCPhys field in a slightly
1073	* different way than the other kinds.
1074	*/
1075	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1076	{
1077	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1078	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED will be returned by pgmPoolTracInsert. */
1079	PGM_INVL_GUEST_TLBS(); /* see PT handler. */
1080	break;
1081	}
1082	}
1083
1084	/* next */
1085	i = pPage->iNext;
1086	} while (i != NIL_PGMPOOL_IDX);
1087	}
1088
1089	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%d\n", GCPhys, enmKind));
1090	STAM_COUNTER_INC(&pPool->StatCacheMisses);
1091	return VERR_FILE_NOT_FOUND;
1092	}
1093
1094
1095	/**
1096	* Inserts a page into the cache.
1097	*
1098	* @param pPool The pool.
1099	* @param pPage The cached page.
1100	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1101	*/
1102	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1103	{
1104	/*
1105	* Insert into the GCPhys hash if the page is fit for that.
1106	*/
1107	Assert(!pPage->fCached);
1108	if (fCanBeCached)
1109	{
1110	pPage->fCached = true;
1111	pgmPoolHashInsert(pPool, pPage);
1112	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1113	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1114	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1115	}
1116	else
1117	{
1118	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1119	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1120	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1121	}
1122
1123	/*
1124	* Insert at the head of the age list.
1125	*/
1126	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1127	pPage->iAgeNext = pPool->iAgeHead;
1128	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1129	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1130	else
1131	pPool->iAgeTail = pPage->idx;
1132	pPool->iAgeHead = pPage->idx;
1133	}
1134
1135
1136	/**
1137	* Flushes a cached page.
1138	*
1139	* @param pPool The pool.
1140	* @param pPage The cached page.
1141	*/
1142	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1143	{
1144	Log3(("pgmPoolCacheFlushPage: %VGp\n", pPage->GCPhys));
1145
1146	/*
1147	* Remove the page from the hash.
1148	*/
1149	if (pPage->fCached)
1150	{
1151	pPage->fCached = false;
1152	pgmPoolHashRemove(pPool, pPage);
1153	}
1154	else
1155	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1156
1157	/*
1158	* Remove it from the age list.
1159	*/
1160	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1161	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1162	else
1163	pPool->iAgeTail = pPage->iAgePrev;
1164	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1165	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1166	else
1167	pPool->iAgeHead = pPage->iAgeNext;
1168	pPage->iAgeNext = NIL_PGMPOOL_IDX;
1169	pPage->iAgePrev = NIL_PGMPOOL_IDX;
1170	}
1171	#endif /* PGMPOOL_WITH_CACHE */
1172
1173
1174	#ifdef PGMPOOL_WITH_MONITORING
1175	/**
1176	* Looks for pages sharing the monitor.
1177	*
1178	* @returns Pointer to the head page.
1179	* @returns NULL if not found.
1180	* @param pPool The Pool
1181	* @param pNewPage The page which is going to be monitored.
1182	*/
1183	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1184	{
1185	#ifdef PGMPOOL_WITH_CACHE
1186	/*
1187	* Look up the GCPhys in the hash.
1188	*/
1189	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1190	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1191	if (i == NIL_PGMPOOL_IDX)
1192	return NULL;
1193	do
1194	{
1195	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1196	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1197	&& pPage != pNewPage)
1198	{
1199	switch (pPage->enmKind)
1200	{
1201	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1202	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1203	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1204	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1205	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1206	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1207	case PGMPOOLKIND_ROOT_32BIT_PD:
1208	case PGMPOOLKIND_ROOT_PAE_PD:
1209	case PGMPOOLKIND_ROOT_PDPT:
1210	case PGMPOOLKIND_ROOT_PML4:
1211	{
1212	/* find the head */
1213	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1214	{
1215	Assert(pPage->iMonitoredPrev != pPage->idx);
1216	pPage = &pPool->aPages[pPage->iMonitoredPrev];
1217	}
1218	return pPage;
1219	}
1220
1221	/* ignore, no monitoring. */
1222	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1223	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1224	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1225	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1226	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1227	break;
1228	default:
1229	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
1230	}
1231	}
1232
1233	/* next */
1234	i = pPage->iNext;
1235	} while (i != NIL_PGMPOOL_IDX);
1236	#endif
1237	return NULL;
1238	}
1239
1240	/**
1241	* Enabled write monitoring of a guest page.
1242	*
1243	* @returns VBox status code.
1244	* @retval VINF_SUCCESS on success.
1245	* @retval VERR_PGM_POOL_CLEARED if the registration of the physical handler will cause a light weight pool flush.
1246	* @param pPool The pool.
1247	* @param pPage The cached page.
1248	*/
1249	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1250	{
1251	LogFlow(("pgmPoolMonitorInsert %VGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
1252
1253	/*
1254	* Filter out the relevant kinds.
1255	*/
1256	switch (pPage->enmKind)
1257	{
1258	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1259	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1260	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1261	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1262	case PGMPOOLKIND_ROOT_PDPT:
1263	break;
1264
1265	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1266	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1267	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1268	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1269	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1270	/* Nothing to monitor here. */
1271	return VINF_SUCCESS;
1272
1273	case PGMPOOLKIND_ROOT_32BIT_PD:
1274	case PGMPOOLKIND_ROOT_PAE_PD:
1275	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1276	break;
1277	#endif
1278	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1279	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1280	case PGMPOOLKIND_ROOT_PML4:
1281	default:
1282	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1283	}
1284
1285	/*
1286	* Install handler.
1287	*/
1288	int rc;
1289	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
1290	if (pPageHead)
1291	{
1292	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
1293	Assert(pPageHead->iMonitoredPrev != pPage->idx);
1294	pPage->iMonitoredPrev = pPageHead->idx;
1295	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
1296	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
1297	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
1298	pPageHead->iMonitoredNext = pPage->idx;
1299	rc = VINF_SUCCESS;
1300	}
1301	else
1302	{
1303	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1304	PVM pVM = pPool->CTXSUFF(pVM);
1305	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1306	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
1307	GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
1308	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
1309	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
1310	pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pPage),
1311	pPool->pszAccessHandler);
1312	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
1313	* the heap size should suffice. */
1314	AssertFatalRC(rc);
1315	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1316	rc = VERR_PGM_POOL_CLEARED;
1317	}
1318	pPage->fMonitored = true;
1319	return rc;
1320	}
1321
1322
1323	/**
1324	* Disables write monitoring of a guest page.
1325	*
1326	* @returns VBox status code.
1327	* @retval VINF_SUCCESS on success.
1328	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1329	* @param pPool The pool.
1330	* @param pPage The cached page.
1331	*/
1332	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1333	{
1334	/*
1335	* Filter out the relevant kinds.
1336	*/
1337	switch (pPage->enmKind)
1338	{
1339	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1340	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1341	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1342	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1343	case PGMPOOLKIND_ROOT_PDPT:
1344	break;
1345
1346	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1347	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1348	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1349	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1350	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1351	/* Nothing to monitor here. */
1352	return VINF_SUCCESS;
1353
1354	case PGMPOOLKIND_ROOT_32BIT_PD:
1355	case PGMPOOLKIND_ROOT_PAE_PD:
1356	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1357	break;
1358	#endif
1359	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1360	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1361	case PGMPOOLKIND_ROOT_PML4:
1362	default:
1363	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1364	}
1365
1366	/*
1367	* Remove the page from the monitored list or uninstall it if last.
1368	*/
1369	const PVM pVM = pPool->CTXSUFF(pVM);
1370	int rc;
1371	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1372	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1373	{
1374	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1375	{
1376	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
1377	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
1378	pNewHead->fCR3Mix = pPage->fCR3Mix;
1379	rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
1380	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
1381	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
1382	pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pNewHead),
1383	pPool->pszAccessHandler);
1384	AssertFatalRCSuccess(rc);
1385	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1386	}
1387	else
1388	{
1389	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
1390	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
1391	{
1392	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
1393	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1394	}
1395	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
1396	rc = VINF_SUCCESS;
1397	}
1398	}
1399	else
1400	{
1401	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
1402	AssertFatalRC(rc);
1403	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1404	rc = VERR_PGM_POOL_CLEARED;
1405	}
1406	pPage->fMonitored = false;
1407
1408	/*
1409	* Remove it from the list of modified pages (if in it).
1410	*/
1411	pgmPoolMonitorModifiedRemove(pPool, pPage);
1412
1413	return rc;
1414	}
1415
1416
1417	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1418	/**
1419	* Set or clear the fCR3Mix attribute in a chain of monitored pages.
1420	*
1421	* @param pPool The Pool.
1422	* @param pPage A page in the chain.
1423	* @param fCR3Mix The new fCR3Mix value.
1424	*/
1425	static void pgmPoolMonitorChainChangeCR3Mix(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCR3Mix)
1426	{
1427	/* current */
1428	pPage->fCR3Mix = fCR3Mix;
1429
1430	/* before */
1431	int16_t idx = pPage->iMonitoredPrev;
1432	while (idx != NIL_PGMPOOL_IDX)
1433	{
1434	pPool->aPages[idx].fCR3Mix = fCR3Mix;
1435	idx = pPool->aPages[idx].iMonitoredPrev;
1436	}
1437
1438	/* after */
1439	idx = pPage->iMonitoredNext;
1440	while (idx != NIL_PGMPOOL_IDX)
1441	{
1442	pPool->aPages[idx].fCR3Mix = fCR3Mix;
1443	idx = pPool->aPages[idx].iMonitoredNext;
1444	}
1445	}
1446
1447
1448	/**
1449	* Installs or modifies monitoring of a CR3 page (special).
1450	*
1451	* We're pretending the CR3 page is shadowed by the pool so we can use the
1452	* generic mechanisms in detecting chained monitoring. (This also gives us a
1453	* tast of what code changes are required to really pool CR3 shadow pages.)
1454	*
1455	* @returns VBox status code.
1456	* @param pPool The pool.
1457	* @param idxRoot The CR3 (root) page index.
1458	* @param GCPhysCR3 The (new) CR3 value.
1459	*/
1460	int pgmPoolMonitorMonitorCR3(PPGMPOOL pPool, uint16_t idxRoot, RTGCPHYS GCPhysCR3)
1461	{
1462	Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
1463	PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
1464	LogFlow(("pgmPoolMonitorMonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%VGp, .fMonitored=%d} GCPhysCR3=%VGp\n",
1465	idxRoot, pPage, pPage->GCPhys, pPage->fMonitored, GCPhysCR3));
1466
1467	/*
1468	* The unlikely case where it already matches.
1469	*/
1470	if (pPage->GCPhys == GCPhysCR3)
1471	{
1472	Assert(pPage->fMonitored);
1473	return VINF_SUCCESS;
1474	}
1475
1476	/*
1477	* Flush the current monitoring and remove it from the hash.
1478	*/
1479	int rc = VINF_SUCCESS;
1480	if (pPage->fMonitored)
1481	{
1482	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
1483	rc = pgmPoolMonitorFlush(pPool, pPage);
1484	if (rc == VERR_PGM_POOL_CLEARED)
1485	rc = VINF_SUCCESS;
1486	else
1487	AssertFatalRC(rc);
1488	pgmPoolHashRemove(pPool, pPage);
1489	}
1490
1491	/*
1492	* Monitor the page at the new location and insert it into the hash.
1493	*/
1494	pPage->GCPhys = GCPhysCR3;
1495	int rc2 = pgmPoolMonitorInsert(pPool, pPage);
1496	if (rc2 != VERR_PGM_POOL_CLEARED)
1497	{
1498	AssertFatalRC(rc2);
1499	if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS)
1500	rc = rc2;
1501	}
1502	pgmPoolHashInsert(pPool, pPage);
1503	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, true);
1504	return rc;
1505	}
1506
1507
1508	/**
1509	* Removes the monitoring of a CR3 page (special).
1510	*
1511	* @returns VBox status code.
1512	* @param pPool The pool.
1513	* @param idxRoot The CR3 (root) page index.
1514	*/
1515	int pgmPoolMonitorUnmonitorCR3(PPGMPOOL pPool, uint16_t idxRoot)
1516	{
1517	Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
1518	PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
1519	LogFlow(("pgmPoolMonitorUnmonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%VGp, .fMonitored=%d}\n",
1520	idxRoot, pPage, pPage->GCPhys, pPage->fMonitored));
1521
1522	if (!pPage->fMonitored)
1523	return VINF_SUCCESS;
1524
1525	pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
1526	int rc = pgmPoolMonitorFlush(pPool, pPage);
1527	if (rc != VERR_PGM_POOL_CLEARED)
1528	AssertFatalRC(rc);
1529	else
1530	rc = VINF_SUCCESS;
1531	pgmPoolHashRemove(pPool, pPage);
1532	Assert(!pPage->fMonitored);
1533	pPage->GCPhys = NIL_RTGCPHYS;
1534	return rc;
1535	}
1536	#endif /* PGMPOOL_WITH_MIXED_PT_CR3 */
1537
1538
1539	/**
1540	* Inserts the page into the list of modified pages.
1541	*
1542	* @param pPool The pool.
1543	* @param pPage The page.
1544	*/
1545	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1546	{
1547	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
1548	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
1549	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
1550	&& pPool->iModifiedHead != pPage->idx,
1551	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
1552	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
1553	pPool->iModifiedHead, pPool->cModifiedPages));
1554
1555	pPage->iModifiedNext = pPool->iModifiedHead;
1556	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
1557	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
1558	pPool->iModifiedHead = pPage->idx;
1559	pPool->cModifiedPages++;
1560	#ifdef VBOX_WITH_STATISTICS
1561	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
1562	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
1563	#endif
1564	}
1565
1566
1567	/**
1568	* Removes the page from the list of modified pages and resets the
1569	* moficiation counter.
1570	*
1571	* @param pPool The pool.
1572	* @param pPage The page which is believed to be in the list of modified pages.
1573	*/
1574	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1575	{
1576	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
1577	if (pPool->iModifiedHead == pPage->idx)
1578	{
1579	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1580	pPool->iModifiedHead = pPage->iModifiedNext;
1581	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1582	{
1583	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
1584	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1585	}
1586	pPool->cModifiedPages--;
1587	}
1588	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
1589	{
1590	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
1591	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1592	{
1593	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
1594	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1595	}
1596	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1597	pPool->cModifiedPages--;
1598	}
1599	else
1600	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1601	pPage->cModifications = 0;
1602	}
1603
1604
1605	/**
1606	* Zaps the list of modified pages, resetting their modification counters in the process.
1607	*
1608	* @param pVM The VM handle.
1609	*/
1610	void pgmPoolMonitorModifiedClearAll(PVM pVM)
1611	{
1612	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1613	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
1614
1615	unsigned cPages = 0; NOREF(cPages);
1616	uint16_t idx = pPool->iModifiedHead;
1617	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1618	while (idx != NIL_PGMPOOL_IDX)
1619	{
1620	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
1621	idx = pPage->iModifiedNext;
1622	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1623	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1624	pPage->cModifications = 0;
1625	Assert(++cPages);
1626	}
1627	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
1628	pPool->cModifiedPages = 0;
1629	}
1630
1631
1632	/**
1633	* Clear all shadow pages and clear all modification counters.
1634	*
1635	* @param pVM The VM handle.
1636	* @remark Should only be used when monitoring is available, thus placed in
1637	* the PGMPOOL_WITH_MONITORING #ifdef.
1638	*/
1639	void pgmPoolClearAll(PVM pVM)
1640	{
1641	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1642	STAM_PROFILE_START(&pPool->StatClearAll, c);
1643	LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
1644
1645	/*
1646	* Iterate all the pages until we've encountered all that in use.
1647	* This is simple but not quite optimal solution.
1648	*/
1649	unsigned cModifiedPages = 0; NOREF(cModifiedPages);
1650	unsigned cLeft = pPool->cUsedPages;
1651	unsigned iPage = pPool->cCurPages;
1652	while (--iPage >= PGMPOOL_IDX_FIRST)
1653	{
1654	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1655	if (pPage->GCPhys != NIL_RTGCPHYS)
1656	{
1657	switch (pPage->enmKind)
1658	{
1659	/*
1660	* We only care about shadow page tables.
1661	*/
1662	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1663	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1664	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1665	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1666	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1667	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1668	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1669	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1670	{
1671	#ifdef PGMPOOL_WITH_USER_TRACKING
1672	if (pPage->cPresent)
1673	#endif
1674	{
1675	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
1676	STAM_PROFILE_START(&pPool->StatZeroPage, z);
1677	ASMMemZeroPage(pvShw);
1678	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
1679	#ifdef PGMPOOL_WITH_USER_TRACKING
1680	pPage->cPresent = 0;
1681	pPage->iFirstPresent = ~0;
1682	#endif
1683	}
1684	}
1685	/* fall thru */
1686
1687	default:
1688	Assert(!pPage->cModifications \|\| ++cModifiedPages);
1689	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1690	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1691	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1692	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1693	pPage->cModifications = 0;
1694	break;
1695
1696	}
1697	if (!--cLeft)
1698	break;
1699	}
1700	}
1701
1702	/* swipe the special pages too. */
1703	for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
1704	{
1705	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1706	if (pPage->GCPhys != NIL_RTGCPHYS)
1707	{
1708	Assert(!pPage->cModifications \|\| ++cModifiedPages);
1709	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1710	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
1711	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1712	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1713	pPage->cModifications = 0;
1714	}
1715	}
1716
1717	AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
1718	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1719	pPool->cModifiedPages = 0;
1720
1721	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1722	/*
1723	* Clear all the GCPhys links and rebuild the phys ext free list.
1724	*/
1725	for (PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
1726	pRam;
1727	pRam = CTXALLSUFF(pRam->pNext))
1728	{
1729	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1730	while (iPage-- > 0)
1731	pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
1732	}
1733
1734	pPool->iPhysExtFreeHead = 0;
1735	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
1736	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
1737	for (unsigned i = 0; i < cMaxPhysExts; i++)
1738	{
1739	paPhysExts[i].iNext = i + 1;
1740	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
1741	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
1742	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
1743	}
1744	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
1745	#endif
1746
1747
1748	pPool->cPresent = 0;
1749	STAM_PROFILE_STOP(&pPool->StatClearAll, c);
1750	}
1751	#endif /* PGMPOOL_WITH_MONITORING */
1752
1753
1754	#ifdef PGMPOOL_WITH_USER_TRACKING
1755	/**
1756	* Frees up at least one user entry.
1757	*
1758	* @returns VBox status code.
1759	* @retval VINF_SUCCESS if successfully added.
1760	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1761	* @param pPool The pool.
1762	* @param iUser The user index.
1763	*/
1764	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
1765	{
1766	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
1767	#ifdef PGMPOOL_WITH_CACHE
1768	/*
1769	* Just free cached pages in a braindead fashion.
1770	*/
1771	/** @todo walk the age list backwards and free the first with usage. */
1772	int rc = VINF_SUCCESS;
1773	do
1774	{
1775	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
1776	if (VBOX_FAILURE(rc2) && rc == VINF_SUCCESS)
1777	rc = rc2;
1778	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
1779	return rc;
1780	#else
1781	/*
1782	* Lazy approach.
1783	*/
1784	pgmPoolFlushAllInt(pPool);
1785	return VERR_PGM_POOL_FLUSHED;
1786	#endif
1787	}
1788
1789
1790	/**
1791	* Inserts a page into the cache.
1792	*
1793	* This will create user node for the page, insert it into the GCPhys
1794	* hash, and insert it into the age list.
1795	*
1796	* @returns VBox status code.
1797	* @retval VINF_SUCCESS if successfully added.
1798	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1799	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1800	* @param pPool The pool.
1801	* @param pPage The cached page.
1802	* @param GCPhys The GC physical address of the page we're gonna shadow.
1803	* @param iUser The user index.
1804	* @param iUserTable The user table index.
1805	*/
1806	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint16_t iUserTable)
1807	{
1808	int rc = VINF_SUCCESS;
1809	PPGMPOOLUSER pUser = pPool->CTXSUFF(paUsers);
1810
1811	LogFlow(("pgmPoolTrackInsert iUser %d iUserTable %d\n", iUser, iUserTable));
1812
1813	/*
1814	* Find free a user node.
1815	*/
1816	uint16_t i = pPool->iUserFreeHead;
1817	if (i == NIL_PGMPOOL_USER_INDEX)
1818	{
1819	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
1820	if (VBOX_FAILURE(rc))
1821	return rc;
1822	i = pPool->iUserFreeHead;
1823	}
1824
1825	/*
1826	* Unlink the user node from the free list,
1827	* initialize and insert it into the user list.
1828	*/
1829	pPool->iUserFreeHead = pUser[i].iNext;
1830	pUser[i].iNext = NIL_PGMPOOL_USER_INDEX;
1831	pUser[i].iUser = iUser;
1832	pUser[i].iUserTable = iUserTable;
1833	pPage->iUserHead = i;
1834
1835	/*
1836	* Insert into cache and enable monitoring of the guest page if enabled.
1837	*
1838	* Until we implement caching of all levels, including the CR3 one, we'll
1839	* have to make sure we don't try monitor & cache any recursive reuse of
1840	* a monitored CR3 page. Because all windows versions are doing this we'll
1841	* have to be able to do combined access monitoring, CR3 + PT and
1842	* PD + PT (guest PAE).
1843	*
1844	* Update:
1845	* We're now cooperating with the CR3 monitor if an uncachable page is found.
1846	*/
1847	#if defined(PGMPOOL_WITH_MONITORING) \|\| defined(PGMPOOL_WITH_CACHE)
1848	# ifdef PGMPOOL_WITH_MIXED_PT_CR3
1849	const bool fCanBeMonitored = true;
1850	# else
1851	bool fCanBeMonitored = pPool->CTXSUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
1852	\|\| (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTXSUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
1853	\|\| pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
1854	# endif
1855	# ifdef PGMPOOL_WITH_CACHE
1856	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
1857	# endif
1858	if (fCanBeMonitored)
1859	{
1860	# ifdef PGMPOOL_WITH_MONITORING
1861	rc = pgmPoolMonitorInsert(pPool, pPage);
1862	if (rc == VERR_PGM_POOL_CLEARED)
1863	{
1864	/* 'Failed' - free the usage, and keep it in the cache (if enabled). */
1865	# ifndef PGMPOOL_WITH_CACHE
1866	pgmPoolMonitorFlush(pPool, pPage);
1867	rc = VERR_PGM_POOL_FLUSHED;
1868	# endif
1869	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
1870	pUser[i].iNext = pPool->iUserFreeHead;
1871	pUser[i].iUser = NIL_PGMPOOL_IDX;
1872	pPool->iUserFreeHead = i;
1873	}
1874	}
1875	# endif
1876	#endif /* PGMPOOL_WITH_MONITORING */
1877	return rc;
1878	}
1879
1880
1881	# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
1882	/**
1883	* Adds a user reference to a page.
1884	*
1885	* This will
1886	* This will move the page to the head of the
1887	*
1888	* @returns VBox status code.
1889	* @retval VINF_SUCCESS if successfully added.
1890	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1891	* @param pPool The pool.
1892	* @param pPage The cached page.
1893	* @param iUser The user index.
1894	* @param iUserTable The user table.
1895	*/
1896	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
1897	{
1898	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
1899
1900	LogFlow(("pgmPoolTrackAddUser iUser %d iUserTable %d\n", iUser, iUserTable));
1901	# ifdef VBOX_STRICT
1902	/*
1903	* Check that the entry doesn't already exists.
1904	*/
1905	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
1906	{
1907	uint16_t i = pPage->iUserHead;
1908	do
1909	{
1910	Assert(i < pPool->cMaxUsers);
1911	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
1912	i = paUsers[i].iNext;
1913	} while (i != NIL_PGMPOOL_USER_INDEX);
1914	}
1915	# endif
1916
1917	/*
1918	* Allocate a user node.
1919	*/
1920	uint16_t i = pPool->iUserFreeHead;
1921	if (i == NIL_PGMPOOL_USER_INDEX)
1922	{
1923	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
1924	if (VBOX_FAILURE(rc))
1925	return rc;
1926	i = pPool->iUserFreeHead;
1927	}
1928	pPool->iUserFreeHead = paUsers[i].iNext;
1929
1930	/*
1931	* Initialize the user node and insert it.
1932	*/
1933	paUsers[i].iNext = pPage->iUserHead;
1934	paUsers[i].iUser = iUser;
1935	paUsers[i].iUserTable = iUserTable;
1936	pPage->iUserHead = i;
1937
1938	# ifdef PGMPOOL_WITH_CACHE
1939	/*
1940	* Tell the cache to update its replacement stats for this page.
1941	*/
1942	pgmPoolCacheUsed(pPool, pPage);
1943	# endif
1944	return VINF_SUCCESS;
1945	}
1946	# endif /* PGMPOOL_WITH_CACHE */
1947
1948
1949	/**
1950	* Frees a user record associated with a page.
1951	*
1952	* This does not clear the entry in the user table, it simply replaces the
1953	* user record to the chain of free records.
1954	*
1955	* @param pPool The pool.
1956	* @param HCPhys The HC physical address of the shadow page.
1957	* @param iUser The shadow page pool index of the user table.
1958	* @param iUserTable The index into the user table (shadowed).
1959	*/
1960	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
1961	{
1962	/*
1963	* Unlink and free the specified user entry.
1964	*/
1965	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
1966
1967	/* Special: For PAE and 32-bit paging, there are usually no more than one user. */
1968	uint16_t i = pPage->iUserHead;
1969	if ( i != NIL_PGMPOOL_USER_INDEX
1970	&& paUsers[i].iUser == iUser
1971	&& paUsers[i].iUserTable == iUserTable)
1972	{
1973	pPage->iUserHead = paUsers[i].iNext;
1974
1975	paUsers[i].iUser = NIL_PGMPOOL_IDX;
1976	paUsers[i].iNext = pPool->iUserFreeHead;
1977	pPool->iUserFreeHead = i;
1978	return;
1979	}
1980
1981	/* General: Linear search. */
1982	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
1983	while (i != NIL_PGMPOOL_USER_INDEX)
1984	{
1985	if ( paUsers[i].iUser == iUser
1986	&& paUsers[i].iUserTable == iUserTable)
1987	{
1988	if (iPrev != NIL_PGMPOOL_USER_INDEX)
1989	paUsers[iPrev].iNext = paUsers[i].iNext;
1990	else
1991	pPage->iUserHead = paUsers[i].iNext;
1992
1993	paUsers[i].iUser = NIL_PGMPOOL_IDX;
1994	paUsers[i].iNext = pPool->iUserFreeHead;
1995	pPool->iUserFreeHead = i;
1996	return;
1997	}
1998	iPrev = i;
1999	i = paUsers[i].iNext;
2000	}
2001
2002	/* Fatal: didn't find it */
2003	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%VGp\n",
2004	iUser, iUserTable, pPage->GCPhys));
2005	}
2006
2007
2008	/**
2009	* Gets the entry size of a shadow table.
2010	*
2011	* @param enmKind The kind of page.
2012	*
2013	* @returns The size of the entry in bytes. That is, 4 or 8.
2014	* @returns If the kind is not for a table, an assertion is raised and 0 is
2015	* returned.
2016	*/
2017	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2018	{
2019	switch (enmKind)
2020	{
2021	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2022	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2023	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2024	case PGMPOOLKIND_ROOT_32BIT_PD:
2025	return 4;
2026
2027	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2028	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2029	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2030	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2031	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2032	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2033	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2034	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2035	case PGMPOOLKIND_ROOT_PAE_PD:
2036	case PGMPOOLKIND_ROOT_PDPT:
2037	case PGMPOOLKIND_ROOT_PML4:
2038	return 8;
2039
2040	default:
2041	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2042	}
2043	}
2044
2045
2046	/**
2047	* Gets the entry size of a guest table.
2048	*
2049	* @param enmKind The kind of page.
2050	*
2051	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
2052	* @returns If the kind is not for a table, an assertion is raised and 0 is
2053	* returned.
2054	*/
2055	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2056	{
2057	switch (enmKind)
2058	{
2059	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2060	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2061	case PGMPOOLKIND_ROOT_32BIT_PD:
2062	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2063	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2064	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2065	return 4;
2066
2067	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2068	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2069	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2070	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2071	case PGMPOOLKIND_ROOT_PAE_PD:
2072	case PGMPOOLKIND_ROOT_PDPT:
2073	case PGMPOOLKIND_ROOT_PML4:
2074	return 8;
2075
2076	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2077	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2078	/** @todo can we return 0? (nobody is calling this...) */
2079	return 0;
2080
2081	default:
2082	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2083	}
2084	}
2085
2086
2087	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2088	/**
2089	* Scans one shadow page table for mappings of a physical page.
2090	*
2091	* @param pVM The VM handle.
2092	* @param pPhysPage The guest page in question.
2093	* @param iShw The shadow page table.
2094	* @param cRefs The number of references made in that PT.
2095	*/
2096	static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2097	{
2098	LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2099	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2100
2101	/*
2102	* Assert sanity.
2103	*/
2104	Assert(cRefs == 1);
2105	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2106	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2107
2108	/*
2109	* Then, clear the actual mappings to the page in the shadow PT.
2110	*/
2111	switch (pPage->enmKind)
2112	{
2113	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2114	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2115	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2116	{
2117	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2118	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2119	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2120	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2121	{
2122	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2123	pPT->a[i].u = 0;
2124	cRefs--;
2125	if (!cRefs)
2126	return;
2127	}
2128	#if defined(DEBUG) && !defined(IN_RING0) ///@todo RTLogPrintf is missing in R0.
2129	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2130	for (unsigned i = 0; i < ELEMENTS(pPT->a); i++)
2131	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2132	{
2133	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2134	pPT->a[i].u = 0;
2135	}
2136	#endif
2137	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2138	break;
2139	}
2140
2141	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2142	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2143	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2144	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2145	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2146	{
2147	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2148	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2149	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2150	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2151	{
2152	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2153	pPT->a[i].u = 0;
2154	cRefs--;
2155	if (!cRefs)
2156	return;
2157	}
2158	#if defined(DEBUG) && !defined(IN_RING0) ///@todo RTLogPrintf is missing in R0.
2159	RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2160	for (unsigned i = 0; i < ELEMENTS(pPT->a); i++)
2161	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2162	{
2163	RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2164	pPT->a[i].u = 0;
2165	}
2166	#endif
2167	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2168	break;
2169	}
2170
2171	default:
2172	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
2173	}
2174	}
2175
2176
2177	/**
2178	* Scans one shadow page table for mappings of a physical page.
2179	*
2180	* @param pVM The VM handle.
2181	* @param pPhysPage The guest page in question.
2182	* @param iShw The shadow page table.
2183	* @param cRefs The number of references made in that PT.
2184	*/
2185	void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2186	{
2187	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool); NOREF(pPool);
2188	LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2189	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
2190	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
2191	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2192	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
2193	}
2194
2195
2196	/**
2197	* Flushes a list of shadow page tables mapping the same physical page.
2198	*
2199	* @param pVM The VM handle.
2200	* @param pPhysPage The guest page in question.
2201	* @param iPhysExt The physical cross reference extent list to flush.
2202	*/
2203	void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
2204	{
2205	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2206	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
2207	LogFlow(("pgmPoolTrackFlushGCPhysPTs: HCPhys=%RHp iPhysExt\n", pPhysPage->HCPhys, iPhysExt));
2208
2209	const uint16_t iPhysExtStart = iPhysExt;
2210	PPGMPOOLPHYSEXT pPhysExt;
2211	do
2212	{
2213	Assert(iPhysExt < pPool->cMaxPhysExts);
2214	pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2215	for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2216	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
2217	{
2218	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
2219	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2220	}
2221
2222	/* next */
2223	iPhysExt = pPhysExt->iNext;
2224	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2225
2226	/* insert the list into the free list and clear the ram range entry. */
2227	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2228	pPool->iPhysExtFreeHead = iPhysExtStart;
2229	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2230
2231	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
2232	}
2233	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2234
2235
2236	/**
2237	* Scans all shadow page tables for mappings of a physical page.
2238	*
2239	* This may be slow, but it's most likely more efficient than cleaning
2240	* out the entire page pool / cache.
2241	*
2242	* @returns VBox status code.
2243	* @retval VINF_SUCCESS if all references has been successfully cleared.
2244	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
2245	* a page pool cleaning.
2246	*
2247	* @param pVM The VM handle.
2248	* @param pPhysPage The guest page in question.
2249	*/
2250	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
2251	{
2252	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2253	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2254	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d HCPhys=%RHp\n",
2255	pPool->cUsedPages, pPool->cPresent, pPhysPage->HCPhys));
2256
2257	#if 1
2258	/*
2259	* There is a limit to what makes sense.
2260	*/
2261	if (pPool->cPresent > 1024)
2262	{
2263	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
2264	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2265	return VINF_PGM_GCPHYS_ALIASED;
2266	}
2267	#endif
2268
2269	/*
2270	* Iterate all the pages until we've encountered all that in use.
2271	* This is simple but not quite optimal solution.
2272	*/
2273	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
2274	const uint32_t u32 = u64;
2275	unsigned cLeft = pPool->cUsedPages;
2276	unsigned iPage = pPool->cCurPages;
2277	while (--iPage >= PGMPOOL_IDX_FIRST)
2278	{
2279	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2280	if (pPage->GCPhys != NIL_RTGCPHYS)
2281	{
2282	switch (pPage->enmKind)
2283	{
2284	/*
2285	* We only care about shadow page tables.
2286	*/
2287	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2288	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2289	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2290	{
2291	unsigned cPresent = pPage->cPresent;
2292	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2293	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2294	if (pPT->a[i].n.u1Present)
2295	{
2296	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
2297	{
2298	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
2299	pPT->a[i].u = 0;
2300	}
2301	if (!--cPresent)
2302	break;
2303	}
2304	break;
2305	}
2306
2307	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2308	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2309	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2310	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2311	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2312	{
2313	unsigned cPresent = pPage->cPresent;
2314	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2315	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2316	if (pPT->a[i].n.u1Present)
2317	{
2318	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
2319	{
2320	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
2321	pPT->a[i].u = 0;
2322	}
2323	if (!--cPresent)
2324	break;
2325	}
2326	break;
2327	}
2328	}
2329	if (!--cLeft)
2330	break;
2331	}
2332	}
2333
2334	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2335	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2336	return VINF_SUCCESS;
2337	}
2338
2339
2340	/**
2341	* Clears the user entry in a user table.
2342	*
2343	* This is used to remove all references to a page when flushing it.
2344	*/
2345	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
2346	{
2347	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
2348	Assert(pUser->iUser < pPool->cCurPages);
2349
2350	/*
2351	* Map the user page.
2352	*/
2353	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
2354	union
2355	{
2356	uint64_t *pau64;
2357	uint32_t *pau32;
2358	} u;
2359	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pUserPage);
2360
2361	#ifdef VBOX_STRICT
2362	/*
2363	* Some sanity checks.
2364	*/
2365	switch (pUserPage->enmKind)
2366	{
2367	case PGMPOOLKIND_ROOT_32BIT_PD:
2368	Assert(!(u.pau32[pUser->iUser] & PGM_PDFLAGS_MAPPING));
2369	Assert(pUser->iUserTable < X86_PG_ENTRIES);
2370	break;
2371	case PGMPOOLKIND_ROOT_PAE_PD:
2372	Assert(!(u.pau64[pUser->iUser] & PGM_PDFLAGS_MAPPING));
2373	Assert(pUser->iUserTable < 2048 && pUser->iUser == PGMPOOL_IDX_PAE_PD);
2374	break;
2375	case PGMPOOLKIND_ROOT_PDPT:
2376	Assert(!(u.pau64[pUser->iUserTable] & PGM_PLXFLAGS_PERMANENT));
2377	Assert(pUser->iUserTable < 4);
2378	break;
2379	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2380	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2381	Assert(pUser->iUserTable < X86_PG_PAE_ENTRIES);
2382	break;
2383	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2384	case PGMPOOLKIND_ROOT_PML4:
2385	Assert(!(u.pau64[pUser->iUserTable] & PGM_PLXFLAGS_PERMANENT));
2386	Assert(pUser->iUserTable < X86_PG_PAE_ENTRIES);
2387	break;
2388	default:
2389	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
2390	break;
2391	}
2392	#endif /* VBOX_STRICT */
2393
2394	/*
2395	* Clear the entry in the user page.
2396	*/
2397	switch (pUserPage->enmKind)
2398	{
2399	/* 32-bit entries */
2400	case PGMPOOLKIND_ROOT_32BIT_PD:
2401	u.pau32[pUser->iUserTable] = 0;
2402	break;
2403
2404	/* 64-bit entries */
2405	case PGMPOOLKIND_ROOT_PAE_PD:
2406	case PGMPOOLKIND_ROOT_PDPT:
2407	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2408	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2409	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2410	case PGMPOOLKIND_ROOT_PML4:
2411	u.pau64[pUser->iUserTable] = 0;
2412	break;
2413
2414	default:
2415	AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
2416	}
2417	}
2418
2419
2420	/**
2421	* Clears all users of a page.
2422	*/
2423	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2424	{
2425	/*
2426	* Free all the user records.
2427	*/
2428	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
2429	uint16_t i = pPage->iUserHead;
2430	while (i != NIL_PGMPOOL_USER_INDEX)
2431	{
2432	/* Clear enter in user table. */
2433	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
2434
2435	/* Free it. */
2436	const uint16_t iNext = paUsers[i].iNext;
2437	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2438	paUsers[i].iNext = pPool->iUserFreeHead;
2439	pPool->iUserFreeHead = i;
2440
2441	/* Next. */
2442	i = iNext;
2443	}
2444	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2445	}
2446
2447
2448	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2449	/**
2450	* Allocates a new physical cross reference extent.
2451	*
2452	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
2453	* @param pVM The VM handle.
2454	* @param piPhysExt Where to store the phys ext index.
2455	*/
2456	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
2457	{
2458	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2459	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
2460	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
2461	{
2462	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
2463	return NULL;
2464	}
2465	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2466	pPool->iPhysExtFreeHead = pPhysExt->iNext;
2467	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2468	*piPhysExt = iPhysExt;
2469	return pPhysExt;
2470	}
2471
2472
2473	/**
2474	* Frees a physical cross reference extent.
2475	*
2476	* @param pVM The VM handle.
2477	* @param iPhysExt The extent to free.
2478	*/
2479	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
2480	{
2481	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2482	Assert(iPhysExt < pPool->cMaxPhysExts);
2483	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2484	for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2485	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2486	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2487	pPool->iPhysExtFreeHead = iPhysExt;
2488	}
2489
2490
2491	/**
2492	* Frees a physical cross reference extent.
2493	*
2494	* @param pVM The VM handle.
2495	* @param iPhysExt The extent to free.
2496	*/
2497	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
2498	{
2499	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2500
2501	const uint16_t iPhysExtStart = iPhysExt;
2502	PPGMPOOLPHYSEXT pPhysExt;
2503	do
2504	{
2505	Assert(iPhysExt < pPool->cMaxPhysExts);
2506	pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2507	for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2508	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2509
2510	/* next */
2511	iPhysExt = pPhysExt->iNext;
2512	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2513
2514	pPhysExt->iNext = pPool->iPhysExtFreeHead;
2515	pPool->iPhysExtFreeHead = iPhysExtStart;
2516	}
2517
2518	/**
2519	* Insert a reference into a list of physical cross reference extents.
2520	*
2521	* @returns The new ram range flags (top 16-bits).
2522	*
2523	* @param pVM The VM handle.
2524	* @param iPhysExt The physical extent index of the list head.
2525	* @param iShwPT The shadow page table index.
2526	*
2527	*/
2528	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
2529	{
2530	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2531	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
2532
2533	/* special common case. */
2534	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
2535	{
2536	paPhysExts[iPhysExt].aidx[2] = iShwPT;
2537	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
2538	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{,,%d}\n", iPhysExt, iShwPT));
2539	return iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2540	}
2541
2542	/* general treatment. */
2543	const uint16_t iPhysExtStart = iPhysExt;
2544	unsigned cMax = 15;
2545	for (;;)
2546	{
2547	Assert(iPhysExt < pPool->cMaxPhysExts);
2548	for (unsigned i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2549	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
2550	{
2551	paPhysExts[iPhysExt].aidx[i] = iShwPT;
2552	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
2553	LogFlow(("pgmPoolTrackPhysExtAddref: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
2554	return iPhysExtStart \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2555	}
2556	if (!--cMax)
2557	{
2558	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
2559	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
2560	LogFlow(("pgmPoolTrackPhysExtAddref: overflow (1) iShwPT=%d\n", iShwPT));
2561	return MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2562	}
2563	}
2564
2565	/* add another extent to the list. */
2566	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
2567	if (!pNew)
2568	{
2569	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
2570	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
2571	return MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2572	}
2573	pNew->iNext = iPhysExtStart;
2574	pNew->aidx[0] = iShwPT;
2575	LogFlow(("pgmPoolTrackPhysExtAddref: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
2576	return iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2577	}
2578
2579
2580	/**
2581	* Add a reference to guest physical page where extents are in use.
2582	*
2583	* @returns The new ram range flags (top 16-bits).
2584	*
2585	* @param pVM The VM handle.
2586	* @param u16 The ram range flags (top 16-bits).
2587	* @param iShwPT The shadow page table index.
2588	*/
2589	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
2590	{
2591	if ((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) != MM_RAM_FLAGS_CREFS_PHYSEXT)
2592	{
2593	/*
2594	* Convert to extent list.
2595	*/
2596	Assert((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) == 1);
2597	uint16_t iPhysExt;
2598	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
2599	if (pPhysExt)
2600	{
2601	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT));
2602	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
2603	pPhysExt->aidx[0] = u16 & MM_RAM_FLAGS_IDX_MASK;
2604	pPhysExt->aidx[1] = iShwPT;
2605	u16 = iPhysExt \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2606	}
2607	else
2608	u16 = MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2609	}
2610	else if (u16 != (MM_RAM_FLAGS_IDX_OVERFLOWED \| (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT))))
2611	{
2612	/*
2613	* Insert into the extent list.
2614	*/
2615	u16 = pgmPoolTrackPhysExtInsert(pVM, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT);
2616	}
2617	else
2618	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
2619	return u16;
2620	}
2621
2622
2623	/**
2624	* Clear references to guest physical memory.
2625	*
2626	* @param pPool The pool.
2627	* @param pPage The page.
2628	* @param pPhysPage Pointer to the aPages entry in the ram range.
2629	*/
2630	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
2631	{
2632	const unsigned cRefs = pPhysPage->HCPhys >> MM_RAM_FLAGS_CREFS_SHIFT; /** @todo PAGE FLAGS */
2633	AssertFatalMsg(cRefs == MM_RAM_FLAGS_CREFS_PHYSEXT, ("cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
2634
2635	uint16_t iPhysExt = (pPhysPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT) & MM_RAM_FLAGS_IDX_MASK;
2636	if (iPhysExt != MM_RAM_FLAGS_IDX_OVERFLOWED)
2637	{
2638	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
2639	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
2640	do
2641	{
2642	Assert(iPhysExt < pPool->cMaxPhysExts);
2643
2644	/*
2645	* Look for the shadow page and check if it's all freed.
2646	*/
2647	for (unsigned i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2648	{
2649	if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
2650	{
2651	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
2652
2653	for (i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2654	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
2655	{
2656	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
2657	return;
2658	}
2659
2660	/* we can free the node. */
2661	PVM pVM = pPool->CTXSUFF(pVM);
2662	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
2663	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
2664	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
2665	{
2666	/* lonely node */
2667	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2668	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d lonely\n", pPhysPage->HCPhys, pPage->idx));
2669	pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2670	}
2671	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
2672	{
2673	/* head */
2674	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d head\n", pPhysPage->HCPhys, pPage->idx));
2675	pPhysPage->HCPhys = (pPhysPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) /** @todo PAGE FLAGS */
2676	\| ((uint64_t)MM_RAM_FLAGS_CREFS_PHYSEXT << MM_RAM_FLAGS_CREFS_SHIFT)
2677	\| ((uint64_t)iPhysExtNext << MM_RAM_FLAGS_IDX_SHIFT);
2678	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2679	}
2680	else
2681	{
2682	/* in list */
2683	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
2684	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
2685	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2686	}
2687	iPhysExt = iPhysExtNext;
2688	return;
2689	}
2690	}
2691
2692	/* next */
2693	iPhysExtPrev = iPhysExt;
2694	iPhysExt = paPhysExts[iPhysExt].iNext;
2695	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2696
2697	AssertFatalMsgFailed(("not-found! cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
2698	}
2699	else /* nothing to do */
2700	LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64\n", pPhysPage->HCPhys));
2701	}
2702
2703
2704
2705	/**
2706	* Clear references to guest physical memory.
2707	*
2708	* This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
2709	* is assumed to be correct, so the linear search can be skipped and we can assert
2710	* at an earlier point.
2711	*
2712	* @param pPool The pool.
2713	* @param pPage The page.
2714	* @param HCPhys The host physical address corresponding to the guest page.
2715	* @param GCPhys The guest physical address corresponding to HCPhys.
2716	*/
2717	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
2718	{
2719	/*
2720	* Walk range list.
2721	*/
2722	PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2723	while (pRam)
2724	{
2725	RTGCPHYS off = GCPhys - pRam->GCPhys;
2726	if (off < pRam->cb)
2727	{
2728	/* does it match? */
2729	const unsigned iPage = off >> PAGE_SHIFT;
2730	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
2731	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2732	{
2733	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2734	return;
2735	}
2736	break;
2737	}
2738	pRam = CTXALLSUFF(pRam->pNext);
2739	}
2740	AssertFatalMsgFailed(("HCPhys=%VHp GCPhys=%VGp\n", HCPhys, GCPhys));
2741	}
2742
2743
2744	/**
2745	* Clear references to guest physical memory.
2746	*
2747	* @param pPool The pool.
2748	* @param pPage The page.
2749	* @param HCPhys The host physical address corresponding to the guest page.
2750	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
2751	*/
2752	static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
2753	{
2754	/*
2755	* Walk range list.
2756	*/
2757	PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2758	while (pRam)
2759	{
2760	RTGCPHYS off = GCPhysHint - pRam->GCPhys;
2761	if (off < pRam->cb)
2762	{
2763	/* does it match? */
2764	const unsigned iPage = off >> PAGE_SHIFT;
2765	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
2766	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2767	{
2768	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2769	return;
2770	}
2771	break;
2772	}
2773	pRam = CTXALLSUFF(pRam->pNext);
2774	}
2775
2776	/*
2777	* Damn, the hint didn't work. We'll have to do an expensive linear search.
2778	*/
2779	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
2780	pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2781	while (pRam)
2782	{
2783	unsigned iPage = pRam->cb >> PAGE_SHIFT;
2784	while (iPage-- > 0)
2785	{
2786	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2787	{
2788	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%VHp GCPhysHint=%VGp GCPhysReal=%VGp\n",
2789	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
2790	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2791	return;
2792	}
2793	}
2794	pRam = CTXALLSUFF(pRam->pNext);
2795	}
2796
2797	AssertFatalMsgFailed(("HCPhys=%VHp GCPhysHint=%VGp\n", HCPhys, GCPhysHint));
2798	}
2799
2800
2801	/**
2802	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
2803	*
2804	* @param pPool The pool.
2805	* @param pPage The page.
2806	* @param pShwPT The shadow page table (mapping of the page).
2807	* @param pGstPT The guest page table.
2808	*/
2809	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
2810	{
2811	for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pShwPT->a); i++)
2812	if (pShwPT->a[i].n.u1Present)
2813	{
2814	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
2815	i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
2816	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
2817	if (!--pPage->cPresent)
2818	break;
2819	}
2820	}
2821
2822
2823	/**
2824	* Clear references to guest physical memory in a PAE / 32-bit page table.
2825	*
2826	* @param pPool The pool.
2827	* @param pPage The page.
2828	* @param pShwPT The shadow page table (mapping of the page).
2829	* @param pGstPT The guest page table (just a half one).
2830	*/
2831	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
2832	{
2833	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++)
2834	if (pShwPT->a[i].n.u1Present)
2835	{
2836	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX32 hint=%RX32\n",
2837	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
2838	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
2839	}
2840	}
2841
2842
2843	/**
2844	* Clear references to guest physical memory in a PAE / PAE page table.
2845	*
2846	* @param pPool The pool.
2847	* @param pPage The page.
2848	* @param pShwPT The shadow page table (mapping of the page).
2849	* @param pGstPT The guest page table.
2850	*/
2851	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
2852	{
2853	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++)
2854	if (pShwPT->a[i].n.u1Present)
2855	{
2856	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
2857	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
2858	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
2859	}
2860	}
2861
2862
2863	/**
2864	* Clear references to guest physical memory in a 32-bit / 4MB page table.
2865	*
2866	* @param pPool The pool.
2867	* @param pPage The page.
2868	* @param pShwPT The shadow page table (mapping of the page).
2869	*/
2870	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
2871	{
2872	RTGCPHYS GCPhys = pPage->GCPhys;
2873	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
2874	if (pShwPT->a[i].n.u1Present)
2875	{
2876	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
2877	i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
2878	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
2879	}
2880	}
2881
2882
2883	/**
2884	* Clear references to guest physical memory in a PAE / 2/4MB page table.
2885	*
2886	* @param pPool The pool.
2887	* @param pPage The page.
2888	* @param pShwPT The shadow page table (mapping of the page).
2889	*/
2890	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
2891	{
2892	RTGCPHYS GCPhys = pPage->GCPhys;
2893	for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
2894	if (pShwPT->a[i].n.u1Present)
2895	{
2896	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX32 hint=%RX32\n",
2897	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
2898	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
2899	}
2900	}
2901	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2902
2903
2904	/**
2905	* Clear references to shadowed pages in a PAE page directory.
2906	*
2907	* @param pPool The pool.
2908	* @param pPage The page.
2909	* @param pShwPD The shadow page directory (mapping of the page).
2910	*/
2911	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
2912	{
2913	for (unsigned i = 0; i < ELEMENTS(pShwPD->a); i++)
2914	{
2915	if (pShwPD->a[i].n.u1Present)
2916	{
2917	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
2918	if (pSubPage)
2919	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
2920	else
2921	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
2922	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
2923	}
2924	}
2925	}
2926
2927
2928	/**
2929	* Clear references to shadowed pages in a 64-bit page directory pointer table.
2930	*
2931	* @param pPool The pool.
2932	* @param pPage The page.
2933	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
2934	*/
2935	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
2936	{
2937	for (unsigned i = 0; i < ELEMENTS(pShwPDPT->a); i++)
2938	{
2939	if (pShwPDPT->a[i].n.u1Present)
2940	{
2941	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
2942	if (pSubPage)
2943	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
2944	else
2945	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
2946	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
2947	}
2948	}
2949	}
2950
2951
2952	/**
2953	* Clears all references made by this page.
2954	*
2955	* This includes other shadow pages and GC physical addresses.
2956	*
2957	* @param pPool The pool.
2958	* @param pPage The page.
2959	*/
2960	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2961	{
2962	/*
2963	* Map the shadow page and take action according to the page kind.
2964	*/
2965	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
2966	switch (pPage->enmKind)
2967	{
2968	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2969	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2970	{
2971	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2972	void *pvGst;
2973	int rc = PGM_GCPHYS_2_PTR(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2974	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
2975	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2976	break;
2977	}
2978
2979	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2980	{
2981	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2982	void *pvGst;
2983	int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2984	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
2985	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2986	break;
2987	}
2988
2989	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2990	{
2991	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2992	void *pvGst;
2993	int rc = PGM_GCPHYS_2_PTR(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2994	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
2995	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2996	break;
2997	}
2998
2999	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
3000	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3001	{
3002	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3003	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
3004	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3005	break;
3006	}
3007
3008	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 4 MB page */
3009	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3010	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3011	{
3012	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3013	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
3014	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3015	break;
3016	}
3017
3018	#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3019	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3020	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3021	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3022	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3023	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3024	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3025	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3026	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3027	break;
3028	#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3029
3030	case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
3031	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3032	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
3033	break;
3034
3035	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3036	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
3037	break;
3038
3039	default:
3040	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
3041	}
3042
3043	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
3044	STAM_PROFILE_START(&pPool->StatZeroPage, z);
3045	ASMMemZeroPage(pvShw);
3046	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3047	pPage->fZeroed = true;
3048	}
3049	#endif /* PGMPOOL_WITH_USER_TRACKING */
3050
3051
3052	/**
3053	* Flushes all the special root pages as part of a pgmPoolFlushAllInt operation.
3054	*
3055	* @param pPool The pool.
3056	*/
3057	static void pgmPoolFlushAllSpecialRoots(PPGMPOOL pPool)
3058	{
3059	/*
3060	* These special pages are all mapped into the indexes 1..PGMPOOL_IDX_FIRST.
3061	*/
3062	Assert(NIL_PGMPOOL_IDX == 0);
3063	for (unsigned i = 1; i < PGMPOOL_IDX_FIRST; i++)
3064	{
3065	/*
3066	* Get the page address.
3067	*/
3068	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3069	union
3070	{
3071	uint64_t *pau64;
3072	uint32_t *pau32;
3073	} u;
3074	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
3075
3076	/*
3077	* Mark stuff not present.
3078	*/
3079	switch (pPage->enmKind)
3080	{
3081	case PGMPOOLKIND_ROOT_32BIT_PD:
3082	for (unsigned iPage = 0; iPage < X86_PG_ENTRIES; iPage++)
3083	if ((u.pau32[iPage] & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == X86_PDE_P)
3084	u.pau32[iPage] = 0;
3085	break;
3086
3087	case PGMPOOLKIND_ROOT_PAE_PD:
3088	for (unsigned iPage = 0; iPage < X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES; iPage++)
3089	if ((u.pau64[iPage] & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == X86_PDE_P)
3090	u.pau64[iPage] = 0;
3091	break;
3092
3093	case PGMPOOLKIND_ROOT_PML4:
3094	for (unsigned iPage = 0; iPage < X86_PG_PAE_ENTRIES; iPage++)
3095	if ((u.pau64[iPage] & (PGM_PLXFLAGS_PERMANENT \| X86_PML4E_P)) == X86_PML4E_P)
3096	u.pau64[iPage] = 0;
3097	break;
3098
3099	case PGMPOOLKIND_ROOT_PDPT:
3100	/* Not root of shadowed pages currently, ignore it. */
3101	break;
3102	}
3103	}
3104
3105	/*
3106	* Paranoia (to be removed), flag a global CR3 sync.
3107	*/
3108	VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
3109	}
3110
3111
3112	/**
3113	* Flushes the entire cache.
3114	*
3115	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3116	* and execute this CR3 flush.
3117	*
3118	* @param pPool The pool.
3119	*/
3120	static void pgmPoolFlushAllInt(PPGMPOOL pPool)
3121	{
3122	STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
3123	LogFlow(("pgmPoolFlushAllInt:\n"));
3124
3125	/*
3126	* If there are no pages in the pool, there is nothing to do.
3127	*/
3128	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
3129	{
3130	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3131	return;
3132	}
3133
3134	/*
3135	* Nuke the free list and reinsert all pages into it.
3136	*/
3137	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
3138	{
3139	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3140
3141	#ifdef IN_RING3
3142	Assert(pPage->Core.Key == MMPage2Phys(pPool->pVMHC, pPage->pvPageHC));
3143	#endif
3144	#ifdef PGMPOOL_WITH_MONITORING
3145	if (pPage->fMonitored)
3146	pgmPoolMonitorFlush(pPool, pPage);
3147	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3148	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3149	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3150	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3151	pPage->cModifications = 0;
3152	#endif
3153	pPage->GCPhys = NIL_RTGCPHYS;
3154	pPage->enmKind = PGMPOOLKIND_FREE;
3155	Assert(pPage->idx == i);
3156	pPage->iNext = i + 1;
3157	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
3158	pPage->fSeenNonGlobal = false;
3159	pPage->fMonitored= false;
3160	pPage->fCached = false;
3161	pPage->fReusedFlushPending = false;
3162	pPage->fCR3Mix = false;
3163	#ifdef PGMPOOL_WITH_USER_TRACKING
3164	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3165	#endif
3166	#ifdef PGMPOOL_WITH_CACHE
3167	pPage->iAgeNext = NIL_PGMPOOL_IDX;
3168	pPage->iAgePrev = NIL_PGMPOOL_IDX;
3169	#endif
3170	}
3171	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
3172	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
3173	pPool->cUsedPages = 0;
3174
3175	#ifdef PGMPOOL_WITH_USER_TRACKING
3176	/*
3177	* Zap and reinitialize the user records.
3178	*/
3179	pPool->cPresent = 0;
3180	pPool->iUserFreeHead = 0;
3181	PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
3182	const unsigned cMaxUsers = pPool->cMaxUsers;
3183	for (unsigned i = 0; i < cMaxUsers; i++)
3184	{
3185	paUsers[i].iNext = i + 1;
3186	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3187	paUsers[i].iUserTable = 0xfffe;
3188	}
3189	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
3190	#endif
3191
3192	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3193	/*
3194	* Clear all the GCPhys links and rebuild the phys ext free list.
3195	*/
3196	for (PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
3197	pRam;
3198	pRam = CTXALLSUFF(pRam->pNext))
3199	{
3200	unsigned iPage = pRam->cb >> PAGE_SHIFT;
3201	while (iPage-- > 0)
3202	pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
3203	}
3204
3205	pPool->iPhysExtFreeHead = 0;
3206	PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
3207	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
3208	for (unsigned i = 0; i < cMaxPhysExts; i++)
3209	{
3210	paPhysExts[i].iNext = i + 1;
3211	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
3212	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
3213	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
3214	}
3215	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3216	#endif
3217
3218	#ifdef PGMPOOL_WITH_MONITORING
3219	/*
3220	* Just zap the modified list.
3221	*/
3222	pPool->cModifiedPages = 0;
3223	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
3224	#endif
3225
3226	#ifdef PGMPOOL_WITH_CACHE
3227	/*
3228	* Clear the GCPhys hash and the age list.
3229	*/
3230	for (unsigned i = 0; i < ELEMENTS(pPool->aiHash); i++)
3231	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
3232	pPool->iAgeHead = NIL_PGMPOOL_IDX;
3233	pPool->iAgeTail = NIL_PGMPOOL_IDX;
3234	#endif
3235
3236	/*
3237	* Flush all the special root pages.
3238	* Reinsert active pages into the hash and ensure monitoring chains are correct.
3239	*/
3240	pgmPoolFlushAllSpecialRoots(pPool);
3241	for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
3242	{
3243	PPGMPOOLPAGE pPage = &pPool->aPages[i];
3244	pPage->iNext = NIL_PGMPOOL_IDX;
3245	#ifdef PGMPOOL_WITH_MONITORING
3246	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3247	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3248	pPage->cModifications = 0;
3249	/* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
3250	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3251	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3252	if (pPage->fMonitored)
3253	{
3254	PVM pVM = pPool->CTXSUFF(pVM);
3255	int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
3256	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
3257	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
3258	pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pPage),
3259	pPool->pszAccessHandler);
3260	AssertFatalRCSuccess(rc);
3261	# ifdef PGMPOOL_WITH_CACHE
3262	pgmPoolHashInsert(pPool, pPage);
3263	# endif
3264	}
3265	#endif
3266	#ifdef PGMPOOL_WITH_USER_TRACKING
3267	Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
3268	#endif
3269	#ifdef PGMPOOL_WITH_CACHE
3270	Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
3271	Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
3272	#endif
3273	}
3274
3275	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3276	}
3277
3278
3279	/**
3280	* Flushes a pool page.
3281	*
3282	* This moves the page to the free list after removing all user references to it.
3283	* In GC this will cause a CR3 reload if the page is traced back to an active root page.
3284	*
3285	* @returns VBox status code.
3286	* @retval VINF_SUCCESS on success.
3287	* @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
3288	* @param pPool The pool.
3289	* @param HCPhys The HC physical address of the shadow page.
3290	*/
3291	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3292	{
3293	int rc = VINF_SUCCESS;
3294	STAM_PROFILE_START(&pPool->StatFlushPage, f);
3295	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%VHp, .idx=%d, .enmKind=%d, .GCPhys=%VGp}\n",
3296	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
3297
3298	/*
3299	* Quietly reject any attempts at flushing any of the special root pages.
3300	*/
3301	if (pPage->idx < PGMPOOL_IDX_FIRST)
3302	{
3303	Log(("pgmPoolFlushPage: specaial root page, rejected. enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
3304	return VINF_SUCCESS;
3305	}
3306
3307	/*
3308	* Mark the page as being in need of a ASMMemZeroPage().
3309	*/
3310	pPage->fZeroed = false;
3311
3312	#ifdef PGMPOOL_WITH_USER_TRACKING
3313	/*
3314	* Clear the page.
3315	*/
3316	pgmPoolTrackClearPageUsers(pPool, pPage);
3317	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
3318	pgmPoolTrackDeref(pPool, pPage);
3319	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
3320	#endif
3321
3322	#ifdef PGMPOOL_WITH_CACHE
3323	/*
3324	* Flush it from the cache.
3325	*/
3326	pgmPoolCacheFlushPage(pPool, pPage);
3327	#endif /* PGMPOOL_WITH_CACHE */
3328
3329	#ifdef PGMPOOL_WITH_MONITORING
3330	/*
3331	* Deregistering the monitoring.
3332	*/
3333	if (pPage->fMonitored)
3334	rc = pgmPoolMonitorFlush(pPool, pPage);
3335	#endif
3336
3337	/*
3338	* Free the page.
3339	*/
3340	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
3341	pPage->iNext = pPool->iFreeHead;
3342	pPool->iFreeHead = pPage->idx;
3343	pPage->enmKind = PGMPOOLKIND_FREE;
3344	pPage->GCPhys = NIL_RTGCPHYS;
3345	pPage->fReusedFlushPending = false;
3346
3347	pPool->cUsedPages--;
3348	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
3349	return rc;
3350	}
3351
3352
3353	/**
3354	* Frees a usage of a pool page.
3355	*
3356	* The caller is responsible to updating the user table so that it no longer
3357	* references the shadow page.
3358	*
3359	* @param pPool The pool.
3360	* @param HCPhys The HC physical address of the shadow page.
3361	* @param iUser The shadow page pool index of the user table.
3362	* @param iUserTable The index into the user table (shadowed).
3363	*/
3364	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
3365	{
3366	STAM_PROFILE_START(&pPool->StatFree, a);
3367	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%VHp, .idx=%d, enmKind=%d} iUser=%#x iUserTable=%#x\n",
3368	pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, iUser, iUserTable));
3369	Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
3370	#ifdef PGMPOOL_WITH_USER_TRACKING
3371	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
3372	#endif
3373	#ifdef PGMPOOL_WITH_CACHE
3374	if (!pPage->fCached)
3375	#endif
3376	pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED can be ignored here. */
3377	STAM_PROFILE_STOP(&pPool->StatFree, a);
3378	}
3379
3380
3381	/**
3382	* Makes one or more free page free.
3383	*
3384	* @returns VBox status code.
3385	* @retval VINF_SUCCESS on success.
3386	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3387	*
3388	* @param pPool The pool.
3389	* @param iUser The user of the page.
3390	*/
3391	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, uint16_t iUser)
3392	{
3393	LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
3394
3395	/*
3396	* If the pool isn't full grown yet, expand it.
3397	*/
3398	if (pPool->cCurPages < pPool->cMaxPages)
3399	{
3400	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
3401	#ifdef IN_RING3
3402	int rc = PGMR3PoolGrow(pPool->pVMHC);
3403	#else
3404	int rc = CTXALLMID(VMM, CallHost)(pPool->CTXSUFF(pVM), VMMCALLHOST_PGM_POOL_GROW, 0);
3405	#endif
3406	if (VBOX_FAILURE(rc))
3407	return rc;
3408	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
3409	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
3410	return VINF_SUCCESS;
3411	}
3412
3413	#ifdef PGMPOOL_WITH_CACHE
3414	/*
3415	* Free one cached page.
3416	*/
3417	return pgmPoolCacheFreeOne(pPool, iUser);
3418	#else
3419	/*
3420	* Flush the pool.
3421	* If we have tracking enabled, it should be possible to come up with
3422	* a cheap replacement strategy...
3423	*/
3424	pgmPoolFlushAllInt(pPool);
3425	return VERR_PGM_POOL_FLUSHED;
3426	#endif
3427	}
3428
3429
3430	/**
3431	* Allocates a page from the pool.
3432	*
3433	* This page may actually be a cached page and not in need of any processing
3434	* on the callers part.
3435	*
3436	* @returns VBox status code.
3437	* @retval VINF_SUCCESS if a NEW page was allocated.
3438	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
3439	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3440	* @param pVM The VM handle.
3441	* @param GCPhys The GC physical address of the page we're gonna shadow.
3442	* For 4MB and 2MB PD entries, it's the first address the
3443	* shadow PT is covering.
3444	* @param enmKind The kind of mapping.
3445	* @param iUser The shadow page pool index of the user table.
3446	* @param iUserTable The index into the user table (shadowed).
3447	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
3448	*/
3449	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint16_t iUserTable, PPPGMPOOLPAGE ppPage)
3450	{
3451	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3452	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
3453	LogFlow(("pgmPoolAlloc: GCPhys=%VGp enmKind=%d iUser=%#x iUserTable=%#x\n", GCPhys, enmKind, iUser, iUserTable));
3454	*ppPage = NULL;
3455
3456	#ifdef PGMPOOL_WITH_CACHE
3457	if (pPool->fCacheEnabled)
3458	{
3459	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, iUser, iUserTable, ppPage);
3460	if (VBOX_SUCCESS(rc2))
3461	{
3462	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3463	LogFlow(("pgmPoolAlloc: returns %Vrc ppPage=%p:{.Key=%VHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
3464	return rc2;
3465	}
3466	}
3467	#endif
3468
3469	/*
3470	* Allocate a new one.
3471	*/
3472	int rc = VINF_SUCCESS;
3473	uint16_t iNew = pPool->iFreeHead;
3474	if (iNew == NIL_PGMPOOL_IDX)
3475	{
3476	rc = pgmPoolMakeMoreFreePages(pPool, iUser);
3477	if (VBOX_FAILURE(rc))
3478	{
3479	if (rc != VERR_PGM_POOL_CLEARED)
3480	{
3481	Log(("pgmPoolAlloc: returns %Vrc (Free)\n", rc));
3482	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3483	return rc;
3484	}
3485	rc = VERR_PGM_POOL_FLUSHED;
3486	}
3487	iNew = pPool->iFreeHead;
3488	AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
3489	}
3490
3491	/* unlink the free head */
3492	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
3493	pPool->iFreeHead = pPage->iNext;
3494	pPage->iNext = NIL_PGMPOOL_IDX;
3495
3496	/*
3497	* Initialize it.
3498	*/
3499	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
3500	pPage->enmKind = enmKind;
3501	pPage->GCPhys = GCPhys;
3502	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
3503	pPage->fMonitored = false;
3504	pPage->fCached = false;
3505	pPage->fReusedFlushPending = false;
3506	pPage->fCR3Mix = false;
3507	#ifdef PGMPOOL_WITH_MONITORING
3508	pPage->cModifications = 0;
3509	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3510	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3511	#endif
3512	#ifdef PGMPOOL_WITH_USER_TRACKING
3513	pPage->cPresent = 0;
3514	pPage->iFirstPresent = ~0;
3515
3516	/*
3517	* Insert into the tracking and cache. If this fails, free the page.
3518	*/
3519	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
3520	if (VBOX_FAILURE(rc3))
3521	{
3522	if (rc3 != VERR_PGM_POOL_CLEARED)
3523	{
3524	pPool->cUsedPages--;
3525	pPage->enmKind = PGMPOOLKIND_FREE;
3526	pPage->GCPhys = NIL_RTGCPHYS;
3527	pPage->iNext = pPool->iFreeHead;
3528	pPool->iFreeHead = pPage->idx;
3529	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3530	Log(("pgmPoolAlloc: returns %Vrc (Insert)\n", rc3));
3531	return rc3;
3532	}
3533	rc = VERR_PGM_POOL_FLUSHED;
3534	}
3535	#endif /* PGMPOOL_WITH_USER_TRACKING */
3536
3537	/*
3538	* Commit the allocation, clear the page and return.
3539	*/
3540	#ifdef VBOX_WITH_STATISTICS
3541	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
3542	pPool->cUsedPagesHigh = pPool->cUsedPages;
3543	#endif
3544
3545	if (!pPage->fZeroed)
3546	{
3547	STAM_PROFILE_START(&pPool->StatZeroPage, z);
3548	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
3549	ASMMemZeroPage(pv);
3550	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3551	}
3552
3553	*ppPage = pPage;
3554	LogFlow(("pgmPoolAlloc: returns %Vrc *ppPage=%p:{.Key=%VHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
3555	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
3556	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3557	return rc;
3558	}
3559
3560
3561	/**
3562	* Frees a usage of a pool page.
3563	*
3564	* @param pVM The VM handle.
3565	* @param HCPhys The HC physical address of the shadow page.
3566	* @param iUser The shadow page pool index of the user table.
3567	* @param iUserTable The index into the user table (shadowed).
3568	*/
3569	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint16_t iUserTable)
3570	{
3571	LogFlow(("pgmPoolFree: HCPhys=%VHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
3572	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3573	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
3574	}
3575
3576
3577	/**
3578	* Gets a in-use page in the pool by it's physical address.
3579	*
3580	* @returns Pointer to the page.
3581	* @param pVM The VM handle.
3582	* @param HCPhys The HC physical address of the shadow page.
3583	* @remark This function will NEVER return NULL. It will assert if HCPhys is invalid.
3584	*/
3585	PPGMPOOLPAGE pgmPoolGetPageByHCPhys(PVM pVM, RTHCPHYS HCPhys)
3586	{
3587	/** @todo profile this! */
3588	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3589	PPGMPOOLPAGE pPage = pgmPoolGetPage(pPool, HCPhys);
3590	Log3(("pgmPoolGetPageByHCPhys: HCPhys=%VHp -> %p:{.idx=%d .GCPhys=%VGp .enmKind=%d}\n",
3591	HCPhys, pPage, pPage->idx, pPage->GCPhys, pPage->enmKind));
3592	return pPage;
3593	}
3594
3595
3596	/**
3597	* Flushes the entire cache.
3598	*
3599	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3600	* and execute this CR3 flush.
3601	*
3602	* @param pPool The pool.
3603	*/
3604	void pgmPoolFlushAll(PVM pVM)
3605	{
3606	LogFlow(("pgmPoolFlushAll:\n"));
3607	pgmPoolFlushAllInt(pVM->pgm.s.CTXSUFF(pPool));
3608	}
3609

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 8090

Download in other formats: