PGMAllPool.cpp@ 87626

Last change on this file since 87626 was 87141, checked in by vboxsync, 4 years ago
VMM: Remove VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 and the code it encloses as it is unused since the removal of x86 darwin support
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 208.7 KB

Line
1	/* $Id: PGMAllPool.cpp 87141 2020-12-29 19:12:45Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_POOL
23	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
24	#include <VBox/vmm/pgm.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/em.h>
27	#include <VBox/vmm/cpum.h>
28	#include "PGMInternal.h"
29	#include <VBox/vmm/vmcc.h>
30	#include "PGMInline.h"
31	#include <VBox/disopcode.h>
32	#include <VBox/vmm/hm_vmx.h>
33
34	#include <VBox/log.h>
35	#include <VBox/err.h>
36	#include <iprt/asm.h>
37	#include <iprt/asm-amd64-x86.h>
38	#include <iprt/string.h>
39
40
41	/*********************************************************************************************************************************
42	* Internal Functions *
43	*********************************************************************************************************************************/
44	RT_C_DECLS_BEGIN
45	#if 0 /* unused */
46	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
47	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
48	#endif /* unused */
49	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
50	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
51	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
52	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
53	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
54	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
55	#endif
56	#if 0 /defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)/
57	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
58	#endif
59
60	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage);
61	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
62	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
63	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
64
65	RT_C_DECLS_END
66
67
68	#if 0 /* unused */
69	/**
70	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
71	*
72	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
73	* @param enmKind The page kind.
74	*/
75	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
76	{
77	switch (enmKind)
78	{
79	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
80	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
81	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
82	return true;
83	default:
84	return false;
85	}
86	}
87	#endif /* unused */
88
89
90	/**
91	* Flushes a chain of pages sharing the same access monitor.
92	*
93	* @param pPool The pool.
94	* @param pPage A page in the chain.
95	*/
96	void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
97	{
98	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
99
100	/*
101	* Find the list head.
102	*/
103	uint16_t idx = pPage->idx;
104	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
105	{
106	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
107	{
108	idx = pPage->iMonitoredPrev;
109	Assert(idx != pPage->idx);
110	pPage = &pPool->aPages[idx];
111	}
112	}
113
114	/*
115	* Iterate the list flushing each shadow page.
116	*/
117	for (;;)
118	{
119	idx = pPage->iMonitoredNext;
120	Assert(idx != pPage->idx);
121	if (pPage->idx >= PGMPOOL_IDX_FIRST)
122	{
123	int rc2 = pgmPoolFlushPage(pPool, pPage);
124	AssertRC(rc2);
125	}
126	/* next */
127	if (idx == NIL_PGMPOOL_IDX)
128	break;
129	pPage = &pPool->aPages[idx];
130	}
131	}
132
133
134	/**
135	* Wrapper for getting the current context pointer to the entry being modified.
136	*
137	* @returns VBox status code suitable for scheduling.
138	* @param pVM The cross context VM structure.
139	* @param pvDst Destination address
140	* @param pvSrc Pointer to the mapping of @a GCPhysSrc or NULL depending
141	* on the context (e.g. \#PF in R0 & RC).
142	* @param GCPhysSrc The source guest physical address.
143	* @param cb Size of data to read
144	*/
145	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVMCC pVM, void pvDst, void const pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
146	{
147	#if defined(IN_RING3)
148	NOREF(pVM); NOREF(GCPhysSrc);
149	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
150	return VINF_SUCCESS;
151	#else
152	/** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
153	NOREF(pvSrc);
154	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
155	#endif
156	}
157
158
159	/**
160	* Process shadow entries before they are changed by the guest.
161	*
162	* For PT entries we will clear them. For PD entries, we'll simply check
163	* for mapping conflicts and set the SyncCR3 FF if found.
164	*
165	* @param pVCpu The cross context virtual CPU structure.
166	* @param pPool The pool.
167	* @param pPage The head page.
168	* @param GCPhysFault The guest physical fault address.
169	* @param pvAddress Pointer to the mapping of @a GCPhysFault or NULL
170	* depending on the context (e.g. \#PF in R0 & RC).
171	* @param cbWrite Write size; might be zero if the caller knows we're not crossing entry boundaries
172	*/
173	static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
174	void const *pvAddress, unsigned cbWrite)
175	{
176	AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
177	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
178	PVMCC pVM = pPool->CTX_SUFF(pVM);
179	NOREF(pVCpu);
180
181	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
182	(RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
183
184	for (;;)
185	{
186	union
187	{
188	void *pv;
189	PX86PT pPT;
190	PPGMSHWPTPAE pPTPae;
191	PX86PD pPD;
192	PX86PDPAE pPDPae;
193	PX86PDPT pPDPT;
194	PX86PML4 pPML4;
195	} uShw;
196
197	LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
198	pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
199
200	uShw.pv = NULL;
201	switch (pPage->enmKind)
202	{
203	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
204	{
205	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
206	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
207	const unsigned iShw = off / sizeof(X86PTE);
208	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
209	X86PGUINT const uPde = uShw.pPT->a[iShw].u;
210	if (uPde & X86_PTE_P)
211	{
212	X86PTE GstPte;
213	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
214	AssertRC(rc);
215	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
216	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK, iShw);
217	ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
218	}
219	break;
220	}
221
222	/* page/2 sized */
223	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
224	{
225	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
226	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
227	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
228	{
229	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
230	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
231	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
232	{
233	X86PTE GstPte;
234	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
235	AssertRC(rc);
236
237	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
238	pgmPoolTracDerefGCPhysHint(pPool, pPage,
239	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
240	GstPte.u & X86_PTE_PG_MASK,
241	iShw);
242	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
243	}
244	}
245	break;
246	}
247
248	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
249	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
250	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
251	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
252	{
253	unsigned iGst = off / sizeof(X86PDE);
254	unsigned iShwPdpt = iGst / 256;
255	unsigned iShw = (iGst % 256) * 2;
256	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
257
258	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
259	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
260	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
261	{
262	for (unsigned i = 0; i < 2; i++)
263	{
264	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw + i].u;
265	if (uPde & X86_PDE_P)
266	{
267	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw + i, uPde));
268	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw + i);
269	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw + i].u, 0);
270	}
271
272	/* paranoia / a bit assumptive. */
273	if ( (off & 3)
274	&& (off & 3) + cbWrite > 4)
275	{
276	const unsigned iShw2 = iShw + 2 + i;
277	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
278	{
279	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
280	if (uPde2 & X86_PDE_P)
281	{
282	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
283	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
284	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
285	}
286	}
287	}
288	}
289	}
290	break;
291	}
292
293	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
294	{
295	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
296	const unsigned iShw = off / sizeof(X86PTEPAE);
297	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
298	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
299	{
300	X86PTEPAE GstPte;
301	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
302	AssertRC(rc);
303
304	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
305	pgmPoolTracDerefGCPhysHint(pPool, pPage,
306	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
307	GstPte.u & X86_PTE_PAE_PG_MASK,
308	iShw);
309	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
310	}
311
312	/* paranoia / a bit assumptive. */
313	if ( (off & 7)
314	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
315	{
316	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
317	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
318
319	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
320	{
321	X86PTEPAE GstPte;
322	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
323	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
324	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
325	AssertRC(rc);
326	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
327	pgmPoolTracDerefGCPhysHint(pPool, pPage,
328	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
329	GstPte.u & X86_PTE_PAE_PG_MASK,
330	iShw2);
331	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
332	}
333	}
334	break;
335	}
336
337	case PGMPOOLKIND_32BIT_PD:
338	{
339	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
340	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
341
342	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
343	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
344	X86PGUINT const uPde = uShw.pPD->a[iShw].u;
345	if (uPde & X86_PDE_P)
346	{
347	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
348	pgmPoolFree(pVM, uPde & X86_PDE_PG_MASK, pPage->idx, iShw);
349	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
350	}
351
352	/* paranoia / a bit assumptive. */
353	if ( (off & 3)
354	&& (off & 3) + cbWrite > sizeof(X86PTE))
355	{
356	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
357	if ( iShw2 != iShw
358	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
359	{
360	X86PGUINT const uPde2 = uShw.pPD->a[iShw2].u;
361	if (uPde2 & X86_PDE_P)
362	{
363	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
364	pgmPoolFree(pVM, uPde2 & X86_PDE_PG_MASK, pPage->idx, iShw2);
365	ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
366	}
367	}
368	}
369	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
370	if ( uShw.pPD->a[iShw].n.u1Present
371	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
372	{
373	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
374	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
375	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
376	}
377	#endif
378	break;
379	}
380
381	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
382	{
383	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
384	const unsigned iShw = off / sizeof(X86PDEPAE);
385	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
386
387	/*
388	* Causes trouble when the guest uses a PDE to refer to the whole page table level
389	* structure. (Invalidate here; faults later on when it tries to change the page
390	* table entries -> recheck; probably only applies to the RC case.)
391	*/
392	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
393	if (uPde & X86_PDE_P)
394	{
395	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
396	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
397	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
398	}
399
400	/* paranoia / a bit assumptive. */
401	if ( (off & 7)
402	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
403	{
404	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
405	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
406
407	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
408	if (uPde2 & X86_PDE_P)
409	{
410	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
411	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
412	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
413	}
414	}
415	break;
416	}
417
418	case PGMPOOLKIND_PAE_PDPT:
419	{
420	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
421	/*
422	* Hopefully this doesn't happen very often:
423	* - touching unused parts of the page
424	* - messing with the bits of pd pointers without changing the physical address
425	*/
426	/* PDPT roots are not page aligned; 32 byte only! */
427	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
428
429	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
430	const unsigned iShw = offPdpt / sizeof(X86PDPE);
431	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
432	{
433	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
434	if (uPdpe & X86_PDPE_P)
435	{
436	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
437	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
438	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
439	}
440
441	/* paranoia / a bit assumptive. */
442	if ( (offPdpt & 7)
443	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
444	{
445	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
446	if ( iShw2 != iShw
447	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
448	{
449	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
450	if (uPdpe2 & X86_PDPE_P)
451	{
452	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
453	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
454	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
455	}
456	}
457	}
458	}
459	break;
460	}
461
462	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
463	{
464	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
465	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
466	const unsigned iShw = off / sizeof(X86PDEPAE);
467	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
468	#ifndef PGM_WITHOUT_MAPPINGS
469	Assert(!(uPde & PGM_PDFLAGS_MAPPING));
470	#endif
471	if (uPde & X86_PDE_P)
472	{
473	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
474	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
475	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
476	}
477
478	/* paranoia / a bit assumptive. */
479	if ( (off & 7)
480	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
481	{
482	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
483	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
484	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
485	#ifndef PGM_WITHOUT_MAPPINGS
486	Assert(!(uPde2 & PGM_PDFLAGS_MAPPING));
487	#endif
488	if (uPde2 & X86_PDE_P)
489	{
490	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
491	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
492	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
493	}
494	}
495	break;
496	}
497
498	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
499	{
500	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
501	/*
502	* Hopefully this doesn't happen very often:
503	* - messing with the bits of pd pointers without changing the physical address
504	*/
505	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
506	const unsigned iShw = off / sizeof(X86PDPE);
507	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
508	if (uPdpe & X86_PDPE_P)
509	{
510	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uPdpe));
511	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
512	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
513	}
514	/* paranoia / a bit assumptive. */
515	if ( (off & 7)
516	&& (off & 7) + cbWrite > sizeof(X86PDPE))
517	{
518	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
519	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
520	if (uPdpe2 & X86_PDPE_P)
521	{
522	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpe2));
523	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
524	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
525	}
526	}
527	break;
528	}
529
530	case PGMPOOLKIND_64BIT_PML4:
531	{
532	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
533	/*
534	* Hopefully this doesn't happen very often:
535	* - messing with the bits of pd pointers without changing the physical address
536	*/
537	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
538	const unsigned iShw = off / sizeof(X86PDPE);
539	X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
540	if (uPml4e & X86_PML4E_P)
541	{
542	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uPml4e));
543	pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
544	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
545	}
546	/* paranoia / a bit assumptive. */
547	if ( (off & 7)
548	&& (off & 7) + cbWrite > sizeof(X86PDPE))
549	{
550	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
551	X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
552	if (uPml4e2 & X86_PML4E_P)
553	{
554	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
555	pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
556	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
557	}
558	}
559	break;
560	}
561
562	default:
563	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
564	}
565	PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
566
567	/* next */
568	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
569	return;
570	pPage = &pPool->aPages[pPage->iMonitoredNext];
571	}
572	}
573
574	#ifndef IN_RING3
575
576	/**
577	* Checks if a access could be a fork operation in progress.
578	*
579	* Meaning, that the guest is setting up the parent process for Copy-On-Write.
580	*
581	* @returns true if it's likely that we're forking, otherwise false.
582	* @param pPool The pool.
583	* @param pDis The disassembled instruction.
584	* @param offFault The access offset.
585	*/
586	DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
587	{
588	/*
589	* i386 linux is using btr to clear X86_PTE_RW.
590	* The functions involved are (2.6.16 source inspection):
591	* clear_bit
592	* ptep_set_wrprotect
593	* copy_one_pte
594	* copy_pte_range
595	* copy_pmd_range
596	* copy_pud_range
597	* copy_page_range
598	* dup_mmap
599	* dup_mm
600	* copy_mm
601	* copy_process
602	* do_fork
603	*/
604	if ( pDis->pCurInstr->uOpcode == OP_BTR
605	&& !(offFault & 4)
606	/** @todo Validate that the bit index is X86_PTE_RW. */
607	)
608	{
609	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
610	return true;
611	}
612	return false;
613	}
614
615
616	/**
617	* Determine whether the page is likely to have been reused.
618	*
619	* @returns true if we consider the page as being reused for a different purpose.
620	* @returns false if we consider it to still be a paging page.
621	* @param pVM The cross context VM structure.
622	* @param pVCpu The cross context virtual CPU structure.
623	* @param pRegFrame Trap register frame.
624	* @param pDis The disassembly info for the faulting instruction.
625	* @param pvFault The fault address.
626	* @param pPage The pool page being accessed.
627	*
628	* @remark The REP prefix check is left to the caller because of STOSD/W.
629	*/
630	DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault,
631	PPGMPOOLPAGE pPage)
632	{
633	/* Locked (CR3, PDPTR*4) should not be reusable. Considering them as
634	such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
635	if (pPage->cLocked)
636	{
637	Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
638	return false;
639	}
640
641	/** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
642	if ( HMHasPendingIrq(pVM)
643	&& pRegFrame->rsp - pvFault < 32)
644	{
645	/* Fault caused by stack writes while trying to inject an interrupt event. */
646	Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
647	return true;
648	}
649
650	LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse, pDis->Param1.Base.idxGenReg));
651
652	/* Non-supervisor mode write means it's used for something else. */
653	if (CPUMGetGuestCPL(pVCpu) == 3)
654	return true;
655
656	switch (pDis->pCurInstr->uOpcode)
657	{
658	/* call implies the actual push of the return address faulted */
659	case OP_CALL:
660	Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
661	return true;
662	case OP_PUSH:
663	Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
664	return true;
665	case OP_PUSHF:
666	Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
667	return true;
668	case OP_PUSHA:
669	Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
670	return true;
671	case OP_FXSAVE:
672	Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
673	return true;
674	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
675	Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
676	return true;
677	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
678	Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
679	return true;
680	case OP_MOVSWD:
681	case OP_STOSWD:
682	if ( pDis->fPrefix == (DISPREFIX_REP\|DISPREFIX_REX)
683	&& pRegFrame->rcx >= 0x40
684	)
685	{
686	Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
687
688	Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
689	return true;
690	}
691	break;
692
693	default:
694	/*
695	* Anything having ESP on the left side means stack writes.
696	*/
697	if ( ( (pDis->Param1.fUse & DISUSE_REG_GEN32)
698	\|\| (pDis->Param1.fUse & DISUSE_REG_GEN64))
699	&& (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
700	{
701	Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
702	return true;
703	}
704	break;
705	}
706
707	/*
708	* Page table updates are very very unlikely to be crossing page boundraries,
709	* and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
710	*/
711	uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
712	if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
713	{
714	Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
715	return true;
716	}
717
718	/*
719	* Nobody does an unaligned 8 byte write to a page table, right.
720	*/
721	if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
722	{
723	Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
724	return true;
725	}
726
727	return false;
728	}
729
730
731	/**
732	* Flushes the page being accessed.
733	*
734	* @returns VBox status code suitable for scheduling.
735	* @param pVM The cross context VM structure.
736	* @param pVCpu The cross context virtual CPU structure.
737	* @param pPool The pool.
738	* @param pPage The pool page (head).
739	* @param pDis The disassembly of the write instruction.
740	* @param pRegFrame The trap register frame.
741	* @param GCPhysFault The fault address as guest physical address.
742	* @param pvFault The fault address.
743	* @todo VBOXSTRICTRC
744	*/
745	static int pgmRZPoolAccessPfHandlerFlush(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
746	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
747	{
748	NOREF(pVM); NOREF(GCPhysFault);
749
750	/*
751	* First, do the flushing.
752	*/
753	pgmPoolMonitorChainFlush(pPool, pPage);
754
755	/*
756	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
757	* Must do this in raw mode (!); XP boot will fail otherwise.
758	*/
759	int rc = VINF_SUCCESS;
760	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
761	if (rc2 == VINF_SUCCESS)
762	{ /* do nothing */ }
763	else if (rc2 == VINF_EM_RESCHEDULE)
764	{
765	rc = VBOXSTRICTRC_VAL(rc2);
766	# ifndef IN_RING3
767	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
768	# endif
769	}
770	else if (rc2 == VERR_EM_INTERPRETER)
771	{
772	rc = VINF_EM_RAW_EMULATE_INSTR;
773	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
774	}
775	else if (RT_FAILURE_NP(rc2))
776	rc = VBOXSTRICTRC_VAL(rc2);
777	else
778	AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
779
780	LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
781	return rc;
782	}
783
784
785	/**
786	* Handles the STOSD write accesses.
787	*
788	* @returns VBox status code suitable for scheduling.
789	* @param pVM The cross context VM structure.
790	* @param pPool The pool.
791	* @param pPage The pool page (head).
792	* @param pDis The disassembly of the write instruction.
793	* @param pRegFrame The trap register frame.
794	* @param GCPhysFault The fault address as guest physical address.
795	* @param pvFault The fault address.
796	*/
797	DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
798	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
799	{
800	unsigned uIncrement = pDis->Param1.cb;
801	NOREF(pVM);
802
803	Assert(pDis->uCpuMode == DISCPUMODE_32BIT \|\| pDis->uCpuMode == DISCPUMODE_64BIT);
804	Assert(pRegFrame->rcx <= 0x20);
805
806	# ifdef VBOX_STRICT
807	if (pDis->uOpMode == DISCPUMODE_32BIT)
808	Assert(uIncrement == 4);
809	else
810	Assert(uIncrement == 8);
811	# endif
812
813	Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
814
815	/*
816	* Increment the modification counter and insert it into the list
817	* of modified pages the first time.
818	*/
819	if (!pPage->cModifications++)
820	pgmPoolMonitorModifiedInsert(pPool, pPage);
821
822	/*
823	* Execute REP STOSD.
824	*
825	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
826	* write situation, meaning that it's safe to write here.
827	*/
828	PVMCPUCC pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
829	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
830	while (pRegFrame->rcx)
831	{
832	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
833	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
834	pu32 += uIncrement;
835	GCPhysFault += uIncrement;
836	pRegFrame->rdi += uIncrement;
837	pRegFrame->rcx--;
838	}
839	pRegFrame->rip += pDis->cbInstr;
840
841	LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
842	return VINF_SUCCESS;
843	}
844
845
846	/**
847	* Handles the simple write accesses.
848	*
849	* @returns VBox status code suitable for scheduling.
850	* @param pVM The cross context VM structure.
851	* @param pVCpu The cross context virtual CPU structure.
852	* @param pPool The pool.
853	* @param pPage The pool page (head).
854	* @param pDis The disassembly of the write instruction.
855	* @param pRegFrame The trap register frame.
856	* @param GCPhysFault The fault address as guest physical address.
857	* @param pvFault The fault address.
858	* @param pfReused Reused state (in/out)
859	*/
860	DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
861	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
862	{
863	Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
864	NOREF(pVM);
865	NOREF(pfReused); /* initialized by caller */
866
867	/*
868	* Increment the modification counter and insert it into the list
869	* of modified pages the first time.
870	*/
871	if (!pPage->cModifications++)
872	pgmPoolMonitorModifiedInsert(pPool, pPage);
873
874	/*
875	* Clear all the pages. ASSUMES that pvFault is readable.
876	*/
877	uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
878	if (cbWrite <= 8)
879	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
880	else if (cbWrite <= 16)
881	{
882	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
883	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
884	}
885	else
886	{
887	Assert(cbWrite <= 32);
888	for (uint32_t off = 0; off < cbWrite; off += 8)
889	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
890	}
891
892	/*
893	* Interpret the instruction.
894	*/
895	VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
896	if (RT_SUCCESS(rc))
897	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
898	else if (rc == VERR_EM_INTERPRETER)
899	{
900	LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
901	pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode));
902	rc = VINF_EM_RAW_EMULATE_INSTR;
903	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
904	}
905
906	# if 0 /* experimental code */
907	if (rc == VINF_SUCCESS)
908	{
909	switch (pPage->enmKind)
910	{
911	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
912	{
913	X86PTEPAE GstPte;
914	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
915	AssertRC(rc);
916
917	/* Check the new value written by the guest. If present and with a bogus physical address, then
918	* it's fairly safe to assume the guest is reusing the PT.
919	*/
920	if (GstPte.n.u1Present)
921	{
922	RTHCPHYS HCPhys = -1;
923	int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
924	if (rc != VINF_SUCCESS)
925	{
926	*pfReused = true;
927	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
928	}
929	}
930	break;
931	}
932	}
933	}
934	# endif
935
936	LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
937	return VBOXSTRICTRC_VAL(rc);
938	}
939
940
941	/**
942	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
943	* \#PF access handler callback for page table pages.}
944	*
945	* @remarks The @a pvUser argument points to the PGMPOOLPAGE.
946	*/
947	DECLEXPORT(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
948	RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
949	{
950	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
951	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
952	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
953	unsigned cMaxModifications;
954	bool fForcedFlush = false;
955	NOREF(uErrorCode);
956
957	LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
958
959	pgmLock(pVM);
960	if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
961	{
962	/* Pool page changed while we were waiting for the lock; ignore. */
963	Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
964	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
965	pgmUnlock(pVM);
966	return VINF_SUCCESS;
967	}
968	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
969	if (pPage->fDirty)
970	{
971	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
972	pgmUnlock(pVM);
973	return VINF_SUCCESS; /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
974	}
975	# endif
976
977	# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
978	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
979	{
980	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
981	void *pvGst;
982	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
983	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
984	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
985	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
986	}
987	# endif
988
989	/*
990	* Disassemble the faulting instruction.
991	*/
992	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
993	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
994	if (RT_UNLIKELY(rc != VINF_SUCCESS))
995	{
996	AssertMsg(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
997	pgmUnlock(pVM);
998	return rc;
999	}
1000
1001	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1002
1003	/*
1004	* We should ALWAYS have the list head as user parameter. This
1005	* is because we use that page to record the changes.
1006	*/
1007	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1008
1009	# ifdef IN_RING0
1010	/* Maximum nr of modifications depends on the page type. */
1011	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1012	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1013	cMaxModifications = 4;
1014	else
1015	cMaxModifications = 24;
1016	# else
1017	cMaxModifications = 48;
1018	# endif
1019
1020	/*
1021	* Incremental page table updates should weigh more than random ones.
1022	* (Only applies when started from offset 0)
1023	*/
1024	pVCpu->pgm.s.cPoolAccessHandler++;
1025	if ( pPage->GCPtrLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1026	&& pPage->GCPtrLastAccessHandlerRip < pRegFrame->rip + 0x40
1027	&& pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb)
1028	&& pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
1029	{
1030	Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1031	Assert(pPage->cModifications < 32000);
1032	pPage->cModifications = pPage->cModifications * 2;
1033	pPage->GCPtrLastAccessHandlerFault = pvFault;
1034	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1035	if (pPage->cModifications >= cMaxModifications)
1036	{
1037	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
1038	fForcedFlush = true;
1039	}
1040	}
1041
1042	if (pPage->cModifications >= cMaxModifications)
1043	Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1044
1045	/*
1046	* Check if it's worth dealing with.
1047	*/
1048	bool fReused = false;
1049	bool fNotReusedNotForking = false;
1050	if ( ( pPage->cModifications < cMaxModifications /** @todo \#define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1051	\|\| pgmPoolIsPageLocked(pPage)
1052	)
1053	&& !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage))
1054	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1055	{
1056	/*
1057	* Simple instructions, no REP prefix.
1058	*/
1059	if (!(pDis->fPrefix & (DISPREFIX_REP \| DISPREFIX_REPNE)))
1060	{
1061	rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
1062	if (fReused)
1063	goto flushPage;
1064
1065	/* A mov instruction to change the first page table entry will be remembered so we can detect
1066	* full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1067	*/
1068	if ( rc == VINF_SUCCESS
1069	&& !pPage->cLocked /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
1070	&& pDis->pCurInstr->uOpcode == OP_MOV
1071	&& (pvFault & PAGE_OFFSET_MASK) == 0)
1072	{
1073	pPage->GCPtrLastAccessHandlerFault = pvFault;
1074	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1075	pPage->GCPtrLastAccessHandlerRip = pRegFrame->rip;
1076	/* Make sure we don't kick out a page too quickly. */
1077	if (pPage->cModifications > 8)
1078	pPage->cModifications = 2;
1079	}
1080	else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
1081	{
1082	/* ignore the 2nd write to this page table entry. */
1083	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1084	}
1085	else
1086	{
1087	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
1088	pPage->GCPtrLastAccessHandlerRip = 0;
1089	}
1090
1091	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1092	pgmUnlock(pVM);
1093	return rc;
1094	}
1095
1096	/*
1097	* Windows is frequently doing small memset() operations (netio test 4k+).
1098	* We have to deal with these or we'll kill the cache and performance.
1099	*/
1100	if ( pDis->pCurInstr->uOpcode == OP_STOSWD
1101	&& !pRegFrame->eflags.Bits.u1DF
1102	&& pDis->uOpMode == pDis->uCpuMode
1103	&& pDis->uAddrMode == pDis->uCpuMode)
1104	{
1105	bool fValidStosd = false;
1106
1107	if ( pDis->uCpuMode == DISCPUMODE_32BIT
1108	&& pDis->fPrefix == DISPREFIX_REP
1109	&& pRegFrame->ecx <= 0x20
1110	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1111	&& !((uintptr_t)pvFault & 3)
1112	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1113	)
1114	{
1115	fValidStosd = true;
1116	pRegFrame->rcx &= 0xffffffff; /* paranoia */
1117	}
1118	else
1119	if ( pDis->uCpuMode == DISCPUMODE_64BIT
1120	&& pDis->fPrefix == (DISPREFIX_REP \| DISPREFIX_REX)
1121	&& pRegFrame->rcx <= 0x20
1122	&& pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1123	&& !((uintptr_t)pvFault & 7)
1124	&& (pRegFrame->rax == 0 \|\| pRegFrame->rax == 0x80) /* the two values observed. */
1125	)
1126	{
1127	fValidStosd = true;
1128	}
1129
1130	if (fValidStosd)
1131	{
1132	rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1133	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
1134	pgmUnlock(pVM);
1135	return rc;
1136	}
1137	}
1138
1139	/* REP prefix, don't bother. */
1140	STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
1141	Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1142	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
1143	fNotReusedNotForking = true;
1144	}
1145
1146	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
1147	/* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1148	* leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1149	*/
1150	if ( pPage->cModifications >= cMaxModifications
1151	&& !fForcedFlush
1152	&& (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT \|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1153	&& ( fNotReusedNotForking
1154	\|\| ( !pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage)
1155	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1156	)
1157	)
1158	{
1159	Assert(!pgmPoolIsPageLocked(pPage));
1160	Assert(pPage->fDirty == false);
1161
1162	/* Flush any monitored duplicates as we will disable write protection. */
1163	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1164	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1165	{
1166	PPGMPOOLPAGE pPageHead = pPage;
1167
1168	/* Find the monitor head. */
1169	while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1170	pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1171
1172	while (pPageHead)
1173	{
1174	unsigned idxNext = pPageHead->iMonitoredNext;
1175
1176	if (pPageHead != pPage)
1177	{
1178	STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1179	Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1180	int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1181	AssertRC(rc2);
1182	}
1183
1184	if (idxNext == NIL_PGMPOOL_IDX)
1185	break;
1186
1187	pPageHead = &pPool->aPages[idxNext];
1188	}
1189	}
1190
1191	/* The flushing above might fail for locked pages, so double check. */
1192	if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX
1193	&& pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1194	{
1195	pgmPoolAddDirtyPage(pVM, pPool, pPage);
1196
1197	/* Temporarily allow write access to the page table again. */
1198	rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK, pPage->GCPhys & PAGE_BASE_GC_MASK);
1199	if (rc == VINF_SUCCESS)
1200	{
1201	rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
1202	AssertMsg(rc == VINF_SUCCESS
1203	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1204	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1205	\|\| rc == VERR_PAGE_NOT_PRESENT,
1206	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1207	# ifdef VBOX_STRICT
1208	pPage->GCPtrDirtyFault = pvFault;
1209	# endif
1210
1211	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
1212	pgmUnlock(pVM);
1213	return rc;
1214	}
1215	}
1216	}
1217	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT && IN_RING0 */
1218
1219	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
1220	flushPage:
1221	/*
1222	* Not worth it, so flush it.
1223	*
1224	* If we considered it to be reused, don't go back to ring-3
1225	* to emulate failed instructions since we usually cannot
1226	* interpret then. This may be a bit risky, in which case
1227	* the reuse detection must be fixed.
1228	*/
1229	rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1230	if ( rc == VINF_EM_RAW_EMULATE_INSTR
1231	&& fReused)
1232	{
1233	/* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
1234	if (PGMShwGetPage(pVCpu, pRegFrame->rip, NULL, NULL) == VINF_SUCCESS)
1235	rc = VINF_SUCCESS; /* safe to restart the instruction. */
1236	}
1237	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
1238	pgmUnlock(pVM);
1239	return rc;
1240	}
1241
1242	#endif /* !IN_RING3 */
1243
1244	/**
1245	* @callback_method_impl{FNPGMPHYSHANDLER,
1246	* Access handler for shadowed page table pages.}
1247	*
1248	* @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
1249	*/
1250	PGM_ALL_CB2_DECL(VBOXSTRICTRC)
1251	pgmPoolAccessHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
1252	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
1253	{
1254	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1255	STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
1256	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1257	LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
1258	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
1259
1260	NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
1261
1262	pgmLock(pVM);
1263
1264	#ifdef VBOX_WITH_STATISTICS
1265	/*
1266	* Collect stats on the access.
1267	*/
1268	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
1269	if (cbBuf <= 16 && cbBuf > 0)
1270	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
1271	else if (cbBuf >= 17 && cbBuf < 32)
1272	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
1273	else if (cbBuf >= 32 && cbBuf < 64)
1274	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
1275	else if (cbBuf >= 64)
1276	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
1277
1278	uint8_t cbAlign;
1279	switch (pPage->enmKind)
1280	{
1281	default:
1282	cbAlign = 7;
1283	break;
1284	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1285	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1286	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1287	case PGMPOOLKIND_32BIT_PD:
1288	case PGMPOOLKIND_32BIT_PD_PHYS:
1289	cbAlign = 3;
1290	break;
1291	}
1292	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
1293	if ((uint8_t)GCPhys & cbAlign)
1294	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
1295	#endif
1296
1297	/*
1298	* Make sure the pool page wasn't modified by a different CPU.
1299	*/
1300	if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
1301	{
1302	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1303
1304	/* The max modification count before flushing depends on the context and page type. */
1305	#ifdef IN_RING3
1306	uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
1307	#else
1308	uint16_t cMaxModifications;
1309	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1310	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1311	cMaxModifications = 4;
1312	else
1313	cMaxModifications = 24;
1314	#endif
1315
1316	/*
1317	* We don't have to be very sophisticated about this since there are relativly few calls here.
1318	* However, we must try our best to detect any non-cpu accesses (disk / networking).
1319	*/
1320	if ( ( pPage->cModifications < cMaxModifications
1321	\|\| pgmPoolIsPageLocked(pPage) )
1322	&& enmOrigin != PGMACCESSORIGIN_DEVICE
1323	&& cbBuf <= 16)
1324	{
1325	/* Clear the shadow entry. */
1326	if (!pPage->cModifications++)
1327	pgmPoolMonitorModifiedInsert(pPool, pPage);
1328
1329	if (cbBuf <= 8)
1330	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
1331	else
1332	{
1333	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
1334	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
1335	}
1336	}
1337	else
1338	pgmPoolMonitorChainFlush(pPool, pPage);
1339
1340	STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1341	}
1342	else
1343	Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1344	pgmUnlock(pVM);
1345	return VINF_PGM_HANDLER_DO_DEFAULT;
1346	}
1347
1348
1349	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1350
1351	# if defined(VBOX_STRICT) && !defined(IN_RING3)
1352
1353	/**
1354	* Check references to guest physical memory in a PAE / PAE page table.
1355	*
1356	* @param pPool The pool.
1357	* @param pPage The page.
1358	* @param pShwPT The shadow page table (mapping of the page).
1359	* @param pGstPT The guest page table.
1360	*/
1361	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
1362	{
1363	unsigned cErrors = 0;
1364	int LastRc = -1; /* initialized to shut up gcc */
1365	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1366	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1367	PVMCC pVM = pPool->CTX_SUFF(pVM);
1368
1369	# ifdef VBOX_STRICT
1370	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1371	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1372	# endif
1373	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1374	{
1375	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1376	{
1377	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1378	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1379	if ( rc != VINF_SUCCESS
1380	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1381	{
1382	Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1383	LastPTE = i;
1384	LastRc = rc;
1385	LastHCPhys = HCPhys;
1386	cErrors++;
1387
1388	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1389	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1390	AssertRC(rc);
1391
1392	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1393	{
1394	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1395
1396	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1397	{
1398	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1399
1400	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1401	{
1402	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1403	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1404	{
1405	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1406	}
1407	}
1408
1409	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1410	}
1411	}
1412	}
1413	}
1414	}
1415	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1416	}
1417
1418
1419	/**
1420	* Check references to guest physical memory in a PAE / 32-bit page table.
1421	*
1422	* @param pPool The pool.
1423	* @param pPage The page.
1424	* @param pShwPT The shadow page table (mapping of the page).
1425	* @param pGstPT The guest page table.
1426	*/
1427	static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
1428	{
1429	unsigned cErrors = 0;
1430	int LastRc = -1; /* initialized to shut up gcc */
1431	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1432	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1433	PVMCC pVM = pPool->CTX_SUFF(pVM);
1434
1435	# ifdef VBOX_STRICT
1436	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1437	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1438	# endif
1439	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1440	{
1441	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1442	{
1443	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1444	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1445	if ( rc != VINF_SUCCESS
1446	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1447	{
1448	Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1449	LastPTE = i;
1450	LastRc = rc;
1451	LastHCPhys = HCPhys;
1452	cErrors++;
1453
1454	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1455	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1456	AssertRC(rc);
1457
1458	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1459	{
1460	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1461
1462	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1463	{
1464	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1465
1466	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1467	{
1468	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1469	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1470	{
1471	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1472	}
1473	}
1474
1475	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1476	}
1477	}
1478	}
1479	}
1480	}
1481	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1482	}
1483
1484	# endif /* VBOX_STRICT && !IN_RING3 */
1485
1486	/**
1487	* Clear references to guest physical memory in a PAE / PAE page table.
1488	*
1489	* @returns nr of changed PTEs
1490	* @param pPool The pool.
1491	* @param pPage The page.
1492	* @param pShwPT The shadow page table (mapping of the page).
1493	* @param pGstPT The guest page table.
1494	* @param pOldGstPT The old cached guest page table.
1495	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1496	* @param pfFlush Flush reused page table (out)
1497	*/
1498	DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
1499	PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1500	{
1501	unsigned cChanged = 0;
1502
1503	# ifdef VBOX_STRICT
1504	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1505	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1506	# endif
1507	*pfFlush = false;
1508
1509	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1510	{
1511	/* Check the new value written by the guest. If present and with a bogus physical address, then
1512	* it's fairly safe to assume the guest is reusing the PT.
1513	*/
1514	if ( fAllowRemoval
1515	&& (pGstPT->a[i].u & X86_PTE_P))
1516	{
1517	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1518	{
1519	*pfFlush = true;
1520	return ++cChanged;
1521	}
1522	}
1523	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1524	{
1525	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1526	if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1527	{
1528	# ifdef VBOX_STRICT
1529	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1530	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1531	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1532	# endif
1533	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1534	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1535	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1536	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1537
1538	if ( uHostAttr == uGuestAttr
1539	&& fHostRW <= fGuestRW)
1540	continue;
1541	}
1542	cChanged++;
1543	/* Something was changed, so flush it. */
1544	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
1545	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1546	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
1547	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1548	}
1549	}
1550	return cChanged;
1551	}
1552
1553
1554	/**
1555	* Clear references to guest physical memory in a PAE / PAE page table.
1556	*
1557	* @returns nr of changed PTEs
1558	* @param pPool The pool.
1559	* @param pPage The page.
1560	* @param pShwPT The shadow page table (mapping of the page).
1561	* @param pGstPT The guest page table.
1562	* @param pOldGstPT The old cached guest page table.
1563	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1564	* @param pfFlush Flush reused page table (out)
1565	*/
1566	DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
1567	PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1568	{
1569	unsigned cChanged = 0;
1570
1571	# ifdef VBOX_STRICT
1572	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1573	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1574	# endif
1575	*pfFlush = false;
1576
1577	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1578	{
1579	/* Check the new value written by the guest. If present and with a bogus physical address, then
1580	* it's fairly safe to assume the guest is reusing the PT. */
1581	if (fAllowRemoval)
1582	{
1583	X86PGUINT const uPte = pGstPT->a[i].u;
1584	if ( (uPte & X86_PTE_P)
1585	&& !PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), uPte & X86_PTE_PG_MASK))
1586	{
1587	*pfFlush = true;
1588	return ++cChanged;
1589	}
1590	}
1591	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1592	{
1593	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1594	if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
1595	{
1596	# ifdef VBOX_STRICT
1597	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1598	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1599	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1600	# endif
1601	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1602	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1603	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1604	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1605
1606	if ( uHostAttr == uGuestAttr
1607	&& fHostRW <= fGuestRW)
1608	continue;
1609	}
1610	cChanged++;
1611	/* Something was changed, so flush it. */
1612	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
1613	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
1614	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
1615	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1616	}
1617	}
1618	return cChanged;
1619	}
1620
1621
1622	/**
1623	* Flush a dirty page
1624	*
1625	* @param pVM The cross context VM structure.
1626	* @param pPool The pool.
1627	* @param idxSlot Dirty array slot index
1628	* @param fAllowRemoval Allow a reused page table to be removed
1629	*/
1630	static void pgmPoolFlushDirtyPage(PVMCC pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
1631	{
1632	AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages));
1633
1634	Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
1635	unsigned idxPage = pPool->aidxDirtyPages[idxSlot];
1636	if (idxPage == NIL_PGMPOOL_IDX)
1637	return;
1638
1639	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1640	Assert(pPage->idx == idxPage);
1641	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1642
1643	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1644	Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1645
1646	/* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
1647	int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
1648	Assert(rc == VINF_SUCCESS);
1649	pPage->fDirty = false;
1650
1651	# ifdef VBOX_STRICT
1652	uint64_t fFlags = 0;
1653	RTHCPHYS HCPhys;
1654	rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
1655	AssertMsg( ( rc == VINF_SUCCESS
1656	&& (!(fFlags & X86_PTE_RW) \|\| HCPhys != pPage->Core.Key))
1657	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1658	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1659	\|\| rc == VERR_PAGE_NOT_PRESENT,
1660	("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
1661	# endif
1662
1663	/* Flush those PTEs that have changed. */
1664	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1665	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1666	void *pvGst;
1667	rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1668	bool fFlush;
1669	unsigned cChanges;
1670
1671	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1672	cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
1673	(PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1674	else
1675	cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
1676	(PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1677
1678	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1679	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1680	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1681	/* Note: we might want to consider keeping the dirty page active in case there were many changes. */
1682
1683	/* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1684	Assert(pPage->cModifications);
1685	if (cChanges < 4)
1686	pPage->cModifications = 1; /* must use > 0 here */
1687	else
1688	pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1689
1690	STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1691	if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
1692	pPool->idxFreeDirtyPage = idxSlot;
1693
1694	pPool->cDirtyPages--;
1695	pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1696	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1697	if (fFlush)
1698	{
1699	Assert(fAllowRemoval);
1700	Log(("Flush reused page table!\n"));
1701	pgmPoolFlushPage(pPool, pPage);
1702	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1703	}
1704	else
1705	Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
1706	}
1707
1708
1709	# ifndef IN_RING3
1710	/**
1711	* Add a new dirty page
1712	*
1713	* @param pVM The cross context VM structure.
1714	* @param pPool The pool.
1715	* @param pPage The page.
1716	*/
1717	void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1718	{
1719	PGM_LOCK_ASSERT_OWNER(pVM);
1720	AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 \|\| RT_ELEMENTS(pPool->aDirtyPages) == 16);
1721	Assert(!pPage->fDirty);
1722
1723	unsigned idxFree = pPool->idxFreeDirtyPage;
1724	Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
1725	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1726
1727	if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
1728	{
1729	STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
1730	pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
1731	}
1732	Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
1733	AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1734
1735	Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1736
1737	/*
1738	* Make a copy of the guest page table as we require valid GCPhys addresses
1739	* when removing references to physical pages.
1740	* (The HCPhys linear lookup is extremely expensive!)
1741	*/
1742	void *pvGst;
1743	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1744	memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst, (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) ? PAGE_SIZE : PAGE_SIZE/2);
1745	# ifdef VBOX_STRICT
1746	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1747	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1748	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1749	else
1750	pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
1751	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1752	# endif
1753	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1754
1755	STAM_COUNTER_INC(&pPool->StatDirtyPage);
1756	pPage->fDirty = true;
1757	pPage->idxDirtyEntry = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
1758	pPool->aidxDirtyPages[idxFree] = pPage->idx;
1759	pPool->cDirtyPages++;
1760
1761	pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1762	if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
1763	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1764	{
1765	unsigned i;
1766	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1767	{
1768	idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1769	if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1770	{
1771	pPool->idxFreeDirtyPage = idxFree;
1772	break;
1773	}
1774	}
1775	Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
1776	}
1777
1778	Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) \|\| pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1779
1780	/*
1781	* Clear all references to this shadow table. See @bugref{7298}.
1782	*/
1783	pgmPoolTrackClearPageUsers(pPool, pPage);
1784	}
1785	# endif /* !IN_RING3 */
1786
1787
1788	/**
1789	* Check if the specified page is dirty (not write monitored)
1790	*
1791	* @return dirty or not
1792	* @param pVM The cross context VM structure.
1793	* @param GCPhys Guest physical address
1794	*/
1795	bool pgmPoolIsDirtyPageSlow(PVM pVM, RTGCPHYS GCPhys)
1796	{
1797	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1798	PGM_LOCK_ASSERT_OWNER(pVM);
1799	if (!pPool->cDirtyPages)
1800	return false;
1801
1802	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1803
1804	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1805	{
1806	unsigned idxPage = pPool->aidxDirtyPages[i];
1807	if (idxPage != NIL_PGMPOOL_IDX)
1808	{
1809	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1810	if (pPage->GCPhys == GCPhys)
1811	return true;
1812	}
1813	}
1814	return false;
1815	}
1816
1817
1818	/**
1819	* Reset all dirty pages by reinstating page monitoring.
1820	*
1821	* @param pVM The cross context VM structure.
1822	*/
1823	void pgmPoolResetDirtyPages(PVMCC pVM)
1824	{
1825	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1826	PGM_LOCK_ASSERT_OWNER(pVM);
1827	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1828
1829	if (!pPool->cDirtyPages)
1830	return;
1831
1832	Log(("pgmPoolResetDirtyPages\n"));
1833	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1834	pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
1835
1836	pPool->idxFreeDirtyPage = 0;
1837	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
1838	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1839	{
1840	unsigned i;
1841	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1842	{
1843	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
1844	{
1845	pPool->idxFreeDirtyPage = i;
1846	break;
1847	}
1848	}
1849	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
1850	}
1851
1852	Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX \|\| pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
1853	return;
1854	}
1855
1856
1857	/**
1858	* Invalidate the PT entry for the specified page
1859	*
1860	* @param pVM The cross context VM structure.
1861	* @param GCPtrPage Guest page to invalidate
1862	*/
1863	void pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage)
1864	{
1865	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1866	PGM_LOCK_ASSERT_OWNER(pVM);
1867	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1868
1869	if (!pPool->cDirtyPages)
1870	return;
1871
1872	Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
1873	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1874	{
1875	/** @todo What was intended here??? This looks incomplete... */
1876	}
1877	}
1878
1879
1880	/**
1881	* Reset all dirty pages by reinstating page monitoring.
1882	*
1883	* @param pVM The cross context VM structure.
1884	* @param GCPhysPT Physical address of the page table
1885	*/
1886	void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT)
1887	{
1888	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1889	PGM_LOCK_ASSERT_OWNER(pVM);
1890	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1891	unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
1892
1893	if (!pPool->cDirtyPages)
1894	return;
1895
1896	GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1897
1898	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1899	{
1900	unsigned idxPage = pPool->aidxDirtyPages[i];
1901	if (idxPage != NIL_PGMPOOL_IDX)
1902	{
1903	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1904	if (pPage->GCPhys == GCPhysPT)
1905	{
1906	idxDirtyPage = i;
1907	break;
1908	}
1909	}
1910	}
1911
1912	if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
1913	{
1914	pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
1915	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
1916	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1917	{
1918	unsigned i;
1919	for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1920	{
1921	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
1922	{
1923	pPool->idxFreeDirtyPage = i;
1924	break;
1925	}
1926	}
1927	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
1928	}
1929	}
1930	}
1931
1932	#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1933
1934	/**
1935	* Inserts a page into the GCPhys hash table.
1936	*
1937	* @param pPool The pool.
1938	* @param pPage The page.
1939	*/
1940	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1941	{
1942	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1943	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1944	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1945	pPage->iNext = pPool->aiHash[iHash];
1946	pPool->aiHash[iHash] = pPage->idx;
1947	}
1948
1949
1950	/**
1951	* Removes a page from the GCPhys hash table.
1952	*
1953	* @param pPool The pool.
1954	* @param pPage The page.
1955	*/
1956	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1957	{
1958	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1959	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1960	if (pPool->aiHash[iHash] == pPage->idx)
1961	pPool->aiHash[iHash] = pPage->iNext;
1962	else
1963	{
1964	uint16_t iPrev = pPool->aiHash[iHash];
1965	for (;;)
1966	{
1967	const int16_t i = pPool->aPages[iPrev].iNext;
1968	if (i == pPage->idx)
1969	{
1970	pPool->aPages[iPrev].iNext = pPage->iNext;
1971	break;
1972	}
1973	if (i == NIL_PGMPOOL_IDX)
1974	{
1975	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
1976	break;
1977	}
1978	iPrev = i;
1979	}
1980	}
1981	pPage->iNext = NIL_PGMPOOL_IDX;
1982	}
1983
1984
1985	/**
1986	* Frees up one cache page.
1987	*
1988	* @returns VBox status code.
1989	* @retval VINF_SUCCESS on success.
1990	* @param pPool The pool.
1991	* @param iUser The user index.
1992	*/
1993	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1994	{
1995	const PVMCC pVM = pPool->CTX_SUFF(pVM);
1996	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1997	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1998
1999	/*
2000	* Select one page from the tail of the age list.
2001	*/
2002	PPGMPOOLPAGE pPage;
2003	for (unsigned iLoop = 0; ; iLoop++)
2004	{
2005	uint16_t iToFree = pPool->iAgeTail;
2006	if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
2007	iToFree = pPool->aPages[iToFree].iAgePrev;
2008	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
2009	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
2010	{
2011	uint16_t i = pPool->aPages[iToFree].iAgePrev;
2012	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
2013	{
2014	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
2015	continue;
2016	iToFree = i;
2017	break;
2018	}
2019	}
2020	*/
2021	Assert(iToFree != iUser);
2022	AssertReleaseMsg(iToFree != NIL_PGMPOOL_IDX,
2023	("iToFree=%#x (iAgeTail=%#x) iUser=%#x iLoop=%u - pPool=%p LB %#zx\n",
2024	iToFree, pPool->iAgeTail, iUser, iLoop, pPool,
2025	RT_UOFFSETOF_DYN(PGMPOOL, aPages[pPool->cMaxPages])
2026	+ pPool->cMaxUsers * sizeof(PGMPOOLUSER)
2027	+ pPool->cMaxPhysExts * sizeof(PGMPOOLPHYSEXT) ));
2028
2029	pPage = &pPool->aPages[iToFree];
2030
2031	/*
2032	* Reject any attempts at flushing the currently active shadow CR3 mapping.
2033	* Call pgmPoolCacheUsed to move the page to the head of the age list.
2034	*/
2035	if ( !pgmPoolIsPageLocked(pPage)
2036	&& pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
2037	break;
2038	LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
2039	pgmPoolCacheUsed(pPool, pPage);
2040	AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
2041	}
2042
2043	/*
2044	* Found a usable page, flush it and return.
2045	*/
2046	int rc = pgmPoolFlushPage(pPool, pPage);
2047	/* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
2048	/** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
2049	if (rc == VINF_SUCCESS)
2050	PGM_INVL_ALL_VCPU_TLBS(pVM);
2051	return rc;
2052	}
2053
2054
2055	/**
2056	* Checks if a kind mismatch is really a page being reused
2057	* or if it's just normal remappings.
2058	*
2059	* @returns true if reused and the cached page (enmKind1) should be flushed
2060	* @returns false if not reused.
2061	* @param enmKind1 The kind of the cached page.
2062	* @param enmKind2 The kind of the requested page.
2063	*/
2064	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
2065	{
2066	switch (enmKind1)
2067	{
2068	/*
2069	* Never reuse them. There is no remapping in non-paging mode.
2070	*/
2071	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2072	case PGMPOOLKIND_32BIT_PD_PHYS:
2073	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2074	case PGMPOOLKIND_PAE_PD_PHYS:
2075	case PGMPOOLKIND_PAE_PDPT_PHYS:
2076	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2077	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2078	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2079	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2080	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2081	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
2082	return false;
2083
2084	/*
2085	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2086	*/
2087	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2088	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2089	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2090	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2091	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2092	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2093	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2094	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2095	case PGMPOOLKIND_32BIT_PD:
2096	case PGMPOOLKIND_PAE_PDPT:
2097	switch (enmKind2)
2098	{
2099	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2100	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2101	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2102	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2103	case PGMPOOLKIND_64BIT_PML4:
2104	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2105	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2106	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2107	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2108	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2109	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2110	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2111	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2112	return true;
2113	default:
2114	return false;
2115	}
2116
2117	/*
2118	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2119	*/
2120	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2121	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2122	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2123	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2124	case PGMPOOLKIND_64BIT_PML4:
2125	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2126	switch (enmKind2)
2127	{
2128	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2129	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2130	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2131	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2132	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2133	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2134	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2135	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2136	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2137	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2138	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2139	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2140	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2141	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2142	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2143	return true;
2144	default:
2145	return false;
2146	}
2147
2148	/*
2149	* These cannot be flushed, and it's common to reuse the PDs as PTs.
2150	*/
2151	case PGMPOOLKIND_ROOT_NESTED:
2152	return false;
2153
2154	default:
2155	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
2156	}
2157	}
2158
2159
2160	/**
2161	* Attempts to satisfy a pgmPoolAlloc request from the cache.
2162	*
2163	* @returns VBox status code.
2164	* @retval VINF_PGM_CACHED_PAGE on success.
2165	* @retval VERR_FILE_NOT_FOUND if not found.
2166	* @param pPool The pool.
2167	* @param GCPhys The GC physical address of the page we're gonna shadow.
2168	* @param enmKind The kind of mapping.
2169	* @param enmAccess Access type for the mapping (only relevant for big pages)
2170	* @param fA20Enabled Whether the CPU has the A20 gate enabled.
2171	* @param iUser The shadow page pool index of the user table. This is
2172	* NIL_PGMPOOL_IDX for root pages.
2173	* @param iUserTable The index into the user table (shadowed). Ignored if
2174	* root page
2175	* @param ppPage Where to store the pointer to the page.
2176	*/
2177	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
2178	uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
2179	{
2180	/*
2181	* Look up the GCPhys in the hash.
2182	*/
2183	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2184	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
2185	if (i != NIL_PGMPOOL_IDX)
2186	{
2187	do
2188	{
2189	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2190	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
2191	if (pPage->GCPhys == GCPhys)
2192	{
2193	if ( (PGMPOOLKIND)pPage->enmKind == enmKind
2194	&& (PGMPOOLACCESS)pPage->enmAccess == enmAccess
2195	&& pPage->fA20Enabled == fA20Enabled)
2196	{
2197	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
2198	* doesn't flush it in case there are no more free use records.
2199	*/
2200	pgmPoolCacheUsed(pPool, pPage);
2201
2202	int rc = VINF_SUCCESS;
2203	if (iUser != NIL_PGMPOOL_IDX)
2204	rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
2205	if (RT_SUCCESS(rc))
2206	{
2207	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
2208	*ppPage = pPage;
2209	if (pPage->cModifications)
2210	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
2211	STAM_COUNTER_INC(&pPool->StatCacheHits);
2212	return VINF_PGM_CACHED_PAGE;
2213	}
2214	return rc;
2215	}
2216
2217	if ((PGMPOOLKIND)pPage->enmKind != enmKind)
2218	{
2219	/*
2220	* The kind is different. In some cases we should now flush the page
2221	* as it has been reused, but in most cases this is normal remapping
2222	* of PDs as PT or big pages using the GCPhys field in a slightly
2223	* different way than the other kinds.
2224	*/
2225	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
2226	{
2227	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
2228	pgmPoolFlushPage(pPool, pPage);
2229	break;
2230	}
2231	}
2232	}
2233
2234	/* next */
2235	i = pPage->iNext;
2236	} while (i != NIL_PGMPOOL_IDX);
2237	}
2238
2239	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
2240	STAM_COUNTER_INC(&pPool->StatCacheMisses);
2241	return VERR_FILE_NOT_FOUND;
2242	}
2243
2244
2245	/**
2246	* Inserts a page into the cache.
2247	*
2248	* @param pPool The pool.
2249	* @param pPage The cached page.
2250	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
2251	*/
2252	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
2253	{
2254	/*
2255	* Insert into the GCPhys hash if the page is fit for that.
2256	*/
2257	Assert(!pPage->fCached);
2258	if (fCanBeCached)
2259	{
2260	pPage->fCached = true;
2261	pgmPoolHashInsert(pPool, pPage);
2262	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2263	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2264	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
2265	}
2266	else
2267	{
2268	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2269	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2270	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
2271	}
2272
2273	/*
2274	* Insert at the head of the age list.
2275	*/
2276	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2277	pPage->iAgeNext = pPool->iAgeHead;
2278	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
2279	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
2280	else
2281	pPool->iAgeTail = pPage->idx;
2282	pPool->iAgeHead = pPage->idx;
2283	}
2284
2285
2286	/**
2287	* Flushes a cached page.
2288	*
2289	* @param pPool The pool.
2290	* @param pPage The cached page.
2291	*/
2292	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2293	{
2294	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
2295
2296	/*
2297	* Remove the page from the hash.
2298	*/
2299	if (pPage->fCached)
2300	{
2301	pPage->fCached = false;
2302	pgmPoolHashRemove(pPool, pPage);
2303	}
2304	else
2305	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2306
2307	/*
2308	* Remove it from the age list.
2309	*/
2310	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
2311	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
2312	else
2313	pPool->iAgeTail = pPage->iAgePrev;
2314	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
2315	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
2316	else
2317	pPool->iAgeHead = pPage->iAgeNext;
2318	pPage->iAgeNext = NIL_PGMPOOL_IDX;
2319	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2320	}
2321
2322
2323	/**
2324	* Looks for pages sharing the monitor.
2325	*
2326	* @returns Pointer to the head page.
2327	* @returns NULL if not found.
2328	* @param pPool The Pool
2329	* @param pNewPage The page which is going to be monitored.
2330	*/
2331	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
2332	{
2333	/*
2334	* Look up the GCPhys in the hash.
2335	*/
2336	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2337	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2338	if (i == NIL_PGMPOOL_IDX)
2339	return NULL;
2340	do
2341	{
2342	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2343	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
2344	&& pPage != pNewPage)
2345	{
2346	switch (pPage->enmKind)
2347	{
2348	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2349	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2350	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2351	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2352	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2353	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2354	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2355	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2356	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2357	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2358	case PGMPOOLKIND_64BIT_PML4:
2359	case PGMPOOLKIND_32BIT_PD:
2360	case PGMPOOLKIND_PAE_PDPT:
2361	{
2362	/* find the head */
2363	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2364	{
2365	Assert(pPage->iMonitoredPrev != pPage->idx);
2366	pPage = &pPool->aPages[pPage->iMonitoredPrev];
2367	}
2368	return pPage;
2369	}
2370
2371	/* ignore, no monitoring. */
2372	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2373	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2374	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2375	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2376	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2377	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2378	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2379	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2380	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2381	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2382	case PGMPOOLKIND_ROOT_NESTED:
2383	case PGMPOOLKIND_PAE_PD_PHYS:
2384	case PGMPOOLKIND_PAE_PDPT_PHYS:
2385	case PGMPOOLKIND_32BIT_PD_PHYS:
2386	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2387	break;
2388	default:
2389	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2390	}
2391	}
2392
2393	/* next */
2394	i = pPage->iNext;
2395	} while (i != NIL_PGMPOOL_IDX);
2396	return NULL;
2397	}
2398
2399
2400	/**
2401	* Enabled write monitoring of a guest page.
2402	*
2403	* @returns VBox status code.
2404	* @retval VINF_SUCCESS on success.
2405	* @param pPool The pool.
2406	* @param pPage The cached page.
2407	*/
2408	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2409	{
2410	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
2411
2412	/*
2413	* Filter out the relevant kinds.
2414	*/
2415	switch (pPage->enmKind)
2416	{
2417	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2418	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2419	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2420	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2421	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2422	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2423	case PGMPOOLKIND_64BIT_PML4:
2424	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2425	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2426	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2427	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2428	case PGMPOOLKIND_32BIT_PD:
2429	case PGMPOOLKIND_PAE_PDPT:
2430	break;
2431
2432	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2433	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2434	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2435	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2436	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2437	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2438	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2439	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2440	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2441	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2442	case PGMPOOLKIND_ROOT_NESTED:
2443	/* Nothing to monitor here. */
2444	return VINF_SUCCESS;
2445
2446	case PGMPOOLKIND_32BIT_PD_PHYS:
2447	case PGMPOOLKIND_PAE_PDPT_PHYS:
2448	case PGMPOOLKIND_PAE_PD_PHYS:
2449	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2450	/* Nothing to monitor here. */
2451	return VINF_SUCCESS;
2452	default:
2453	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2454	}
2455
2456	/*
2457	* Install handler.
2458	*/
2459	int rc;
2460	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2461	if (pPageHead)
2462	{
2463	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2464	Assert(pPageHead->iMonitoredPrev != pPage->idx);
2465
2466	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2467	if (pPageHead->fDirty)
2468	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
2469	#endif
2470
2471	pPage->iMonitoredPrev = pPageHead->idx;
2472	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2473	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2474	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2475	pPageHead->iMonitoredNext = pPage->idx;
2476	rc = VINF_SUCCESS;
2477	}
2478	else
2479	{
2480	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2481	PVMCC pVM = pPool->CTX_SUFF(pVM);
2482	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2483	rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
2484	MMHyperCCToR3(pVM, pPage), MMHyperCCToR0(pVM, pPage), MMHyperCCToRC(pVM, pPage),
2485	NIL_RTR3PTR /pszDesc/);
2486	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2487	* the heap size should suffice. */
2488	AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2489	PVMCPU pVCpu = VMMGetCpu(pVM);
2490	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
2491	}
2492	pPage->fMonitored = true;
2493	return rc;
2494	}
2495
2496
2497	/**
2498	* Disables write monitoring of a guest page.
2499	*
2500	* @returns VBox status code.
2501	* @retval VINF_SUCCESS on success.
2502	* @param pPool The pool.
2503	* @param pPage The cached page.
2504	*/
2505	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2506	{
2507	/*
2508	* Filter out the relevant kinds.
2509	*/
2510	switch (pPage->enmKind)
2511	{
2512	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2513	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2514	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2515	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2516	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2517	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2518	case PGMPOOLKIND_64BIT_PML4:
2519	case PGMPOOLKIND_32BIT_PD:
2520	case PGMPOOLKIND_PAE_PDPT:
2521	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2522	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2523	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2524	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2525	break;
2526
2527	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2528	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2529	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2530	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2531	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2532	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2533	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2534	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2535	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2536	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2537	case PGMPOOLKIND_ROOT_NESTED:
2538	case PGMPOOLKIND_PAE_PD_PHYS:
2539	case PGMPOOLKIND_PAE_PDPT_PHYS:
2540	case PGMPOOLKIND_32BIT_PD_PHYS:
2541	/* Nothing to monitor here. */
2542	Assert(!pPage->fMonitored);
2543	return VINF_SUCCESS;
2544
2545	default:
2546	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2547	}
2548	Assert(pPage->fMonitored);
2549
2550	/*
2551	* Remove the page from the monitored list or uninstall it if last.
2552	*/
2553	const PVMCC pVM = pPool->CTX_SUFF(pVM);
2554	int rc;
2555	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2556	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2557	{
2558	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2559	{
2560	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2561	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2562	rc = PGMHandlerPhysicalChangeUserArgs(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK,
2563	MMHyperCCToR3(pVM, pNewHead), MMHyperCCToR0(pVM, pNewHead));
2564
2565	AssertFatalRCSuccess(rc);
2566	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2567	}
2568	else
2569	{
2570	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2571	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2572	{
2573	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2574	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2575	}
2576	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2577	rc = VINF_SUCCESS;
2578	}
2579	}
2580	else
2581	{
2582	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
2583	AssertFatalRC(rc);
2584	PVMCPU pVCpu = VMMGetCpu(pVM);
2585	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2586	("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2587	}
2588	pPage->fMonitored = false;
2589
2590	/*
2591	* Remove it from the list of modified pages (if in it).
2592	*/
2593	pgmPoolMonitorModifiedRemove(pPool, pPage);
2594
2595	return rc;
2596	}
2597
2598
2599	/**
2600	* Inserts the page into the list of modified pages.
2601	*
2602	* @param pPool The pool.
2603	* @param pPage The page.
2604	*/
2605	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2606	{
2607	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2608	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2609	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2610	&& pPool->iModifiedHead != pPage->idx,
2611	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2612	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2613	pPool->iModifiedHead, pPool->cModifiedPages));
2614
2615	pPage->iModifiedNext = pPool->iModifiedHead;
2616	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2617	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2618	pPool->iModifiedHead = pPage->idx;
2619	pPool->cModifiedPages++;
2620	#ifdef VBOX_WITH_STATISTICS
2621	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2622	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2623	#endif
2624	}
2625
2626
2627	/**
2628	* Removes the page from the list of modified pages and resets the
2629	* modification counter.
2630	*
2631	* @param pPool The pool.
2632	* @param pPage The page which is believed to be in the list of modified pages.
2633	*/
2634	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2635	{
2636	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2637	if (pPool->iModifiedHead == pPage->idx)
2638	{
2639	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2640	pPool->iModifiedHead = pPage->iModifiedNext;
2641	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2642	{
2643	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2644	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2645	}
2646	pPool->cModifiedPages--;
2647	}
2648	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2649	{
2650	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2651	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2652	{
2653	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2654	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2655	}
2656	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2657	pPool->cModifiedPages--;
2658	}
2659	else
2660	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2661	pPage->cModifications = 0;
2662	}
2663
2664
2665	/**
2666	* Zaps the list of modified pages, resetting their modification counters in the process.
2667	*
2668	* @param pVM The cross context VM structure.
2669	*/
2670	static void pgmPoolMonitorModifiedClearAll(PVMCC pVM)
2671	{
2672	pgmLock(pVM);
2673	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2674	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2675
2676	unsigned cPages = 0; NOREF(cPages);
2677
2678	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2679	pgmPoolResetDirtyPages(pVM);
2680	#endif
2681
2682	uint16_t idx = pPool->iModifiedHead;
2683	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2684	while (idx != NIL_PGMPOOL_IDX)
2685	{
2686	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2687	idx = pPage->iModifiedNext;
2688	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2689	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2690	pPage->cModifications = 0;
2691	Assert(++cPages);
2692	}
2693	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2694	pPool->cModifiedPages = 0;
2695	pgmUnlock(pVM);
2696	}
2697
2698
2699	/**
2700	* Handle SyncCR3 pool tasks
2701	*
2702	* @returns VBox status code.
2703	* @retval VINF_SUCCESS if successfully added.
2704	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2705	* @param pVCpu The cross context virtual CPU structure.
2706	* @remark Should only be used when monitoring is available, thus placed in
2707	* the PGMPOOL_WITH_MONITORING \#ifdef.
2708	*/
2709	int pgmPoolSyncCR3(PVMCPUCC pVCpu)
2710	{
2711	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2712	LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
2713
2714	/*
2715	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2716	* Occasionally we will have to clear all the shadow page tables because we wanted
2717	* to monitor a page which was mapped by too many shadowed page tables. This operation
2718	* sometimes referred to as a 'lightweight flush'.
2719	*/
2720	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2721	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2722	pgmR3PoolClearAll(pVM, false /fFlushRemTlb/);
2723	# else /* !IN_RING3 */
2724	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2725	{
2726	Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2727	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2728
2729	/* Make sure all other VCPUs return to ring 3. */
2730	if (pVM->cCpus > 1)
2731	{
2732	VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
2733	PGM_INVL_ALL_VCPU_TLBS(pVM);
2734	}
2735	return VINF_PGM_SYNC_CR3;
2736	}
2737	# endif /* !IN_RING3 */
2738	else
2739	{
2740	pgmPoolMonitorModifiedClearAll(pVM);
2741
2742	/* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
2743	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2744	{
2745	Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
2746	return pgmPoolSyncCR3(pVCpu);
2747	}
2748	}
2749	return VINF_SUCCESS;
2750	}
2751
2752
2753	/**
2754	* Frees up at least one user entry.
2755	*
2756	* @returns VBox status code.
2757	* @retval VINF_SUCCESS if successfully added.
2758	*
2759	* @param pPool The pool.
2760	* @param iUser The user index.
2761	*/
2762	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2763	{
2764	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2765	/*
2766	* Just free cached pages in a braindead fashion.
2767	*/
2768	/** @todo walk the age list backwards and free the first with usage. */
2769	int rc = VINF_SUCCESS;
2770	do
2771	{
2772	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2773	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2774	rc = rc2;
2775	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2776	return rc;
2777	}
2778
2779
2780	/**
2781	* Inserts a page into the cache.
2782	*
2783	* This will create user node for the page, insert it into the GCPhys
2784	* hash, and insert it into the age list.
2785	*
2786	* @returns VBox status code.
2787	* @retval VINF_SUCCESS if successfully added.
2788	*
2789	* @param pPool The pool.
2790	* @param pPage The cached page.
2791	* @param GCPhys The GC physical address of the page we're gonna shadow.
2792	* @param iUser The user index.
2793	* @param iUserTable The user table index.
2794	*/
2795	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2796	{
2797	int rc = VINF_SUCCESS;
2798	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2799
2800	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
2801
2802	if (iUser != NIL_PGMPOOL_IDX)
2803	{
2804	#ifdef VBOX_STRICT
2805	/*
2806	* Check that the entry doesn't already exists.
2807	*/
2808	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2809	{
2810	uint16_t i = pPage->iUserHead;
2811	do
2812	{
2813	Assert(i < pPool->cMaxUsers);
2814	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2815	i = paUsers[i].iNext;
2816	} while (i != NIL_PGMPOOL_USER_INDEX);
2817	}
2818	#endif
2819
2820	/*
2821	* Find free a user node.
2822	*/
2823	uint16_t i = pPool->iUserFreeHead;
2824	if (i == NIL_PGMPOOL_USER_INDEX)
2825	{
2826	rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2827	if (RT_FAILURE(rc))
2828	return rc;
2829	i = pPool->iUserFreeHead;
2830	}
2831
2832	/*
2833	* Unlink the user node from the free list,
2834	* initialize and insert it into the user list.
2835	*/
2836	pPool->iUserFreeHead = paUsers[i].iNext;
2837	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2838	paUsers[i].iUser = iUser;
2839	paUsers[i].iUserTable = iUserTable;
2840	pPage->iUserHead = i;
2841	}
2842	else
2843	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2844
2845
2846	/*
2847	* Insert into cache and enable monitoring of the guest page if enabled.
2848	*
2849	* Until we implement caching of all levels, including the CR3 one, we'll
2850	* have to make sure we don't try monitor & cache any recursive reuse of
2851	* a monitored CR3 page. Because all windows versions are doing this we'll
2852	* have to be able to do combined access monitoring, CR3 + PT and
2853	* PD + PT (guest PAE).
2854	*
2855	* Update:
2856	* We're now cooperating with the CR3 monitor if an uncachable page is found.
2857	*/
2858	const bool fCanBeMonitored = true;
2859	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2860	if (fCanBeMonitored)
2861	{
2862	rc = pgmPoolMonitorInsert(pPool, pPage);
2863	AssertRC(rc);
2864	}
2865	return rc;
2866	}
2867
2868
2869	/**
2870	* Adds a user reference to a page.
2871	*
2872	* This will move the page to the head of the
2873	*
2874	* @returns VBox status code.
2875	* @retval VINF_SUCCESS if successfully added.
2876	*
2877	* @param pPool The pool.
2878	* @param pPage The cached page.
2879	* @param iUser The user index.
2880	* @param iUserTable The user table.
2881	*/
2882	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2883	{
2884	Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
2885	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2886	Assert(iUser != NIL_PGMPOOL_IDX);
2887
2888	# ifdef VBOX_STRICT
2889	/*
2890	* Check that the entry doesn't already exists. We only allow multiple
2891	* users of top-level paging structures (SHW_POOL_ROOT_IDX).
2892	*/
2893	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2894	{
2895	uint16_t i = pPage->iUserHead;
2896	do
2897	{
2898	Assert(i < pPool->cMaxUsers);
2899	/** @todo this assertion looks odd... Shouldn't it be && here? */
2900	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2901	i = paUsers[i].iNext;
2902	} while (i != NIL_PGMPOOL_USER_INDEX);
2903	}
2904	# endif
2905
2906	/*
2907	* Allocate a user node.
2908	*/
2909	uint16_t i = pPool->iUserFreeHead;
2910	if (i == NIL_PGMPOOL_USER_INDEX)
2911	{
2912	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2913	if (RT_FAILURE(rc))
2914	return rc;
2915	i = pPool->iUserFreeHead;
2916	}
2917	pPool->iUserFreeHead = paUsers[i].iNext;
2918
2919	/*
2920	* Initialize the user node and insert it.
2921	*/
2922	paUsers[i].iNext = pPage->iUserHead;
2923	paUsers[i].iUser = iUser;
2924	paUsers[i].iUserTable = iUserTable;
2925	pPage->iUserHead = i;
2926
2927	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2928	if (pPage->fDirty)
2929	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
2930	# endif
2931
2932	/*
2933	* Tell the cache to update its replacement stats for this page.
2934	*/
2935	pgmPoolCacheUsed(pPool, pPage);
2936	return VINF_SUCCESS;
2937	}
2938
2939
2940	/**
2941	* Frees a user record associated with a page.
2942	*
2943	* This does not clear the entry in the user table, it simply replaces the
2944	* user record to the chain of free records.
2945	*
2946	* @param pPool The pool.
2947	* @param pPage The shadow page.
2948	* @param iUser The shadow page pool index of the user table.
2949	* @param iUserTable The index into the user table (shadowed).
2950	*
2951	* @remarks Don't call this for root pages.
2952	*/
2953	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2954	{
2955	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
2956	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2957	Assert(iUser != NIL_PGMPOOL_IDX);
2958
2959	/*
2960	* Unlink and free the specified user entry.
2961	*/
2962
2963	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2964	uint16_t i = pPage->iUserHead;
2965	if ( i != NIL_PGMPOOL_USER_INDEX
2966	&& paUsers[i].iUser == iUser
2967	&& paUsers[i].iUserTable == iUserTable)
2968	{
2969	pPage->iUserHead = paUsers[i].iNext;
2970
2971	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2972	paUsers[i].iNext = pPool->iUserFreeHead;
2973	pPool->iUserFreeHead = i;
2974	return;
2975	}
2976
2977	/* General: Linear search. */
2978	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2979	while (i != NIL_PGMPOOL_USER_INDEX)
2980	{
2981	if ( paUsers[i].iUser == iUser
2982	&& paUsers[i].iUserTable == iUserTable)
2983	{
2984	if (iPrev != NIL_PGMPOOL_USER_INDEX)
2985	paUsers[iPrev].iNext = paUsers[i].iNext;
2986	else
2987	pPage->iUserHead = paUsers[i].iNext;
2988
2989	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2990	paUsers[i].iNext = pPool->iUserFreeHead;
2991	pPool->iUserFreeHead = i;
2992	return;
2993	}
2994	iPrev = i;
2995	i = paUsers[i].iNext;
2996	}
2997
2998	/* Fatal: didn't find it */
2999	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
3000	iUser, iUserTable, pPage->GCPhys));
3001	}
3002
3003
3004	#if 0 /* unused */
3005	/**
3006	* Gets the entry size of a shadow table.
3007	*
3008	* @param enmKind The kind of page.
3009	*
3010	* @returns The size of the entry in bytes. That is, 4 or 8.
3011	* @returns If the kind is not for a table, an assertion is raised and 0 is
3012	* returned.
3013	*/
3014	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
3015	{
3016	switch (enmKind)
3017	{
3018	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3019	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3020	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3021	case PGMPOOLKIND_32BIT_PD:
3022	case PGMPOOLKIND_32BIT_PD_PHYS:
3023	return 4;
3024
3025	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3026	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3027	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3028	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3029	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3030	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3031	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3032	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3033	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3034	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3035	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3036	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3037	case PGMPOOLKIND_64BIT_PML4:
3038	case PGMPOOLKIND_PAE_PDPT:
3039	case PGMPOOLKIND_ROOT_NESTED:
3040	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3041	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3042	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3043	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3044	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3045	case PGMPOOLKIND_PAE_PD_PHYS:
3046	case PGMPOOLKIND_PAE_PDPT_PHYS:
3047	return 8;
3048
3049	default:
3050	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3051	}
3052	}
3053	#endif /* unused */
3054
3055	#if 0 /* unused */
3056	/**
3057	* Gets the entry size of a guest table.
3058	*
3059	* @param enmKind The kind of page.
3060	*
3061	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
3062	* @returns If the kind is not for a table, an assertion is raised and 0 is
3063	* returned.
3064	*/
3065	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
3066	{
3067	switch (enmKind)
3068	{
3069	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3070	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3071	case PGMPOOLKIND_32BIT_PD:
3072	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3073	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3074	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3075	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3076	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3077	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3078	return 4;
3079
3080	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3081	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3082	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3083	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3084	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3085	case PGMPOOLKIND_64BIT_PML4:
3086	case PGMPOOLKIND_PAE_PDPT:
3087	return 8;
3088
3089	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3090	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3091	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3092	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3093	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3094	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3095	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3096	case PGMPOOLKIND_ROOT_NESTED:
3097	case PGMPOOLKIND_PAE_PD_PHYS:
3098	case PGMPOOLKIND_PAE_PDPT_PHYS:
3099	case PGMPOOLKIND_32BIT_PD_PHYS:
3100	/** @todo can we return 0? (nobody is calling this...) */
3101	AssertFailed();
3102	return 0;
3103
3104	default:
3105	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3106	}
3107	}
3108	#endif /* unused */
3109
3110
3111	/**
3112	* Checks one shadow page table entry for a mapping of a physical page.
3113	*
3114	* @returns true / false indicating removal of all relevant PTEs
3115	*
3116	* @param pVM The cross context VM structure.
3117	* @param pPhysPage The guest page in question.
3118	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3119	* @param iShw The shadow page table.
3120	* @param iPte Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
3121	*/
3122	static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
3123	{
3124	LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
3125	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3126	bool fRet = false;
3127
3128	/*
3129	* Assert sanity.
3130	*/
3131	Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
3132	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
3133	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
3134
3135	/*
3136	* Then, clear the actual mappings to the page in the shadow PT.
3137	*/
3138	switch (pPage->enmKind)
3139	{
3140	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3141	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3142	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3143	{
3144	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3145	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3146	uint32_t u32AndMask = 0;
3147	uint32_t u32OrMask = 0;
3148
3149	if (!fFlushPTEs)
3150	{
3151	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3152	{
3153	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3154	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3155	u32OrMask = X86_PTE_RW;
3156	u32AndMask = UINT32_MAX;
3157	fRet = true;
3158	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3159	break;
3160
3161	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3162	u32OrMask = 0;
3163	u32AndMask = ~X86_PTE_RW;
3164	fRet = true;
3165	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3166	break;
3167	default:
3168	/* (shouldn't be here, will assert below) */
3169	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3170	break;
3171	}
3172	}
3173	else
3174	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3175
3176	/* Update the counter if we're removing references. */
3177	if (!u32AndMask)
3178	{
3179	Assert(pPage->cPresent);
3180	Assert(pPool->cPresent);
3181	pPage->cPresent--;
3182	pPool->cPresent--;
3183	}
3184
3185	if ((pPT->a[iPte].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3186	{
3187	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
3188	X86PTE Pte;
3189	Pte.u = (pPT->a[iPte].u & u32AndMask) \| u32OrMask;
3190	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3191	Pte.u &= ~(X86PGUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3192	ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
3193	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3194	return fRet;
3195	}
3196	#ifdef LOG_ENABLED
3197	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3198	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3199	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3200	{
3201	Log(("i=%d cFound=%d\n", i, ++cFound));
3202	}
3203	#endif
3204	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
3205	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3206	break;
3207	}
3208
3209	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3210	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3211	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3212	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3213	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3214	case PGMPOOLKIND_EPT_PT_FOR_PHYS: /* physical mask the same as PAE; RW bit as well; be careful! */
3215	{
3216	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3217	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3218	uint64_t u64OrMask = 0;
3219	uint64_t u64AndMask = 0;
3220
3221	if (!fFlushPTEs)
3222	{
3223	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3224	{
3225	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3226	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3227	u64OrMask = X86_PTE_RW;
3228	u64AndMask = UINT64_MAX;
3229	fRet = true;
3230	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3231	break;
3232
3233	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3234	u64OrMask = 0;
3235	u64AndMask = ~(uint64_t)X86_PTE_RW;
3236	fRet = true;
3237	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3238	break;
3239
3240	default:
3241	/* (shouldn't be here, will assert below) */
3242	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3243	break;
3244	}
3245	}
3246	else
3247	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3248
3249	/* Update the counter if we're removing references. */
3250	if (!u64AndMask)
3251	{
3252	Assert(pPage->cPresent);
3253	Assert(pPool->cPresent);
3254	pPage->cPresent--;
3255	pPool->cPresent--;
3256	}
3257
3258	if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3259	{
3260	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3261	X86PTEPAE Pte;
3262	Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) \| u64OrMask;
3263	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3264	Pte.u &= ~(X86PGPAEUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3265
3266	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
3267	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3268	return fRet;
3269	}
3270	#ifdef LOG_ENABLED
3271	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3272	Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX), u64));
3273	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3274	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3275	Log(("i=%d cFound=%d\n", i, ++cFound));
3276	#endif
3277	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3278	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3279	break;
3280	}
3281
3282	#ifdef PGM_WITH_LARGE_PAGES
3283	/* Large page case only. */
3284	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3285	{
3286	Assert(pVM->pgm.s.fNestedPaging);
3287
3288	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3289	PEPTPD pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3290
3291	if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3292	{
3293	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3294	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3295	pPD->a[iPte].u = 0;
3296	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3297
3298	/* Update the counter as we're removing references. */
3299	Assert(pPage->cPresent);
3300	Assert(pPool->cPresent);
3301	pPage->cPresent--;
3302	pPool->cPresent--;
3303
3304	return fRet;
3305	}
3306	# ifdef LOG_ENABLED
3307	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3308	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3309	if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3310	Log(("i=%d cFound=%d\n", i, ++cFound));
3311	# endif
3312	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3313	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3314	break;
3315	}
3316
3317	/* AMD-V nested paging / /* @todo merge with EPT as we only check the parts that are identical. */
3318	case PGMPOOLKIND_PAE_PD_PHYS:
3319	{
3320	Assert(pVM->pgm.s.fNestedPaging);
3321
3322	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3323	PX86PDPAE pPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3324
3325	if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3326	{
3327	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3328	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3329	pPD->a[iPte].u = 0;
3330	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3331
3332	/* Update the counter as we're removing references. */
3333	Assert(pPage->cPresent);
3334	Assert(pPool->cPresent);
3335	pPage->cPresent--;
3336	pPool->cPresent--;
3337	return fRet;
3338	}
3339	# ifdef LOG_ENABLED
3340	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3341	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3342	if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3343	Log(("i=%d cFound=%d\n", i, ++cFound));
3344	# endif
3345	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3346	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3347	break;
3348	}
3349	#endif /* PGM_WITH_LARGE_PAGES */
3350
3351	default:
3352	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3353	}
3354
3355	/* not reached. */
3356	#ifndef _MSC_VER
3357	return fRet;
3358	#endif
3359	}
3360
3361
3362	/**
3363	* Scans one shadow page table for mappings of a physical page.
3364	*
3365	* @param pVM The cross context VM structure.
3366	* @param pPhysPage The guest page in question.
3367	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3368	* @param iShw The shadow page table.
3369	*/
3370	static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
3371	{
3372	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3373
3374	/* We should only come here with when there's only one reference to this physical page. */
3375	Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
3376
3377	Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
3378	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3379	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
3380	if (!fKeptPTEs)
3381	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3382	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3383	}
3384
3385
3386	/**
3387	* Flushes a list of shadow page tables mapping the same physical page.
3388	*
3389	* @param pVM The cross context VM structure.
3390	* @param pPhysPage The guest page in question.
3391	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3392	* @param iPhysExt The physical cross reference extent list to flush.
3393	*/
3394	static void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
3395	{
3396	PGM_LOCK_ASSERT_OWNER(pVM);
3397	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3398	bool fKeepList = false;
3399
3400	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3401	Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
3402
3403	const uint16_t iPhysExtStart = iPhysExt;
3404	PPGMPOOLPHYSEXT pPhysExt;
3405	do
3406	{
3407	Assert(iPhysExt < pPool->cMaxPhysExts);
3408	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3409	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3410	{
3411	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3412	{
3413	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
3414	if (!fKeptPTEs)
3415	{
3416	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3417	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3418	}
3419	else
3420	fKeepList = true;
3421	}
3422	}
3423	/* next */
3424	iPhysExt = pPhysExt->iNext;
3425	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3426
3427	if (!fKeepList)
3428	{
3429	/* insert the list into the free list and clear the ram range entry. */
3430	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3431	pPool->iPhysExtFreeHead = iPhysExtStart;
3432	/* Invalidate the tracking data. */
3433	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3434	}
3435
3436	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3437	}
3438
3439
3440	/**
3441	* Flushes all shadow page table mappings of the given guest page.
3442	*
3443	* This is typically called when the host page backing the guest one has been
3444	* replaced or when the page protection was changed due to a guest access
3445	* caught by the monitoring.
3446	*
3447	* @returns VBox status code.
3448	* @retval VINF_SUCCESS if all references has been successfully cleared.
3449	* @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3450	* pool cleaning. FF and sync flags are set.
3451	*
3452	* @param pVM The cross context VM structure.
3453	* @param GCPhysPage GC physical address of the page in question
3454	* @param pPhysPage The guest page in question.
3455	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3456	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3457	* flushed, it is NOT touched if this isn't necessary.
3458	* The caller MUST initialized this to @a false.
3459	*/
3460	int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
3461	{
3462	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3463	pgmLock(pVM);
3464	int rc = VINF_SUCCESS;
3465
3466	#ifdef PGM_WITH_LARGE_PAGES
3467	/* Is this page part of a large page? */
3468	if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
3469	{
3470	RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
3471	GCPhysPage &= X86_PDE_PAE_PG_MASK;
3472
3473	/* Fetch the large page base. */
3474	PPGMPAGE pLargePage;
3475	if (GCPhysBase != GCPhysPage)
3476	{
3477	pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
3478	AssertFatal(pLargePage);
3479	}
3480	else
3481	pLargePage = pPhysPage;
3482
3483	Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
3484
3485	if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
3486	{
3487	/* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
3488	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3489	pVM->pgm.s.cLargePagesDisabled++;
3490
3491	/* Update the base as that only that one has a reference and there's only one PDE to clear. */
3492	rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
3493
3494	*pfFlushTLBs = true;
3495	pgmUnlock(pVM);
3496	return rc;
3497	}
3498	}
3499	#else
3500	NOREF(GCPhysPage);
3501	#endif /* PGM_WITH_LARGE_PAGES */
3502
3503	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3504	if (u16)
3505	{
3506	/*
3507	* The zero page is currently screwing up the tracking and we'll
3508	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3509	* is defined, zero pages won't normally be mapped. Some kind of solution
3510	* will be needed for this problem of course, but it will have to wait...
3511	*/
3512	if ( PGM_PAGE_IS_ZERO(pPhysPage)
3513	\|\| PGM_PAGE_IS_BALLOONED(pPhysPage))
3514	rc = VINF_PGM_GCPHYS_ALIASED;
3515	else
3516	{
3517	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3518	{
3519	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3520	pgmPoolTrackFlushGCPhysPT(pVM,
3521	pPhysPage,
3522	fFlushPTEs,
3523	PGMPOOL_TD_GET_IDX(u16));
3524	}
3525	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3526	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
3527	else
3528	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3529	*pfFlushTLBs = true;
3530	}
3531	}
3532
3533	if (rc == VINF_PGM_GCPHYS_ALIASED)
3534	{
3535	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3536	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3537	rc = VINF_PGM_SYNC_CR3;
3538	}
3539	pgmUnlock(pVM);
3540	return rc;
3541	}
3542
3543
3544	/**
3545	* Scans all shadow page tables for mappings of a physical page.
3546	*
3547	* This may be slow, but it's most likely more efficient than cleaning
3548	* out the entire page pool / cache.
3549	*
3550	* @returns VBox status code.
3551	* @retval VINF_SUCCESS if all references has been successfully cleared.
3552	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3553	* a page pool cleaning.
3554	*
3555	* @param pVM The cross context VM structure.
3556	* @param pPhysPage The guest page in question.
3557	*/
3558	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage)
3559	{
3560	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3561	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3562	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3563	pPool->cUsedPages, pPool->cPresent, pPhysPage));
3564
3565	/*
3566	* There is a limit to what makes sense.
3567	*/
3568	if ( pPool->cPresent > 1024
3569	&& pVM->cCpus == 1)
3570	{
3571	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3572	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3573	return VINF_PGM_GCPHYS_ALIASED;
3574	}
3575
3576	/*
3577	* Iterate all the pages until we've encountered all that in use.
3578	* This is simple but not quite optimal solution.
3579	*/
3580	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage);
3581	unsigned cLeft = pPool->cUsedPages;
3582	unsigned iPage = pPool->cCurPages;
3583	while (--iPage >= PGMPOOL_IDX_FIRST)
3584	{
3585	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3586	if ( pPage->GCPhys != NIL_RTGCPHYS
3587	&& pPage->cPresent)
3588	{
3589	switch (pPage->enmKind)
3590	{
3591	/*
3592	* We only care about shadow page tables.
3593	*/
3594	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3595	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3596	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3597	{
3598	const uint32_t u32 = (uint32_t)u64;
3599	unsigned cPresent = pPage->cPresent;
3600	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3601	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3602	{
3603	const X86PGUINT uPte = pPT->a[i].u;
3604	if (uPte & X86_PTE_P)
3605	{
3606	if ((uPte & X86_PTE_PG_MASK) == u32)
3607	{
3608	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3609	ASMAtomicWriteU32(&pPT->a[i].u, 0);
3610
3611	/* Update the counter as we're removing references. */
3612	Assert(pPage->cPresent);
3613	Assert(pPool->cPresent);
3614	pPage->cPresent--;
3615	pPool->cPresent--;
3616	}
3617	if (!--cPresent)
3618	break;
3619	}
3620	}
3621	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3622	break;
3623	}
3624
3625	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3626	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3627	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3628	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3629	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3630	{
3631	unsigned cPresent = pPage->cPresent;
3632	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3633	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3634	if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
3635	{
3636	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_PAE_PG_MASK) == u64)
3637	{
3638	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3639	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[i], 0); /// @todo why not atomic?
3640
3641	/* Update the counter as we're removing references. */
3642	Assert(pPage->cPresent);
3643	Assert(pPool->cPresent);
3644	pPage->cPresent--;
3645	pPool->cPresent--;
3646	}
3647	if (!--cPresent)
3648	break;
3649	}
3650	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3651	break;
3652	}
3653
3654	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3655	{
3656	unsigned cPresent = pPage->cPresent;
3657	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3658	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3659	{
3660	X86PGPAEUINT const uPte = pPT->a[i].u;
3661	if (uPte & EPT_E_READ)
3662	{
3663	if ((uPte & EPT_PTE_PG_MASK) == u64)
3664	{
3665	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3666	ASMAtomicWriteU64(&pPT->a[i].u, 0);
3667
3668	/* Update the counter as we're removing references. */
3669	Assert(pPage->cPresent);
3670	Assert(pPool->cPresent);
3671	pPage->cPresent--;
3672	pPool->cPresent--;
3673	}
3674	if (!--cPresent)
3675	break;
3676	}
3677	}
3678	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3679	break;
3680	}
3681	}
3682
3683	if (!--cLeft)
3684	break;
3685	}
3686	}
3687
3688	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3689	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3690
3691	/*
3692	* There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
3693	*/
3694	if (pPool->cPresent > 1024)
3695	{
3696	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3697	return VINF_PGM_GCPHYS_ALIASED;
3698	}
3699
3700	return VINF_SUCCESS;
3701	}
3702
3703
3704	/**
3705	* Clears the user entry in a user table.
3706	*
3707	* This is used to remove all references to a page when flushing it.
3708	*/
3709	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3710	{
3711	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3712	Assert(pUser->iUser < pPool->cCurPages);
3713	uint32_t iUserTable = pUser->iUserTable;
3714
3715	/*
3716	* Map the user page. Ignore references made by fictitious pages.
3717	*/
3718	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3719	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3720	union
3721	{
3722	uint64_t *pau64;
3723	uint32_t *pau32;
3724	} u;
3725	if (pUserPage->idx < PGMPOOL_IDX_FIRST)
3726	{
3727	Assert(!pUserPage->pvPageR3);
3728	return;
3729	}
3730	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3731
3732
3733	/* Safety precaution in case we change the paging for other modes too in the future. */
3734	Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
3735
3736	#ifdef VBOX_STRICT
3737	/*
3738	* Some sanity checks.
3739	*/
3740	switch (pUserPage->enmKind)
3741	{
3742	case PGMPOOLKIND_32BIT_PD:
3743	case PGMPOOLKIND_32BIT_PD_PHYS:
3744	Assert(iUserTable < X86_PG_ENTRIES);
3745	break;
3746	case PGMPOOLKIND_PAE_PDPT:
3747	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3748	case PGMPOOLKIND_PAE_PDPT_PHYS:
3749	Assert(iUserTable < 4);
3750	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3751	break;
3752	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3753	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3754	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3755	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3756	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3757	case PGMPOOLKIND_PAE_PD_PHYS:
3758	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3759	break;
3760	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3761	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3762	#ifndef PGM_WITHOUT_MAPPINGS
3763	Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
3764	#endif
3765	break;
3766	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3767	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3768	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3769	break;
3770	case PGMPOOLKIND_64BIT_PML4:
3771	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3772	/* GCPhys >> PAGE_SHIFT is the index here */
3773	break;
3774	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3775	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3776	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3777	break;
3778
3779	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3780	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3781	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3782	break;
3783
3784	case PGMPOOLKIND_ROOT_NESTED:
3785	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3786	break;
3787
3788	default:
3789	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3790	break;
3791	}
3792	#endif /* VBOX_STRICT */
3793
3794	/*
3795	* Clear the entry in the user page.
3796	*/
3797	switch (pUserPage->enmKind)
3798	{
3799	/* 32-bit entries */
3800	case PGMPOOLKIND_32BIT_PD:
3801	case PGMPOOLKIND_32BIT_PD_PHYS:
3802	ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
3803	break;
3804
3805	/* 64-bit entries */
3806	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3807	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3808	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3809	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3810	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3811	case PGMPOOLKIND_PAE_PD_PHYS:
3812	case PGMPOOLKIND_PAE_PDPT_PHYS:
3813	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3814	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3815	case PGMPOOLKIND_64BIT_PML4:
3816	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3817	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3818	case PGMPOOLKIND_PAE_PDPT:
3819	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3820	case PGMPOOLKIND_ROOT_NESTED:
3821	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3822	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3823	ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
3824	break;
3825
3826	default:
3827	AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
3828	}
3829	PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
3830	}
3831
3832
3833	/**
3834	* Clears all users of a page.
3835	*/
3836	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3837	{
3838	/*
3839	* Free all the user records.
3840	*/
3841	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
3842
3843	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3844	uint16_t i = pPage->iUserHead;
3845	while (i != NIL_PGMPOOL_USER_INDEX)
3846	{
3847	/* Clear enter in user table. */
3848	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
3849
3850	/* Free it. */
3851	const uint16_t iNext = paUsers[i].iNext;
3852	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3853	paUsers[i].iNext = pPool->iUserFreeHead;
3854	pPool->iUserFreeHead = i;
3855
3856	/* Next. */
3857	i = iNext;
3858	}
3859	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3860	}
3861
3862
3863	/**
3864	* Allocates a new physical cross reference extent.
3865	*
3866	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
3867	* @param pVM The cross context VM structure.
3868	* @param piPhysExt Where to store the phys ext index.
3869	*/
3870	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
3871	{
3872	PGM_LOCK_ASSERT_OWNER(pVM);
3873	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3874	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3875	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3876	{
3877	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3878	return NULL;
3879	}
3880	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3881	pPool->iPhysExtFreeHead = pPhysExt->iNext;
3882	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3883	*piPhysExt = iPhysExt;
3884	return pPhysExt;
3885	}
3886
3887
3888	/**
3889	* Frees a physical cross reference extent.
3890	*
3891	* @param pVM The cross context VM structure.
3892	* @param iPhysExt The extent to free.
3893	*/
3894	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
3895	{
3896	PGM_LOCK_ASSERT_OWNER(pVM);
3897	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3898	Assert(iPhysExt < pPool->cMaxPhysExts);
3899	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3900	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3901	{
3902	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3903	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3904	}
3905	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3906	pPool->iPhysExtFreeHead = iPhysExt;
3907	}
3908
3909
3910	/**
3911	* Frees a physical cross reference extent.
3912	*
3913	* @param pVM The cross context VM structure.
3914	* @param iPhysExt The extent to free.
3915	*/
3916	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
3917	{
3918	PGM_LOCK_ASSERT_OWNER(pVM);
3919	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3920
3921	const uint16_t iPhysExtStart = iPhysExt;
3922	PPGMPOOLPHYSEXT pPhysExt;
3923	do
3924	{
3925	Assert(iPhysExt < pPool->cMaxPhysExts);
3926	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3927	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3928	{
3929	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3930	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3931	}
3932
3933	/* next */
3934	iPhysExt = pPhysExt->iNext;
3935	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3936
3937	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3938	pPool->iPhysExtFreeHead = iPhysExtStart;
3939	}
3940
3941
3942	/**
3943	* Insert a reference into a list of physical cross reference extents.
3944	*
3945	* @returns The new tracking data for PGMPAGE.
3946	*
3947	* @param pVM The cross context VM structure.
3948	* @param iPhysExt The physical extent index of the list head.
3949	* @param iShwPT The shadow page table index.
3950	* @param iPte Page table entry
3951	*
3952	*/
3953	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
3954	{
3955	PGM_LOCK_ASSERT_OWNER(pVM);
3956	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3957	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3958
3959	/*
3960	* Special common cases.
3961	*/
3962	if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
3963	{
3964	paPhysExts[iPhysExt].aidx[1] = iShwPT;
3965	paPhysExts[iPhysExt].apte[1] = iPte;
3966	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
3967	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
3968	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3969	}
3970	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3971	{
3972	paPhysExts[iPhysExt].aidx[2] = iShwPT;
3973	paPhysExts[iPhysExt].apte[2] = iPte;
3974	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
3975	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
3976	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3977	}
3978	AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
3979
3980	/*
3981	* General treatment.
3982	*/
3983	const uint16_t iPhysExtStart = iPhysExt;
3984	unsigned cMax = 15;
3985	for (;;)
3986	{
3987	Assert(iPhysExt < pPool->cMaxPhysExts);
3988	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3989	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3990	{
3991	paPhysExts[iPhysExt].aidx[i] = iShwPT;
3992	paPhysExts[iPhysExt].apte[i] = iPte;
3993	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
3994	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
3995	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
3996	}
3997	if (!--cMax)
3998	{
3999	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackOverflows);
4000	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4001	LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
4002	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4003	}
4004
4005	/* advance */
4006	iPhysExt = paPhysExts[iPhysExt].iNext;
4007	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4008	break;
4009	}
4010
4011	/*
4012	* Add another extent to the list.
4013	*/
4014	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4015	if (!pNew)
4016	{
4017	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackNoExtentsLeft);
4018	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4019	LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
4020	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4021	}
4022	pNew->iNext = iPhysExtStart;
4023	pNew->aidx[0] = iShwPT;
4024	pNew->apte[0] = iPte;
4025	LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
4026	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4027	}
4028
4029
4030	/**
4031	* Add a reference to guest physical page where extents are in use.
4032	*
4033	* @returns The new tracking data for PGMPAGE.
4034	*
4035	* @param pVM The cross context VM structure.
4036	* @param pPhysPage Pointer to the aPages entry in the ram range.
4037	* @param u16 The ram range flags (top 16-bits).
4038	* @param iShwPT The shadow page table index.
4039	* @param iPte Page table entry
4040	*/
4041	uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
4042	{
4043	pgmLock(pVM);
4044	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
4045	{
4046	/*
4047	* Convert to extent list.
4048	*/
4049	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
4050	uint16_t iPhysExt;
4051	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4052	if (pPhysExt)
4053	{
4054	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
4055	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliased);
4056	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
4057	pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
4058	pPhysExt->aidx[1] = iShwPT;
4059	pPhysExt->apte[1] = iPte;
4060	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4061	}
4062	else
4063	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4064	}
4065	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
4066	{
4067	/*
4068	* Insert into the extent list.
4069	*/
4070	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
4071	}
4072	else
4073	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedLots);
4074	pgmUnlock(pVM);
4075	return u16;
4076	}
4077
4078
4079	/**
4080	* Clear references to guest physical memory.
4081	*
4082	* @param pPool The pool.
4083	* @param pPage The page.
4084	* @param pPhysPage Pointer to the aPages entry in the ram range.
4085	* @param iPte Shadow PTE index
4086	*/
4087	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
4088	{
4089	PVMCC pVM = pPool->CTX_SUFF(pVM);
4090	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4091	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4092
4093	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
4094	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
4095	{
4096	pgmLock(pVM);
4097
4098	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
4099	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4100	do
4101	{
4102	Assert(iPhysExt < pPool->cMaxPhysExts);
4103
4104	/*
4105	* Look for the shadow page and check if it's all freed.
4106	*/
4107	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4108	{
4109	if ( paPhysExts[iPhysExt].aidx[i] == pPage->idx
4110	&& paPhysExts[iPhysExt].apte[i] == iPte)
4111	{
4112	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
4113	paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4114
4115	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4116	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
4117	{
4118	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
4119	pgmUnlock(pVM);
4120	return;
4121	}
4122
4123	/* we can free the node. */
4124	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
4125	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
4126	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
4127	{
4128	/* lonely node */
4129	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4130	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
4131	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
4132	}
4133	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
4134	{
4135	/* head */
4136	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
4137	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
4138	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4139	}
4140	else
4141	{
4142	/* in list */
4143	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
4144	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
4145	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4146	}
4147	iPhysExt = iPhysExtNext;
4148	pgmUnlock(pVM);
4149	return;
4150	}
4151	}
4152
4153	/* next */
4154	iPhysExtPrev = iPhysExt;
4155	iPhysExt = paPhysExts[iPhysExt].iNext;
4156	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4157
4158	pgmUnlock(pVM);
4159	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4160	}
4161	else /* nothing to do */
4162	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
4163	}
4164
4165	/**
4166	* Clear references to guest physical memory.
4167	*
4168	* This is the same as pgmPoolTracDerefGCPhysHint except that the guest
4169	* physical address is assumed to be correct, so the linear search can be
4170	* skipped and we can assert at an earlier point.
4171	*
4172	* @param pPool The pool.
4173	* @param pPage The page.
4174	* @param HCPhys The host physical address corresponding to the guest page.
4175	* @param GCPhys The guest physical address corresponding to HCPhys.
4176	* @param iPte Shadow PTE index
4177	*/
4178	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
4179	{
4180	/*
4181	* Lookup the page and check if it checks out before derefing it.
4182	*/
4183	PVMCC pVM = pPool->CTX_SUFF(pVM);
4184	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
4185	if (pPhysPage)
4186	{
4187	Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
4188	#ifdef LOG_ENABLED
4189	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
4190	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
4191	#endif
4192	if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
4193	{
4194	Assert(pPage->cPresent);
4195	Assert(pPool->cPresent);
4196	pPage->cPresent--;
4197	pPool->cPresent--;
4198	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4199	return;
4200	}
4201
4202	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp\n",
4203	HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage)));
4204	}
4205	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
4206	}
4207
4208
4209	/**
4210	* Clear references to guest physical memory.
4211	*
4212	* @param pPool The pool.
4213	* @param pPage The page.
4214	* @param HCPhys The host physical address corresponding to the guest page.
4215	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
4216	* @param iPte Shadow pte index
4217	*/
4218	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
4219	{
4220	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
4221
4222	/*
4223	* Try the hint first.
4224	*/
4225	RTHCPHYS HCPhysHinted;
4226	PVMCC pVM = pPool->CTX_SUFF(pVM);
4227	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
4228	if (pPhysPage)
4229	{
4230	HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
4231	Assert(HCPhysHinted);
4232	if (HCPhysHinted == HCPhys)
4233	{
4234	Assert(pPage->cPresent);
4235	Assert(pPool->cPresent);
4236	pPage->cPresent--;
4237	pPool->cPresent--;
4238	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4239	return;
4240	}
4241	}
4242	else
4243	HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
4244
4245	/*
4246	* Damn, the hint didn't work. We'll have to do an expensive linear search.
4247	*/
4248	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
4249	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
4250	while (pRam)
4251	{
4252	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4253	while (iPage-- > 0)
4254	{
4255	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
4256	{
4257	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
4258	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
4259	Assert(pPage->cPresent);
4260	Assert(pPool->cPresent);
4261	pPage->cPresent--;
4262	pPool->cPresent--;
4263	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
4264	return;
4265	}
4266	}
4267	pRam = pRam->CTX_SUFF(pNext);
4268	}
4269
4270	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
4271	}
4272
4273
4274	/**
4275	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
4276	*
4277	* @param pPool The pool.
4278	* @param pPage The page.
4279	* @param pShwPT The shadow page table (mapping of the page).
4280	* @param pGstPT The guest page table.
4281	*/
4282	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
4283	{
4284	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4285	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4286	{
4287	const X86PGUINT uPte = pShwPT->a[i].u;
4288	Assert(!(uPte & RT_BIT_32(10)));
4289	if (uPte & X86_PTE_P)
4290	{
4291	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
4292	i, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
4293	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
4294	if (!pPage->cPresent)
4295	break;
4296	}
4297	}
4298	}
4299
4300
4301	/**
4302	* Clear references to guest physical memory in a PAE / 32-bit page table.
4303	*
4304	* @param pPool The pool.
4305	* @param pPage The page.
4306	* @param pShwPT The shadow page table (mapping of the page).
4307	* @param pGstPT The guest page table (just a half one).
4308	*/
4309	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
4310	{
4311	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4312	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4313	{
4314	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4315	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4316	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4317	{
4318	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
4319	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
4320	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4321	if (!pPage->cPresent)
4322	break;
4323	}
4324	}
4325	}
4326
4327
4328	/**
4329	* Clear references to guest physical memory in a PAE / PAE page table.
4330	*
4331	* @param pPool The pool.
4332	* @param pPage The page.
4333	* @param pShwPT The shadow page table (mapping of the page).
4334	* @param pGstPT The guest page table.
4335	*/
4336	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
4337	{
4338	RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
4339	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4340	{
4341	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4342	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4343	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4344	{
4345	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
4346	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
4347	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4348	if (!pPage->cPresent)
4349	break;
4350	}
4351	}
4352	}
4353
4354
4355	/**
4356	* Clear references to guest physical memory in a 32-bit / 4MB page table.
4357	*
4358	* @param pPool The pool.
4359	* @param pPage The page.
4360	* @param pShwPT The shadow page table (mapping of the page).
4361	*/
4362	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
4363	{
4364	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4365	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4366	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4367	{
4368	const X86PGUINT uPte = pShwPT->a[i].u;
4369	Assert(!(uPte & RT_BIT_32(10)));
4370	if (uPte & X86_PTE_P)
4371	{
4372	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
4373	i, uPte & X86_PTE_PG_MASK, GCPhys));
4374	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4375	if (!pPage->cPresent)
4376	break;
4377	}
4378	}
4379	}
4380
4381
4382	/**
4383	* Clear references to guest physical memory in a PAE / 2/4MB page table.
4384	*
4385	* @param pPool The pool.
4386	* @param pPage The page.
4387	* @param pShwPT The shadow page table (mapping of the page).
4388	*/
4389	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
4390	{
4391	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4392	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4393	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4394	{
4395	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4396	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4397	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4398	{
4399	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
4400	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
4401	pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
4402	if (!pPage->cPresent)
4403	break;
4404	}
4405	}
4406	}
4407
4408
4409	/**
4410	* Clear references to shadowed pages in an EPT page table.
4411	*
4412	* @param pPool The pool.
4413	* @param pPage The page.
4414	* @param pShwPT The shadow page directory pointer table (mapping of the
4415	* page).
4416	*/
4417	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4418	{
4419	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4420	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4421	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4422	{
4423	X86PGPAEUINT const uPte = pShwPT->a[i].u;
4424	Assert((uPte & UINT64_C(0xfff0000000000f80)) == 0);
4425	if (uPte & EPT_E_READ)
4426	{
4427	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4428	i, uPte & EPT_PTE_PG_MASK, pPage->GCPhys));
4429	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4430	if (!pPage->cPresent)
4431	break;
4432	}
4433	}
4434	}
4435
4436
4437	/**
4438	* Clear references to shadowed pages in a 32 bits page directory.
4439	*
4440	* @param pPool The pool.
4441	* @param pPage The page.
4442	* @param pShwPD The shadow page directory (mapping of the page).
4443	*/
4444	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
4445	{
4446	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4447	{
4448	X86PGUINT const uPde = pShwPD->a[i].u;
4449	#ifndef PGM_WITHOUT_MAPPINGS
4450	if ((uPde & (X86_PDE_P \| PGM_PDFLAGS_MAPPING)) == X86_PDE_P)
4451	#else
4452	if (uPde & X86_PDE_P)
4453	#endif
4454	{
4455	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
4456	if (pSubPage)
4457	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4458	else
4459	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
4460	}
4461	}
4462	}
4463
4464
4465	/**
4466	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
4467	*
4468	* @param pPool The pool.
4469	* @param pPage The page.
4470	* @param pShwPD The shadow page directory (mapping of the page).
4471	*/
4472	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
4473	{
4474	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4475	{
4476	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4477	#ifndef PGM_WITHOUT_MAPPINGS
4478	if ((uPde & (X86_PDE_P \| PGM_PDFLAGS_MAPPING)) == X86_PDE_P)
4479	#else
4480	if (uPde & X86_PDE_P)
4481	#endif
4482	{
4483	#ifdef PGM_WITH_LARGE_PAGES
4484	if (uPde & X86_PDE_PS)
4485	{
4486	Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
4487	i, uPde & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4488	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & X86_PDE2M_PAE_PG_MASK,
4489	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4490	i);
4491	}
4492	else
4493	#endif
4494	{
4495	Assert((uPde & (X86_PDE_PAE_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000000))) == 0);
4496	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & X86_PDE_PAE_PG_MASK);
4497	if (pSubPage)
4498	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4499	else
4500	AssertFatalMsgFailed(("%RX64\n", uPde & X86_PDE_PAE_PG_MASK));
4501	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4502	}
4503	}
4504	}
4505	}
4506
4507
4508	/**
4509	* Clear references to shadowed pages in a PAE page directory pointer table.
4510	*
4511	* @param pPool The pool.
4512	* @param pPage The page.
4513	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4514	*/
4515	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4516	{
4517	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4518	{
4519	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4520	Assert((uPdpe & (X86_PDPE_PAE_MBZ_MASK \| UINT64_C(0x7ff0000000000200))) == 0);
4521	if ( uPdpe & X86_PDPE_P
4522	#ifndef PGM_WITHOUT_MAPPINGS
4523	&& !(uPdpe & PGM_PLXFLAGS_MAPPING)
4524	#endif
4525	)
4526	{
4527	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4528	if (pSubPage)
4529	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4530	else
4531	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4532	}
4533	}
4534	}
4535
4536
4537	/**
4538	* Clear references to shadowed pages in a 64-bit page directory pointer table.
4539	*
4540	* @param pPool The pool.
4541	* @param pPage The page.
4542	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4543	*/
4544	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4545	{
4546	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4547	{
4548	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4549	Assert((uPdpe & (X86_PDPE_LM_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4550	if (uPdpe & X86_PDPE_P)
4551	{
4552	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4553	if (pSubPage)
4554	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4555	else
4556	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4557	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4558	}
4559	}
4560	}
4561
4562
4563	/**
4564	* Clear references to shadowed pages in a 64-bit level 4 page table.
4565	*
4566	* @param pPool The pool.
4567	* @param pPage The page.
4568	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4569	*/
4570	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4571	{
4572	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4573	{
4574	X86PGPAEUINT const uPml4e = pShwPML4->a[i].u;
4575	Assert((uPml4e & (X86_PML4E_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4576	if (uPml4e & X86_PML4E_P)
4577	{
4578	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & X86_PDPE_PG_MASK);
4579	if (pSubPage)
4580	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4581	else
4582	AssertFatalMsgFailed(("%RX64\n", uPml4e & X86_PML4E_PG_MASK));
4583	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4584	}
4585	}
4586	}
4587
4588
4589	/**
4590	* Clear references to shadowed pages in an EPT page directory.
4591	*
4592	* @param pPool The pool.
4593	* @param pPage The page.
4594	* @param pShwPD The shadow page directory (mapping of the page).
4595	*/
4596	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4597	{
4598	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4599	{
4600	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4601	Assert((uPde & UINT64_C(0xfff0000000000f80)) == 0);
4602	if (uPde & EPT_E_READ)
4603	{
4604	#ifdef PGM_WITH_LARGE_PAGES
4605	if (uPde & EPT_E_LEAF)
4606	{
4607	Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
4608	i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
4609	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK,
4610	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4611	i);
4612	}
4613	else
4614	#endif
4615	{
4616	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK);
4617	if (pSubPage)
4618	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4619	else
4620	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4621	}
4622	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4623	}
4624	}
4625	}
4626
4627
4628	/**
4629	* Clear references to shadowed pages in an EPT page directory pointer table.
4630	*
4631	* @param pPool The pool.
4632	* @param pPage The page.
4633	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4634	*/
4635	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4636	{
4637	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4638	{
4639	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4640	Assert((uPdpe & UINT64_C(0xfff0000000000f80)) == 0);
4641	if (uPdpe & EPT_E_READ)
4642	{
4643	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & EPT_PDPTE_PG_MASK);
4644	if (pSubPage)
4645	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4646	else
4647	AssertFatalMsgFailed(("%RX64\n", uPdpe & EPT_PDPTE_PG_MASK));
4648	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4649	}
4650	}
4651	}
4652
4653
4654	/**
4655	* Clears all references made by this page.
4656	*
4657	* This includes other shadow pages and GC physical addresses.
4658	*
4659	* @param pPool The pool.
4660	* @param pPage The page.
4661	*/
4662	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4663	{
4664	/*
4665	* Map the shadow page and take action according to the page kind.
4666	*/
4667	PVMCC pVM = pPool->CTX_SUFF(pVM);
4668	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4669	switch (pPage->enmKind)
4670	{
4671	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4672	{
4673	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4674	void *pvGst;
4675	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4676	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4677	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4678	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4679	break;
4680	}
4681
4682	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4683	{
4684	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4685	void *pvGst;
4686	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4687	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
4688	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4689	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4690	break;
4691	}
4692
4693	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4694	{
4695	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4696	void *pvGst;
4697	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4698	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
4699	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4700	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4701	break;
4702	}
4703
4704	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4705	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4706	{
4707	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4708	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4709	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4710	break;
4711	}
4712
4713	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
4714	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4715	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4716	{
4717	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4718	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
4719	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4720	break;
4721	}
4722
4723	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4724	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4725	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4726	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4727	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4728	case PGMPOOLKIND_PAE_PD_PHYS:
4729	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4730	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4731	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
4732	break;
4733
4734	case PGMPOOLKIND_32BIT_PD_PHYS:
4735	case PGMPOOLKIND_32BIT_PD:
4736	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
4737	break;
4738
4739	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4740	case PGMPOOLKIND_PAE_PDPT:
4741	case PGMPOOLKIND_PAE_PDPT_PHYS:
4742	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
4743	break;
4744
4745	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4746	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4747	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
4748	break;
4749
4750	case PGMPOOLKIND_64BIT_PML4:
4751	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
4752	break;
4753
4754	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4755	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
4756	break;
4757
4758	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4759	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
4760	break;
4761
4762	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4763	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
4764	break;
4765
4766	default:
4767	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
4768	}
4769
4770	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
4771	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4772	ASMMemZeroPage(pvShw);
4773	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4774	pPage->fZeroed = true;
4775	Assert(!pPage->cPresent);
4776	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
4777	}
4778
4779
4780	/**
4781	* Flushes a pool page.
4782	*
4783	* This moves the page to the free list after removing all user references to it.
4784	*
4785	* @returns VBox status code.
4786	* @retval VINF_SUCCESS on success.
4787	* @param pPool The pool.
4788	* @param pPage The shadow page.
4789	* @param fFlush Flush the TLBS when required (should only be false in very specific use cases!!)
4790	*/
4791	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
4792	{
4793	PVMCC pVM = pPool->CTX_SUFF(pVM);
4794	bool fFlushRequired = false;
4795
4796	int rc = VINF_SUCCESS;
4797	STAM_PROFILE_START(&pPool->StatFlushPage, f);
4798	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4799	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4800
4801	/*
4802	* Reject any attempts at flushing any of the special root pages (shall
4803	* not happen).
4804	*/
4805	AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
4806	("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
4807	pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
4808	VINF_SUCCESS);
4809
4810	pgmLock(pVM);
4811
4812	/*
4813	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4814	*/
4815	if (pgmPoolIsPageLocked(pPage))
4816	{
4817	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4818	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4819	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4820	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
4821	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4822	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4823	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4824	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4825	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
4826	\|\| pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
4827	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
4828	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4829	pgmUnlock(pVM);
4830	return VINF_SUCCESS;
4831	}
4832
4833	/*
4834	* Mark the page as being in need of an ASMMemZeroPage().
4835	*/
4836	pPage->fZeroed = false;
4837
4838	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4839	if (pPage->fDirty)
4840	pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
4841	#endif
4842
4843	/* If there are any users of this table, then we must issue a tlb flush on all VCPUs. */
4844	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
4845	fFlushRequired = true;
4846
4847	/*
4848	* Clear the page.
4849	*/
4850	pgmPoolTrackClearPageUsers(pPool, pPage);
4851	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
4852	pgmPoolTrackDeref(pPool, pPage);
4853	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
4854
4855	/*
4856	* Flush it from the cache.
4857	*/
4858	pgmPoolCacheFlushPage(pPool, pPage);
4859
4860	/*
4861	* Deregistering the monitoring.
4862	*/
4863	if (pPage->fMonitored)
4864	rc = pgmPoolMonitorFlush(pPool, pPage);
4865
4866	/*
4867	* Free the page.
4868	*/
4869	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
4870	pPage->iNext = pPool->iFreeHead;
4871	pPool->iFreeHead = pPage->idx;
4872	pPage->enmKind = PGMPOOLKIND_FREE;
4873	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4874	pPage->GCPhys = NIL_RTGCPHYS;
4875	pPage->fReusedFlushPending = false;
4876
4877	pPool->cUsedPages--;
4878
4879	/* Flush the TLBs of all VCPUs if required. */
4880	if ( fFlushRequired
4881	&& fFlush)
4882	{
4883	PGM_INVL_ALL_VCPU_TLBS(pVM);
4884	}
4885
4886	pgmUnlock(pVM);
4887	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
4888	return rc;
4889	}
4890
4891
4892	/**
4893	* Frees a usage of a pool page.
4894	*
4895	* The caller is responsible to updating the user table so that it no longer
4896	* references the shadow page.
4897	*
4898	* @param pPool The pool.
4899	* @param pPage The shadow page.
4900	* @param iUser The shadow page pool index of the user table.
4901	* NIL_PGMPOOL_IDX for root pages.
4902	* @param iUserTable The index into the user table (shadowed). Ignored if
4903	* root page.
4904	*/
4905	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
4906	{
4907	PVMCC pVM = pPool->CTX_SUFF(pVM);
4908
4909	STAM_PROFILE_START(&pPool->StatFree, a);
4910	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
4911	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
4912	AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
4913
4914	pgmLock(pVM);
4915	if (iUser != NIL_PGMPOOL_IDX)
4916	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
4917	if (!pPage->fCached)
4918	pgmPoolFlushPage(pPool, pPage);
4919	pgmUnlock(pVM);
4920	STAM_PROFILE_STOP(&pPool->StatFree, a);
4921	}
4922
4923
4924	/**
4925	* Makes one or more free page free.
4926	*
4927	* @returns VBox status code.
4928	* @retval VINF_SUCCESS on success.
4929	*
4930	* @param pPool The pool.
4931	* @param enmKind Page table kind
4932	* @param iUser The user of the page.
4933	*/
4934	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
4935	{
4936	PVMCC pVM = pPool->CTX_SUFF(pVM);
4937	LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
4938	NOREF(enmKind);
4939
4940	/*
4941	* If the pool isn't full grown yet, expand it.
4942	*/
4943	if (pPool->cCurPages < pPool->cMaxPages)
4944	{
4945	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
4946	#ifdef IN_RING3
4947	int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM));
4948	#else
4949	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
4950	#endif
4951	if (RT_FAILURE(rc))
4952	return rc;
4953	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
4954	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
4955	return VINF_SUCCESS;
4956	}
4957
4958	/*
4959	* Free one cached page.
4960	*/
4961	return pgmPoolCacheFreeOne(pPool, iUser);
4962	}
4963
4964
4965	/**
4966	* Allocates a page from the pool.
4967	*
4968	* This page may actually be a cached page and not in need of any processing
4969	* on the callers part.
4970	*
4971	* @returns VBox status code.
4972	* @retval VINF_SUCCESS if a NEW page was allocated.
4973	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4974	*
4975	* @param pVM The cross context VM structure.
4976	* @param GCPhys The GC physical address of the page we're gonna shadow.
4977	* For 4MB and 2MB PD entries, it's the first address the
4978	* shadow PT is covering.
4979	* @param enmKind The kind of mapping.
4980	* @param enmAccess Access type for the mapping (only relevant for big pages)
4981	* @param fA20Enabled Whether the A20 gate is enabled or not.
4982	* @param iUser The shadow page pool index of the user table. Root
4983	* pages should pass NIL_PGMPOOL_IDX.
4984	* @param iUserTable The index into the user table (shadowed). Ignored for
4985	* root pages (iUser == NIL_PGMPOOL_IDX).
4986	* @param fLockPage Lock the page
4987	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4988	*/
4989	int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
4990	uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
4991	{
4992	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4993	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4994	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
4995	*ppPage = NULL;
4996	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4997	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
4998	* Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
4999
5000	pgmLock(pVM);
5001
5002	if (pPool->fCacheEnabled)
5003	{
5004	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
5005	if (RT_SUCCESS(rc2))
5006	{
5007	if (fLockPage)
5008	pgmPoolLockPage(pPool, *ppPage);
5009	pgmUnlock(pVM);
5010	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5011	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
5012	return rc2;
5013	}
5014	}
5015
5016	/*
5017	* Allocate a new one.
5018	*/
5019	int rc = VINF_SUCCESS;
5020	uint16_t iNew = pPool->iFreeHead;
5021	if (iNew == NIL_PGMPOOL_IDX)
5022	{
5023	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
5024	if (RT_FAILURE(rc))
5025	{
5026	pgmUnlock(pVM);
5027	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
5028	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5029	return rc;
5030	}
5031	iNew = pPool->iFreeHead;
5032	AssertReleaseMsgReturn(iNew != NIL_PGMPOOL_IDX, ("iNew=%#x\n", iNew), VERR_PGM_POOL_IPE);
5033	}
5034
5035	/* unlink the free head */
5036	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
5037	pPool->iFreeHead = pPage->iNext;
5038	pPage->iNext = NIL_PGMPOOL_IDX;
5039
5040	/*
5041	* Initialize it.
5042	*/
5043	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
5044	pPage->enmKind = enmKind;
5045	pPage->enmAccess = enmAccess;
5046	pPage->GCPhys = GCPhys;
5047	pPage->fA20Enabled = fA20Enabled;
5048	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
5049	pPage->fMonitored = false;
5050	pPage->fCached = false;
5051	pPage->fDirty = false;
5052	pPage->fReusedFlushPending = false;
5053	pPage->cModifications = 0;
5054	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5055	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5056	pPage->cPresent = 0;
5057	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5058	pPage->idxDirtyEntry = 0;
5059	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5060	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5061	pPage->cLastAccessHandler = 0;
5062	pPage->cLocked = 0;
5063	# ifdef VBOX_STRICT
5064	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5065	# endif
5066
5067	/*
5068	* Insert into the tracking and cache. If this fails, free the page.
5069	*/
5070	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
5071	if (RT_FAILURE(rc3))
5072	{
5073	pPool->cUsedPages--;
5074	pPage->enmKind = PGMPOOLKIND_FREE;
5075	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5076	pPage->GCPhys = NIL_RTGCPHYS;
5077	pPage->iNext = pPool->iFreeHead;
5078	pPool->iFreeHead = pPage->idx;
5079	pgmUnlock(pVM);
5080	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5081	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
5082	return rc3;
5083	}
5084
5085	/*
5086	* Commit the allocation, clear the page and return.
5087	*/
5088	#ifdef VBOX_WITH_STATISTICS
5089	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
5090	pPool->cUsedPagesHigh = pPool->cUsedPages;
5091	#endif
5092
5093	if (!pPage->fZeroed)
5094	{
5095	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5096	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
5097	ASMMemZeroPage(pv);
5098	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5099	}
5100
5101	*ppPage = pPage;
5102	if (fLockPage)
5103	pgmPoolLockPage(pPool, pPage);
5104	pgmUnlock(pVM);
5105	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
5106	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
5107	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5108	return rc;
5109	}
5110
5111
5112	/**
5113	* Frees a usage of a pool page.
5114	*
5115	* @param pVM The cross context VM structure.
5116	* @param HCPhys The HC physical address of the shadow page.
5117	* @param iUser The shadow page pool index of the user table.
5118	* NIL_PGMPOOL_IDX if root page.
5119	* @param iUserTable The index into the user table (shadowed). Ignored if
5120	* root page.
5121	*/
5122	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
5123	{
5124	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
5125	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5126	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
5127	}
5128
5129
5130	/**
5131	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
5132	*
5133	* @returns Pointer to the shadow page structure.
5134	* @param pPool The pool.
5135	* @param HCPhys The HC physical address of the shadow page.
5136	*/
5137	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
5138	{
5139	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5140
5141	/*
5142	* Look up the page.
5143	*/
5144	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5145
5146	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
5147	return pPage;
5148	}
5149
5150
5151	/**
5152	* Internal worker for finding a page for debugging purposes, no assertions.
5153	*
5154	* @returns Pointer to the shadow page structure. NULL on if not found.
5155	* @param pPool The pool.
5156	* @param HCPhys The HC physical address of the shadow page.
5157	*/
5158	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
5159	{
5160	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5161	return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5162	}
5163
5164
5165	/**
5166	* Internal worker for PGM_HCPHYS_2_PTR.
5167	*
5168	* @returns VBox status code.
5169	* @param pVM The cross context VM structure.
5170	* @param HCPhys The HC physical address of the shadow page.
5171	* @param ppv Where to return the address.
5172	*/
5173	int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv)
5174	{
5175	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5176	AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE,
5177	("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0),
5178	VERR_PGM_POOL_GET_PAGE_FAILED);
5179	ppv = (uint8_t )pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK);
5180	return VINF_SUCCESS;
5181	}
5182
5183	#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
5184
5185	/**
5186	* Flush the specified page if present
5187	*
5188	* @param pVM The cross context VM structure.
5189	* @param GCPhys Guest physical address of the page to flush
5190	*/
5191	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
5192	{
5193	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5194
5195	VM_ASSERT_EMT(pVM);
5196
5197	/*
5198	* Look up the GCPhys in the hash.
5199	*/
5200	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
5201	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
5202	if (i == NIL_PGMPOOL_IDX)
5203	return;
5204
5205	do
5206	{
5207	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5208	if (pPage->GCPhys - GCPhys < PAGE_SIZE)
5209	{
5210	switch (pPage->enmKind)
5211	{
5212	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5213	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5214	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5215	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5216	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5217	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5218	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5219	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5220	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5221	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5222	case PGMPOOLKIND_64BIT_PML4:
5223	case PGMPOOLKIND_32BIT_PD:
5224	case PGMPOOLKIND_PAE_PDPT:
5225	{
5226	Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
5227	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5228	if (pPage->fDirty)
5229	STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
5230	else
5231	# endif
5232	STAM_COUNTER_INC(&pPool->StatForceFlushPage);
5233	Assert(!pgmPoolIsPageLocked(pPage));
5234	pgmPoolMonitorChainFlush(pPool, pPage);
5235	return;
5236	}
5237
5238	/* ignore, no monitoring. */
5239	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5240	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5241	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5242	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5243	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5244	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5245	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5246	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5247	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5248	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5249	case PGMPOOLKIND_ROOT_NESTED:
5250	case PGMPOOLKIND_PAE_PD_PHYS:
5251	case PGMPOOLKIND_PAE_PDPT_PHYS:
5252	case PGMPOOLKIND_32BIT_PD_PHYS:
5253	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5254	break;
5255
5256	default:
5257	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
5258	}
5259	}
5260
5261	/* next */
5262	i = pPage->iNext;
5263	} while (i != NIL_PGMPOOL_IDX);
5264	return;
5265	}
5266
5267
5268	/**
5269	* Reset CPU on hot plugging.
5270	*
5271	* @param pVM The cross context VM structure.
5272	* @param pVCpu The cross context virtual CPU structure.
5273	*/
5274	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
5275	{
5276	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5277
5278	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5279	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5280	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5281	}
5282
5283
5284	/**
5285	* Flushes the entire cache.
5286	*
5287	* It will assert a global CR3 flush (FF) and assumes the caller is aware of
5288	* this and execute this CR3 flush.
5289	*
5290	* @param pVM The cross context VM structure.
5291	*/
5292	void pgmR3PoolReset(PVM pVM)
5293	{
5294	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5295
5296	PGM_LOCK_ASSERT_OWNER(pVM);
5297	STAM_PROFILE_START(&pPool->StatR3Reset, a);
5298	LogFlow(("pgmR3PoolReset:\n"));
5299
5300	/*
5301	* If there are no pages in the pool, there is nothing to do.
5302	*/
5303	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
5304	{
5305	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5306	return;
5307	}
5308
5309	/*
5310	* Exit the shadow mode since we're going to clear everything,
5311	* including the root page.
5312	*/
5313	VMCC_FOR_EACH_VMCPU(pVM)
5314	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5315	VMCC_FOR_EACH_VMCPU_END(pVM);
5316
5317
5318	/*
5319	* Nuke the free list and reinsert all pages into it.
5320	*/
5321	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
5322	{
5323	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5324
5325	if (pPage->fMonitored)
5326	pgmPoolMonitorFlush(pPool, pPage);
5327	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5328	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5329	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
5330	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
5331	pPage->GCPhys = NIL_RTGCPHYS;
5332	pPage->enmKind = PGMPOOLKIND_FREE;
5333	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5334	Assert(pPage->idx == i);
5335	pPage->iNext = i + 1;
5336	pPage->fA20Enabled = true;
5337	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
5338	pPage->fSeenNonGlobal = false;
5339	pPage->fMonitored = false;
5340	pPage->fDirty = false;
5341	pPage->fCached = false;
5342	pPage->fReusedFlushPending = false;
5343	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
5344	pPage->cPresent = 0;
5345	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5346	pPage->cModifications = 0;
5347	pPage->iAgeNext = NIL_PGMPOOL_IDX;
5348	pPage->iAgePrev = NIL_PGMPOOL_IDX;
5349	pPage->idxDirtyEntry = 0;
5350	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5351	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5352	pPage->cLastAccessHandler = 0;
5353	pPage->cLocked = 0;
5354	# ifdef VBOX_STRICT
5355	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5356	# endif
5357	}
5358	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
5359	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
5360	pPool->cUsedPages = 0;
5361
5362	/*
5363	* Zap and reinitialize the user records.
5364	*/
5365	pPool->cPresent = 0;
5366	pPool->iUserFreeHead = 0;
5367	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
5368	const unsigned cMaxUsers = pPool->cMaxUsers;
5369	for (unsigned i = 0; i < cMaxUsers; i++)
5370	{
5371	paUsers[i].iNext = i + 1;
5372	paUsers[i].iUser = NIL_PGMPOOL_IDX;
5373	paUsers[i].iUserTable = 0xfffffffe;
5374	}
5375	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
5376
5377	/*
5378	* Clear all the GCPhys links and rebuild the phys ext free list.
5379	*/
5380	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
5381	pRam;
5382	pRam = pRam->CTX_SUFF(pNext))
5383	{
5384	unsigned iPage = pRam->cb >> PAGE_SHIFT;
5385	while (iPage-- > 0)
5386	PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
5387	}
5388
5389	pPool->iPhysExtFreeHead = 0;
5390	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
5391	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
5392	for (unsigned i = 0; i < cMaxPhysExts; i++)
5393	{
5394	paPhysExts[i].iNext = i + 1;
5395	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
5396	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5397	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
5398	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5399	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
5400	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5401	}
5402	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
5403
5404	/*
5405	* Just zap the modified list.
5406	*/
5407	pPool->cModifiedPages = 0;
5408	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
5409
5410	/*
5411	* Clear the GCPhys hash and the age list.
5412	*/
5413	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
5414	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
5415	pPool->iAgeHead = NIL_PGMPOOL_IDX;
5416	pPool->iAgeTail = NIL_PGMPOOL_IDX;
5417
5418	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5419	/* Clear all dirty pages. */
5420	pPool->idxFreeDirtyPage = 0;
5421	pPool->cDirtyPages = 0;
5422	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++)
5423	pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
5424	# endif
5425
5426	/*
5427	* Reinsert active pages into the hash and ensure monitoring chains are correct.
5428	*/
5429	VMCC_FOR_EACH_VMCPU(pVM)
5430	{
5431	/*
5432	* Re-enter the shadowing mode and assert Sync CR3 FF.
5433	*/
5434	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5435	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5436	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5437	}
5438	VMCC_FOR_EACH_VMCPU_END(pVM);
5439
5440	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5441	}
5442
5443	#endif /* IN_RING3 */
5444
5445	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
5446	/**
5447	* Stringifies a PGMPOOLKIND value.
5448	*/
5449	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
5450	{
5451	switch ((PGMPOOLKIND)enmKind)
5452	{
5453	case PGMPOOLKIND_INVALID:
5454	return "PGMPOOLKIND_INVALID";
5455	case PGMPOOLKIND_FREE:
5456	return "PGMPOOLKIND_FREE";
5457	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5458	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
5459	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5460	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
5461	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5462	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
5463	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5464	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
5465	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5466	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
5467	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5468	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
5469	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5470	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
5471	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5472	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
5473	case PGMPOOLKIND_32BIT_PD:
5474	return "PGMPOOLKIND_32BIT_PD";
5475	case PGMPOOLKIND_32BIT_PD_PHYS:
5476	return "PGMPOOLKIND_32BIT_PD_PHYS";
5477	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5478	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
5479	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5480	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
5481	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5482	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
5483	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5484	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
5485	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5486	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
5487	case PGMPOOLKIND_PAE_PD_PHYS:
5488	return "PGMPOOLKIND_PAE_PD_PHYS";
5489	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5490	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
5491	case PGMPOOLKIND_PAE_PDPT:
5492	return "PGMPOOLKIND_PAE_PDPT";
5493	case PGMPOOLKIND_PAE_PDPT_PHYS:
5494	return "PGMPOOLKIND_PAE_PDPT_PHYS";
5495	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5496	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
5497	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5498	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
5499	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5500	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
5501	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5502	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
5503	case PGMPOOLKIND_64BIT_PML4:
5504	return "PGMPOOLKIND_64BIT_PML4";
5505	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5506	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
5507	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5508	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
5509	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5510	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
5511	case PGMPOOLKIND_ROOT_NESTED:
5512	return "PGMPOOLKIND_ROOT_NESTED";
5513	}
5514	return "Unknown kind!";
5515	}
5516	#endif /* LOG_ENABLED \|\| VBOX_STRICT */
5517

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 87626

Download in other formats: