PGMAllPool.cpp@ 92561

Last change on this file since 92561 was 91907, checked in by vboxsync, 3 years ago
VMM/MM: Eliminated MMHyperCCToRC and MMHyperR0ToRC. bugref:9517
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 208.0 KB

Line
1	/* $Id: PGMAllPool.cpp 91907 2021-10-20 19:00:05Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_POOL
23	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
24	#include <VBox/vmm/pgm.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/em.h>
27	#include <VBox/vmm/cpum.h>
28	#include "PGMInternal.h"
29	#include <VBox/vmm/vmcc.h>
30	#include "PGMInline.h"
31	#include <VBox/disopcode.h>
32	#include <VBox/vmm/hm_vmx.h>
33
34	#include <VBox/log.h>
35	#include <VBox/err.h>
36	#include <iprt/asm.h>
37	#include <iprt/asm-amd64-x86.h>
38	#include <iprt/string.h>
39
40
41	/*********************************************************************************************************************************
42	* Internal Functions *
43	*********************************************************************************************************************************/
44	RT_C_DECLS_BEGIN
45	#if 0 /* unused */
46	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
47	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
48	#endif /* unused */
49	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
50	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
51	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
52	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
53	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
54	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
55	#endif
56	#if 0 /defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)/
57	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
58	#endif
59
60	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage);
61	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt);
62	void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt);
63	void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt);
64
65	RT_C_DECLS_END
66
67
68	#if 0 /* unused */
69	/**
70	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
71	*
72	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
73	* @param enmKind The page kind.
74	*/
75	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
76	{
77	switch (enmKind)
78	{
79	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
80	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
81	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
82	return true;
83	default:
84	return false;
85	}
86	}
87	#endif /* unused */
88
89
90	/**
91	* Flushes a chain of pages sharing the same access monitor.
92	*
93	* @param pPool The pool.
94	* @param pPage A page in the chain.
95	*/
96	void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
97	{
98	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
99
100	/*
101	* Find the list head.
102	*/
103	uint16_t idx = pPage->idx;
104	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
105	{
106	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
107	{
108	idx = pPage->iMonitoredPrev;
109	Assert(idx != pPage->idx);
110	pPage = &pPool->aPages[idx];
111	}
112	}
113
114	/*
115	* Iterate the list flushing each shadow page.
116	*/
117	for (;;)
118	{
119	idx = pPage->iMonitoredNext;
120	Assert(idx != pPage->idx);
121	if (pPage->idx >= PGMPOOL_IDX_FIRST)
122	{
123	int rc2 = pgmPoolFlushPage(pPool, pPage);
124	AssertRC(rc2);
125	}
126	/* next */
127	if (idx == NIL_PGMPOOL_IDX)
128	break;
129	pPage = &pPool->aPages[idx];
130	}
131	}
132
133
134	/**
135	* Wrapper for getting the current context pointer to the entry being modified.
136	*
137	* @returns VBox status code suitable for scheduling.
138	* @param pVM The cross context VM structure.
139	* @param pvDst Destination address
140	* @param pvSrc Pointer to the mapping of @a GCPhysSrc or NULL depending
141	* on the context (e.g. \#PF in R0 & RC).
142	* @param GCPhysSrc The source guest physical address.
143	* @param cb Size of data to read
144	*/
145	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVMCC pVM, void pvDst, void const pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
146	{
147	#if defined(IN_RING3)
148	NOREF(pVM); NOREF(GCPhysSrc);
149	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
150	return VINF_SUCCESS;
151	#else
152	/** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
153	NOREF(pvSrc);
154	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
155	#endif
156	}
157
158
159	/**
160	* Process shadow entries before they are changed by the guest.
161	*
162	* For PT entries we will clear them. For PD entries, we'll simply check
163	* for mapping conflicts and set the SyncCR3 FF if found.
164	*
165	* @param pVCpu The cross context virtual CPU structure.
166	* @param pPool The pool.
167	* @param pPage The head page.
168	* @param GCPhysFault The guest physical fault address.
169	* @param pvAddress Pointer to the mapping of @a GCPhysFault or NULL
170	* depending on the context (e.g. \#PF in R0 & RC).
171	* @param cbWrite Write size; might be zero if the caller knows we're not crossing entry boundaries
172	*/
173	static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
174	void const *pvAddress, unsigned cbWrite)
175	{
176	AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
177	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
178	PVMCC pVM = pPool->CTX_SUFF(pVM);
179	NOREF(pVCpu);
180
181	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
182	(RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
183
184	for (;;)
185	{
186	union
187	{
188	void *pv;
189	PX86PT pPT;
190	PPGMSHWPTPAE pPTPae;
191	PX86PD pPD;
192	PX86PDPAE pPDPae;
193	PX86PDPT pPDPT;
194	PX86PML4 pPML4;
195	} uShw;
196
197	LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
198	pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
199
200	uShw.pv = NULL;
201	switch (pPage->enmKind)
202	{
203	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
204	{
205	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
206	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
207	const unsigned iShw = off / sizeof(X86PTE);
208	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
209	X86PGUINT const uPde = uShw.pPT->a[iShw].u;
210	if (uPde & X86_PTE_P)
211	{
212	X86PTE GstPte;
213	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
214	AssertRC(rc);
215	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
216	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK, iShw);
217	ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
218	}
219	break;
220	}
221
222	/* page/2 sized */
223	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
224	{
225	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
226	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
227	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
228	{
229	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
230	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
231	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
232	{
233	X86PTE GstPte;
234	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
235	AssertRC(rc);
236
237	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
238	pgmPoolTracDerefGCPhysHint(pPool, pPage,
239	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
240	GstPte.u & X86_PTE_PG_MASK,
241	iShw);
242	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
243	}
244	}
245	break;
246	}
247
248	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
249	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
250	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
251	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
252	{
253	unsigned iGst = off / sizeof(X86PDE);
254	unsigned iShwPdpt = iGst / 256;
255	unsigned iShw = (iGst % 256) * 2;
256	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
257
258	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
259	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
260	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
261	{
262	for (unsigned i = 0; i < 2; i++)
263	{
264	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw + i].u;
265	if (uPde & X86_PDE_P)
266	{
267	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw + i, uPde));
268	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw + i);
269	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw + i].u, 0);
270	}
271
272	/* paranoia / a bit assumptive. */
273	if ( (off & 3)
274	&& (off & 3) + cbWrite > 4)
275	{
276	const unsigned iShw2 = iShw + 2 + i;
277	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
278	{
279	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
280	if (uPde2 & X86_PDE_P)
281	{
282	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
283	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
284	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
285	}
286	}
287	}
288	}
289	}
290	break;
291	}
292
293	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
294	{
295	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
296	const unsigned iShw = off / sizeof(X86PTEPAE);
297	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
298	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
299	{
300	X86PTEPAE GstPte;
301	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
302	AssertRC(rc);
303
304	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
305	pgmPoolTracDerefGCPhysHint(pPool, pPage,
306	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
307	GstPte.u & X86_PTE_PAE_PG_MASK,
308	iShw);
309	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
310	}
311
312	/* paranoia / a bit assumptive. */
313	if ( (off & 7)
314	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
315	{
316	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
317	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
318
319	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
320	{
321	X86PTEPAE GstPte;
322	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
323	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
324	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
325	AssertRC(rc);
326	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
327	pgmPoolTracDerefGCPhysHint(pPool, pPage,
328	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
329	GstPte.u & X86_PTE_PAE_PG_MASK,
330	iShw2);
331	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
332	}
333	}
334	break;
335	}
336
337	case PGMPOOLKIND_32BIT_PD:
338	{
339	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
340	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
341
342	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
343	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
344	X86PGUINT const uPde = uShw.pPD->a[iShw].u;
345	if (uPde & X86_PDE_P)
346	{
347	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
348	pgmPoolFree(pVM, uPde & X86_PDE_PG_MASK, pPage->idx, iShw);
349	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
350	}
351
352	/* paranoia / a bit assumptive. */
353	if ( (off & 3)
354	&& (off & 3) + cbWrite > sizeof(X86PTE))
355	{
356	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
357	if ( iShw2 != iShw
358	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
359	{
360	X86PGUINT const uPde2 = uShw.pPD->a[iShw2].u;
361	if (uPde2 & X86_PDE_P)
362	{
363	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
364	pgmPoolFree(pVM, uPde2 & X86_PDE_PG_MASK, pPage->idx, iShw2);
365	ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
366	}
367	}
368	}
369	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
370	if ( uShw.pPD->a[iShw].n.u1Present
371	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
372	{
373	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
374	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
375	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
376	}
377	#endif
378	break;
379	}
380
381	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
382	{
383	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
384	const unsigned iShw = off / sizeof(X86PDEPAE);
385	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
386
387	/*
388	* Causes trouble when the guest uses a PDE to refer to the whole page table level
389	* structure. (Invalidate here; faults later on when it tries to change the page
390	* table entries -> recheck; probably only applies to the RC case.)
391	*/
392	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
393	if (uPde & X86_PDE_P)
394	{
395	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
396	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
397	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
398	}
399
400	/* paranoia / a bit assumptive. */
401	if ( (off & 7)
402	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
403	{
404	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
405	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
406
407	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
408	if (uPde2 & X86_PDE_P)
409	{
410	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
411	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
412	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
413	}
414	}
415	break;
416	}
417
418	case PGMPOOLKIND_PAE_PDPT:
419	{
420	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
421	/*
422	* Hopefully this doesn't happen very often:
423	* - touching unused parts of the page
424	* - messing with the bits of pd pointers without changing the physical address
425	*/
426	/* PDPT roots are not page aligned; 32 byte only! */
427	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
428
429	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
430	const unsigned iShw = offPdpt / sizeof(X86PDPE);
431	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
432	{
433	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
434	if (uPdpe & X86_PDPE_P)
435	{
436	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
437	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
438	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
439	}
440
441	/* paranoia / a bit assumptive. */
442	if ( (offPdpt & 7)
443	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
444	{
445	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
446	if ( iShw2 != iShw
447	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
448	{
449	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
450	if (uPdpe2 & X86_PDPE_P)
451	{
452	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
453	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
454	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
455	}
456	}
457	}
458	}
459	break;
460	}
461
462	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
463	{
464	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
465	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
466	const unsigned iShw = off / sizeof(X86PDEPAE);
467	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
468	if (uPde & X86_PDE_P)
469	{
470	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
471	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
472	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
473	}
474
475	/* paranoia / a bit assumptive. */
476	if ( (off & 7)
477	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
478	{
479	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
480	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
481	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
482	if (uPde2 & X86_PDE_P)
483	{
484	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
485	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
486	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
487	}
488	}
489	break;
490	}
491
492	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
493	{
494	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
495	/*
496	* Hopefully this doesn't happen very often:
497	* - messing with the bits of pd pointers without changing the physical address
498	*/
499	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
500	const unsigned iShw = off / sizeof(X86PDPE);
501	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
502	if (uPdpe & X86_PDPE_P)
503	{
504	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uPdpe));
505	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
506	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
507	}
508	/* paranoia / a bit assumptive. */
509	if ( (off & 7)
510	&& (off & 7) + cbWrite > sizeof(X86PDPE))
511	{
512	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
513	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
514	if (uPdpe2 & X86_PDPE_P)
515	{
516	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpe2));
517	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
518	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
519	}
520	}
521	break;
522	}
523
524	case PGMPOOLKIND_64BIT_PML4:
525	{
526	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
527	/*
528	* Hopefully this doesn't happen very often:
529	* - messing with the bits of pd pointers without changing the physical address
530	*/
531	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
532	const unsigned iShw = off / sizeof(X86PDPE);
533	X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
534	if (uPml4e & X86_PML4E_P)
535	{
536	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uPml4e));
537	pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
538	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
539	}
540	/* paranoia / a bit assumptive. */
541	if ( (off & 7)
542	&& (off & 7) + cbWrite > sizeof(X86PDPE))
543	{
544	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
545	X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
546	if (uPml4e2 & X86_PML4E_P)
547	{
548	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
549	pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
550	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
551	}
552	}
553	break;
554	}
555
556	default:
557	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
558	}
559	PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
560
561	/* next */
562	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
563	return;
564	pPage = &pPool->aPages[pPage->iMonitoredNext];
565	}
566	}
567
568	#ifndef IN_RING3
569
570	/**
571	* Checks if a access could be a fork operation in progress.
572	*
573	* Meaning, that the guest is setting up the parent process for Copy-On-Write.
574	*
575	* @returns true if it's likely that we're forking, otherwise false.
576	* @param pPool The pool.
577	* @param pDis The disassembled instruction.
578	* @param offFault The access offset.
579	*/
580	DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
581	{
582	/*
583	* i386 linux is using btr to clear X86_PTE_RW.
584	* The functions involved are (2.6.16 source inspection):
585	* clear_bit
586	* ptep_set_wrprotect
587	* copy_one_pte
588	* copy_pte_range
589	* copy_pmd_range
590	* copy_pud_range
591	* copy_page_range
592	* dup_mmap
593	* dup_mm
594	* copy_mm
595	* copy_process
596	* do_fork
597	*/
598	if ( pDis->pCurInstr->uOpcode == OP_BTR
599	&& !(offFault & 4)
600	/** @todo Validate that the bit index is X86_PTE_RW. */
601	)
602	{
603	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
604	return true;
605	}
606	return false;
607	}
608
609
610	/**
611	* Determine whether the page is likely to have been reused.
612	*
613	* @returns true if we consider the page as being reused for a different purpose.
614	* @returns false if we consider it to still be a paging page.
615	* @param pVM The cross context VM structure.
616	* @param pVCpu The cross context virtual CPU structure.
617	* @param pRegFrame Trap register frame.
618	* @param pDis The disassembly info for the faulting instruction.
619	* @param pvFault The fault address.
620	* @param pPage The pool page being accessed.
621	*
622	* @remark The REP prefix check is left to the caller because of STOSD/W.
623	*/
624	DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault,
625	PPGMPOOLPAGE pPage)
626	{
627	/* Locked (CR3, PDPTR*4) should not be reusable. Considering them as
628	such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
629	if (pPage->cLocked)
630	{
631	Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
632	return false;
633	}
634
635	/** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
636	if ( HMHasPendingIrq(pVM)
637	&& pRegFrame->rsp - pvFault < 32)
638	{
639	/* Fault caused by stack writes while trying to inject an interrupt event. */
640	Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
641	return true;
642	}
643
644	LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse, pDis->Param1.Base.idxGenReg));
645
646	/* Non-supervisor mode write means it's used for something else. */
647	if (CPUMGetGuestCPL(pVCpu) == 3)
648	return true;
649
650	switch (pDis->pCurInstr->uOpcode)
651	{
652	/* call implies the actual push of the return address faulted */
653	case OP_CALL:
654	Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
655	return true;
656	case OP_PUSH:
657	Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
658	return true;
659	case OP_PUSHF:
660	Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
661	return true;
662	case OP_PUSHA:
663	Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
664	return true;
665	case OP_FXSAVE:
666	Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
667	return true;
668	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
669	Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
670	return true;
671	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
672	Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
673	return true;
674	case OP_MOVSWD:
675	case OP_STOSWD:
676	if ( pDis->fPrefix == (DISPREFIX_REP\|DISPREFIX_REX)
677	&& pRegFrame->rcx >= 0x40
678	)
679	{
680	Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
681
682	Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
683	return true;
684	}
685	break;
686
687	default:
688	/*
689	* Anything having ESP on the left side means stack writes.
690	*/
691	if ( ( (pDis->Param1.fUse & DISUSE_REG_GEN32)
692	\|\| (pDis->Param1.fUse & DISUSE_REG_GEN64))
693	&& (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
694	{
695	Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
696	return true;
697	}
698	break;
699	}
700
701	/*
702	* Page table updates are very very unlikely to be crossing page boundraries,
703	* and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
704	*/
705	uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
706	if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
707	{
708	Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
709	return true;
710	}
711
712	/*
713	* Nobody does an unaligned 8 byte write to a page table, right.
714	*/
715	if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
716	{
717	Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
718	return true;
719	}
720
721	return false;
722	}
723
724
725	/**
726	* Flushes the page being accessed.
727	*
728	* @returns VBox status code suitable for scheduling.
729	* @param pVM The cross context VM structure.
730	* @param pVCpu The cross context virtual CPU structure.
731	* @param pPool The pool.
732	* @param pPage The pool page (head).
733	* @param pDis The disassembly of the write instruction.
734	* @param pRegFrame The trap register frame.
735	* @param GCPhysFault The fault address as guest physical address.
736	* @param pvFault The fault address.
737	* @todo VBOXSTRICTRC
738	*/
739	static int pgmRZPoolAccessPfHandlerFlush(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
740	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
741	{
742	NOREF(pVM); NOREF(GCPhysFault);
743
744	/*
745	* First, do the flushing.
746	*/
747	pgmPoolMonitorChainFlush(pPool, pPage);
748
749	/*
750	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
751	* Must do this in raw mode (!); XP boot will fail otherwise.
752	*/
753	int rc = VINF_SUCCESS;
754	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
755	if (rc2 == VINF_SUCCESS)
756	{ /* do nothing */ }
757	else if (rc2 == VINF_EM_RESCHEDULE)
758	{
759	rc = VBOXSTRICTRC_VAL(rc2);
760	# ifndef IN_RING3
761	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
762	# endif
763	}
764	else if (rc2 == VERR_EM_INTERPRETER)
765	{
766	rc = VINF_EM_RAW_EMULATE_INSTR;
767	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
768	}
769	else if (RT_FAILURE_NP(rc2))
770	rc = VBOXSTRICTRC_VAL(rc2);
771	else
772	AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
773
774	LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
775	return rc;
776	}
777
778
779	/**
780	* Handles the STOSD write accesses.
781	*
782	* @returns VBox status code suitable for scheduling.
783	* @param pVM The cross context VM structure.
784	* @param pPool The pool.
785	* @param pPage The pool page (head).
786	* @param pDis The disassembly of the write instruction.
787	* @param pRegFrame The trap register frame.
788	* @param GCPhysFault The fault address as guest physical address.
789	* @param pvFault The fault address.
790	*/
791	DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
792	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
793	{
794	unsigned uIncrement = pDis->Param1.cb;
795	NOREF(pVM);
796
797	Assert(pDis->uCpuMode == DISCPUMODE_32BIT \|\| pDis->uCpuMode == DISCPUMODE_64BIT);
798	Assert(pRegFrame->rcx <= 0x20);
799
800	# ifdef VBOX_STRICT
801	if (pDis->uOpMode == DISCPUMODE_32BIT)
802	Assert(uIncrement == 4);
803	else
804	Assert(uIncrement == 8);
805	# endif
806
807	Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
808
809	/*
810	* Increment the modification counter and insert it into the list
811	* of modified pages the first time.
812	*/
813	if (!pPage->cModifications++)
814	pgmPoolMonitorModifiedInsert(pPool, pPage);
815
816	/*
817	* Execute REP STOSD.
818	*
819	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
820	* write situation, meaning that it's safe to write here.
821	*/
822	PVMCPUCC pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
823	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
824	while (pRegFrame->rcx)
825	{
826	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
827	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
828	pu32 += uIncrement;
829	GCPhysFault += uIncrement;
830	pRegFrame->rdi += uIncrement;
831	pRegFrame->rcx--;
832	}
833	pRegFrame->rip += pDis->cbInstr;
834
835	LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
836	return VINF_SUCCESS;
837	}
838
839
840	/**
841	* Handles the simple write accesses.
842	*
843	* @returns VBox status code suitable for scheduling.
844	* @param pVM The cross context VM structure.
845	* @param pVCpu The cross context virtual CPU structure.
846	* @param pPool The pool.
847	* @param pPage The pool page (head).
848	* @param pDis The disassembly of the write instruction.
849	* @param pRegFrame The trap register frame.
850	* @param GCPhysFault The fault address as guest physical address.
851	* @param pvFault The fault address.
852	* @param pfReused Reused state (in/out)
853	*/
854	DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
855	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
856	{
857	Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
858	NOREF(pVM);
859	NOREF(pfReused); /* initialized by caller */
860
861	/*
862	* Increment the modification counter and insert it into the list
863	* of modified pages the first time.
864	*/
865	if (!pPage->cModifications++)
866	pgmPoolMonitorModifiedInsert(pPool, pPage);
867
868	/*
869	* Clear all the pages. ASSUMES that pvFault is readable.
870	*/
871	uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
872	if (cbWrite <= 8)
873	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
874	else if (cbWrite <= 16)
875	{
876	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
877	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
878	}
879	else
880	{
881	Assert(cbWrite <= 32);
882	for (uint32_t off = 0; off < cbWrite; off += 8)
883	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
884	}
885
886	/*
887	* Interpret the instruction.
888	*/
889	VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
890	if (RT_SUCCESS(rc))
891	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
892	else if (rc == VERR_EM_INTERPRETER)
893	{
894	LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
895	pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode));
896	rc = VINF_EM_RAW_EMULATE_INSTR;
897	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
898	}
899
900	# if 0 /* experimental code */
901	if (rc == VINF_SUCCESS)
902	{
903	switch (pPage->enmKind)
904	{
905	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
906	{
907	X86PTEPAE GstPte;
908	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
909	AssertRC(rc);
910
911	/* Check the new value written by the guest. If present and with a bogus physical address, then
912	* it's fairly safe to assume the guest is reusing the PT.
913	*/
914	if (GstPte.n.u1Present)
915	{
916	RTHCPHYS HCPhys = -1;
917	int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
918	if (rc != VINF_SUCCESS)
919	{
920	*pfReused = true;
921	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
922	}
923	}
924	break;
925	}
926	}
927	}
928	# endif
929
930	LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
931	return VBOXSTRICTRC_VAL(rc);
932	}
933
934
935	/**
936	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
937	* \#PF access handler callback for page table pages.}
938	*
939	* @remarks The @a pvUser argument points to the PGMPOOLPAGE.
940	*/
941	DECLEXPORT(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
942	RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
943	{
944	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
945	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
946	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
947	unsigned cMaxModifications;
948	bool fForcedFlush = false;
949	NOREF(uErrorCode);
950
951	LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
952
953	PGM_LOCK_VOID(pVM);
954	if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
955	{
956	/* Pool page changed while we were waiting for the lock; ignore. */
957	Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
958	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
959	PGM_UNLOCK(pVM);
960	return VINF_SUCCESS;
961	}
962	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
963	if (pPage->fDirty)
964	{
965	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
966	PGM_UNLOCK(pVM);
967	return VINF_SUCCESS; /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
968	}
969	# endif
970
971	# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
972	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
973	{
974	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
975	void *pvGst;
976	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
977	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
978	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
979	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
980	}
981	# endif
982
983	/*
984	* Disassemble the faulting instruction.
985	*/
986	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
987	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
988	if (RT_UNLIKELY(rc != VINF_SUCCESS))
989	{
990	AssertMsg(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
991	PGM_UNLOCK(pVM);
992	return rc;
993	}
994
995	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
996
997	/*
998	* We should ALWAYS have the list head as user parameter. This
999	* is because we use that page to record the changes.
1000	*/
1001	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1002
1003	# ifdef IN_RING0
1004	/* Maximum nr of modifications depends on the page type. */
1005	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1006	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1007	cMaxModifications = 4;
1008	else
1009	cMaxModifications = 24;
1010	# else
1011	cMaxModifications = 48;
1012	# endif
1013
1014	/*
1015	* Incremental page table updates should weigh more than random ones.
1016	* (Only applies when started from offset 0)
1017	*/
1018	pVCpu->pgm.s.cPoolAccessHandler++;
1019	if ( pPage->GCPtrLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1020	&& pPage->GCPtrLastAccessHandlerRip < pRegFrame->rip + 0x40
1021	&& pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb)
1022	&& pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
1023	{
1024	Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1025	Assert(pPage->cModifications < 32000);
1026	pPage->cModifications = pPage->cModifications * 2;
1027	pPage->GCPtrLastAccessHandlerFault = pvFault;
1028	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1029	if (pPage->cModifications >= cMaxModifications)
1030	{
1031	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
1032	fForcedFlush = true;
1033	}
1034	}
1035
1036	if (pPage->cModifications >= cMaxModifications)
1037	Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1038
1039	/*
1040	* Check if it's worth dealing with.
1041	*/
1042	bool fReused = false;
1043	bool fNotReusedNotForking = false;
1044	if ( ( pPage->cModifications < cMaxModifications /** @todo \#define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1045	\|\| pgmPoolIsPageLocked(pPage)
1046	)
1047	&& !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage))
1048	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1049	{
1050	/*
1051	* Simple instructions, no REP prefix.
1052	*/
1053	if (!(pDis->fPrefix & (DISPREFIX_REP \| DISPREFIX_REPNE)))
1054	{
1055	rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
1056	if (fReused)
1057	goto flushPage;
1058
1059	/* A mov instruction to change the first page table entry will be remembered so we can detect
1060	* full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1061	*/
1062	if ( rc == VINF_SUCCESS
1063	&& !pPage->cLocked /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
1064	&& pDis->pCurInstr->uOpcode == OP_MOV
1065	&& (pvFault & PAGE_OFFSET_MASK) == 0)
1066	{
1067	pPage->GCPtrLastAccessHandlerFault = pvFault;
1068	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1069	pPage->GCPtrLastAccessHandlerRip = pRegFrame->rip;
1070	/* Make sure we don't kick out a page too quickly. */
1071	if (pPage->cModifications > 8)
1072	pPage->cModifications = 2;
1073	}
1074	else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
1075	{
1076	/* ignore the 2nd write to this page table entry. */
1077	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1078	}
1079	else
1080	{
1081	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
1082	pPage->GCPtrLastAccessHandlerRip = 0;
1083	}
1084
1085	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1086	PGM_UNLOCK(pVM);
1087	return rc;
1088	}
1089
1090	/*
1091	* Windows is frequently doing small memset() operations (netio test 4k+).
1092	* We have to deal with these or we'll kill the cache and performance.
1093	*/
1094	if ( pDis->pCurInstr->uOpcode == OP_STOSWD
1095	&& !pRegFrame->eflags.Bits.u1DF
1096	&& pDis->uOpMode == pDis->uCpuMode
1097	&& pDis->uAddrMode == pDis->uCpuMode)
1098	{
1099	bool fValidStosd = false;
1100
1101	if ( pDis->uCpuMode == DISCPUMODE_32BIT
1102	&& pDis->fPrefix == DISPREFIX_REP
1103	&& pRegFrame->ecx <= 0x20
1104	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1105	&& !((uintptr_t)pvFault & 3)
1106	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1107	)
1108	{
1109	fValidStosd = true;
1110	pRegFrame->rcx &= 0xffffffff; /* paranoia */
1111	}
1112	else
1113	if ( pDis->uCpuMode == DISCPUMODE_64BIT
1114	&& pDis->fPrefix == (DISPREFIX_REP \| DISPREFIX_REX)
1115	&& pRegFrame->rcx <= 0x20
1116	&& pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1117	&& !((uintptr_t)pvFault & 7)
1118	&& (pRegFrame->rax == 0 \|\| pRegFrame->rax == 0x80) /* the two values observed. */
1119	)
1120	{
1121	fValidStosd = true;
1122	}
1123
1124	if (fValidStosd)
1125	{
1126	rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1127	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
1128	PGM_UNLOCK(pVM);
1129	return rc;
1130	}
1131	}
1132
1133	/* REP prefix, don't bother. */
1134	STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
1135	Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1136	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
1137	fNotReusedNotForking = true;
1138	}
1139
1140	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
1141	/* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1142	* leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1143	*/
1144	if ( pPage->cModifications >= cMaxModifications
1145	&& !fForcedFlush
1146	&& (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT \|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1147	&& ( fNotReusedNotForking
1148	\|\| ( !pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage)
1149	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1150	)
1151	)
1152	{
1153	Assert(!pgmPoolIsPageLocked(pPage));
1154	Assert(pPage->fDirty == false);
1155
1156	/* Flush any monitored duplicates as we will disable write protection. */
1157	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1158	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1159	{
1160	PPGMPOOLPAGE pPageHead = pPage;
1161
1162	/* Find the monitor head. */
1163	while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1164	pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1165
1166	while (pPageHead)
1167	{
1168	unsigned idxNext = pPageHead->iMonitoredNext;
1169
1170	if (pPageHead != pPage)
1171	{
1172	STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1173	Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1174	int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1175	AssertRC(rc2);
1176	}
1177
1178	if (idxNext == NIL_PGMPOOL_IDX)
1179	break;
1180
1181	pPageHead = &pPool->aPages[idxNext];
1182	}
1183	}
1184
1185	/* The flushing above might fail for locked pages, so double check. */
1186	if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX
1187	&& pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1188	{
1189	pgmPoolAddDirtyPage(pVM, pPool, pPage);
1190
1191	/* Temporarily allow write access to the page table again. */
1192	rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK, pPage->GCPhys & PAGE_BASE_GC_MASK);
1193	if (rc == VINF_SUCCESS)
1194	{
1195	rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
1196	AssertMsg(rc == VINF_SUCCESS
1197	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1198	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1199	\|\| rc == VERR_PAGE_NOT_PRESENT,
1200	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1201	# ifdef VBOX_STRICT
1202	pPage->GCPtrDirtyFault = pvFault;
1203	# endif
1204
1205	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
1206	PGM_UNLOCK(pVM);
1207	return rc;
1208	}
1209	}
1210	}
1211	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT && IN_RING0 */
1212
1213	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
1214	flushPage:
1215	/*
1216	* Not worth it, so flush it.
1217	*
1218	* If we considered it to be reused, don't go back to ring-3
1219	* to emulate failed instructions since we usually cannot
1220	* interpret then. This may be a bit risky, in which case
1221	* the reuse detection must be fixed.
1222	*/
1223	rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1224	if ( rc == VINF_EM_RAW_EMULATE_INSTR
1225	&& fReused)
1226	{
1227	/* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
1228	if (PGMShwGetPage(pVCpu, pRegFrame->rip, NULL, NULL) == VINF_SUCCESS)
1229	rc = VINF_SUCCESS; /* safe to restart the instruction. */
1230	}
1231	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
1232	PGM_UNLOCK(pVM);
1233	return rc;
1234	}
1235
1236	#endif /* !IN_RING3 */
1237
1238	/**
1239	* @callback_method_impl{FNPGMPHYSHANDLER,
1240	* Access handler for shadowed page table pages.}
1241	*
1242	* @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
1243	*/
1244	PGM_ALL_CB2_DECL(VBOXSTRICTRC)
1245	pgmPoolAccessHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
1246	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
1247	{
1248	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1249	STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
1250	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1251	LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
1252	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
1253
1254	NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
1255
1256	PGM_LOCK_VOID(pVM);
1257
1258	#ifdef VBOX_WITH_STATISTICS
1259	/*
1260	* Collect stats on the access.
1261	*/
1262	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
1263	if (cbBuf <= 16 && cbBuf > 0)
1264	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
1265	else if (cbBuf >= 17 && cbBuf < 32)
1266	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
1267	else if (cbBuf >= 32 && cbBuf < 64)
1268	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
1269	else if (cbBuf >= 64)
1270	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
1271
1272	uint8_t cbAlign;
1273	switch (pPage->enmKind)
1274	{
1275	default:
1276	cbAlign = 7;
1277	break;
1278	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1279	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1280	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1281	case PGMPOOLKIND_32BIT_PD:
1282	case PGMPOOLKIND_32BIT_PD_PHYS:
1283	cbAlign = 3;
1284	break;
1285	}
1286	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
1287	if ((uint8_t)GCPhys & cbAlign)
1288	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
1289	#endif
1290
1291	/*
1292	* Make sure the pool page wasn't modified by a different CPU.
1293	*/
1294	if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
1295	{
1296	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1297
1298	/* The max modification count before flushing depends on the context and page type. */
1299	#ifdef IN_RING3
1300	uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
1301	#else
1302	uint16_t cMaxModifications;
1303	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1304	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1305	cMaxModifications = 4;
1306	else
1307	cMaxModifications = 24;
1308	#endif
1309
1310	/*
1311	* We don't have to be very sophisticated about this since there are relativly few calls here.
1312	* However, we must try our best to detect any non-cpu accesses (disk / networking).
1313	*/
1314	if ( ( pPage->cModifications < cMaxModifications
1315	\|\| pgmPoolIsPageLocked(pPage) )
1316	&& enmOrigin != PGMACCESSORIGIN_DEVICE
1317	&& cbBuf <= 16)
1318	{
1319	/* Clear the shadow entry. */
1320	if (!pPage->cModifications++)
1321	pgmPoolMonitorModifiedInsert(pPool, pPage);
1322
1323	if (cbBuf <= 8)
1324	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
1325	else
1326	{
1327	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
1328	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
1329	}
1330	}
1331	else
1332	pgmPoolMonitorChainFlush(pPool, pPage);
1333
1334	STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1335	}
1336	else
1337	Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1338	PGM_UNLOCK(pVM);
1339	return VINF_PGM_HANDLER_DO_DEFAULT;
1340	}
1341
1342
1343	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1344
1345	# if defined(VBOX_STRICT) && !defined(IN_RING3)
1346
1347	/**
1348	* Check references to guest physical memory in a PAE / PAE page table.
1349	*
1350	* @param pPool The pool.
1351	* @param pPage The page.
1352	* @param pShwPT The shadow page table (mapping of the page).
1353	* @param pGstPT The guest page table.
1354	*/
1355	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
1356	{
1357	unsigned cErrors = 0;
1358	int LastRc = -1; /* initialized to shut up gcc */
1359	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1360	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1361	PVMCC pVM = pPool->CTX_SUFF(pVM);
1362
1363	# ifdef VBOX_STRICT
1364	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1365	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1366	# endif
1367	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1368	{
1369	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1370	{
1371	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1372	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1373	if ( rc != VINF_SUCCESS
1374	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1375	{
1376	Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1377	LastPTE = i;
1378	LastRc = rc;
1379	LastHCPhys = HCPhys;
1380	cErrors++;
1381
1382	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1383	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1384	AssertRC(rc);
1385
1386	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1387	{
1388	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1389
1390	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1391	{
1392	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1393
1394	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1395	{
1396	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1397	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1398	{
1399	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1400	}
1401	}
1402
1403	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1404	}
1405	}
1406	}
1407	}
1408	}
1409	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1410	}
1411
1412
1413	/**
1414	* Check references to guest physical memory in a PAE / 32-bit page table.
1415	*
1416	* @param pPool The pool.
1417	* @param pPage The page.
1418	* @param pShwPT The shadow page table (mapping of the page).
1419	* @param pGstPT The guest page table.
1420	*/
1421	static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
1422	{
1423	unsigned cErrors = 0;
1424	int LastRc = -1; /* initialized to shut up gcc */
1425	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1426	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1427	PVMCC pVM = pPool->CTX_SUFF(pVM);
1428
1429	# ifdef VBOX_STRICT
1430	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1431	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1432	# endif
1433	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1434	{
1435	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1436	{
1437	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1438	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1439	if ( rc != VINF_SUCCESS
1440	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1441	{
1442	Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1443	LastPTE = i;
1444	LastRc = rc;
1445	LastHCPhys = HCPhys;
1446	cErrors++;
1447
1448	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1449	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1450	AssertRC(rc);
1451
1452	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1453	{
1454	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1455
1456	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1457	{
1458	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1459
1460	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1461	{
1462	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1463	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1464	{
1465	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1466	}
1467	}
1468
1469	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1470	}
1471	}
1472	}
1473	}
1474	}
1475	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1476	}
1477
1478	# endif /* VBOX_STRICT && !IN_RING3 */
1479
1480	/**
1481	* Clear references to guest physical memory in a PAE / PAE page table.
1482	*
1483	* @returns nr of changed PTEs
1484	* @param pPool The pool.
1485	* @param pPage The page.
1486	* @param pShwPT The shadow page table (mapping of the page).
1487	* @param pGstPT The guest page table.
1488	* @param pOldGstPT The old cached guest page table.
1489	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1490	* @param pfFlush Flush reused page table (out)
1491	*/
1492	DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
1493	PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1494	{
1495	unsigned cChanged = 0;
1496
1497	# ifdef VBOX_STRICT
1498	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1499	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1500	# endif
1501	*pfFlush = false;
1502
1503	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1504	{
1505	/* Check the new value written by the guest. If present and with a bogus physical address, then
1506	* it's fairly safe to assume the guest is reusing the PT.
1507	*/
1508	if ( fAllowRemoval
1509	&& (pGstPT->a[i].u & X86_PTE_P))
1510	{
1511	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1512	{
1513	*pfFlush = true;
1514	return ++cChanged;
1515	}
1516	}
1517	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1518	{
1519	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1520	if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1521	{
1522	# ifdef VBOX_STRICT
1523	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1524	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1525	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1526	# endif
1527	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1528	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1529	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1530	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1531
1532	if ( uHostAttr == uGuestAttr
1533	&& fHostRW <= fGuestRW)
1534	continue;
1535	}
1536	cChanged++;
1537	/* Something was changed, so flush it. */
1538	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
1539	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1540	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
1541	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1542	}
1543	}
1544	return cChanged;
1545	}
1546
1547
1548	/**
1549	* Clear references to guest physical memory in a PAE / PAE page table.
1550	*
1551	* @returns nr of changed PTEs
1552	* @param pPool The pool.
1553	* @param pPage The page.
1554	* @param pShwPT The shadow page table (mapping of the page).
1555	* @param pGstPT The guest page table.
1556	* @param pOldGstPT The old cached guest page table.
1557	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1558	* @param pfFlush Flush reused page table (out)
1559	*/
1560	DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
1561	PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1562	{
1563	unsigned cChanged = 0;
1564
1565	# ifdef VBOX_STRICT
1566	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1567	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1568	# endif
1569	*pfFlush = false;
1570
1571	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1572	{
1573	/* Check the new value written by the guest. If present and with a bogus physical address, then
1574	* it's fairly safe to assume the guest is reusing the PT. */
1575	if (fAllowRemoval)
1576	{
1577	X86PGUINT const uPte = pGstPT->a[i].u;
1578	if ( (uPte & X86_PTE_P)
1579	&& !PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), uPte & X86_PTE_PG_MASK))
1580	{
1581	*pfFlush = true;
1582	return ++cChanged;
1583	}
1584	}
1585	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1586	{
1587	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1588	if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
1589	{
1590	# ifdef VBOX_STRICT
1591	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1592	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1593	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1594	# endif
1595	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1596	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1597	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1598	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1599
1600	if ( uHostAttr == uGuestAttr
1601	&& fHostRW <= fGuestRW)
1602	continue;
1603	}
1604	cChanged++;
1605	/* Something was changed, so flush it. */
1606	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
1607	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
1608	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
1609	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1610	}
1611	}
1612	return cChanged;
1613	}
1614
1615
1616	/**
1617	* Flush a dirty page
1618	*
1619	* @param pVM The cross context VM structure.
1620	* @param pPool The pool.
1621	* @param idxSlot Dirty array slot index
1622	* @param fAllowRemoval Allow a reused page table to be removed
1623	*/
1624	static void pgmPoolFlushDirtyPage(PVMCC pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
1625	{
1626	AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages));
1627
1628	Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
1629	unsigned idxPage = pPool->aidxDirtyPages[idxSlot];
1630	if (idxPage == NIL_PGMPOOL_IDX)
1631	return;
1632
1633	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1634	Assert(pPage->idx == idxPage);
1635	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1636
1637	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1638	Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1639
1640	/* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
1641	int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
1642	Assert(rc == VINF_SUCCESS);
1643	pPage->fDirty = false;
1644
1645	# ifdef VBOX_STRICT
1646	uint64_t fFlags = 0;
1647	RTHCPHYS HCPhys;
1648	rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
1649	AssertMsg( ( rc == VINF_SUCCESS
1650	&& (!(fFlags & X86_PTE_RW) \|\| HCPhys != pPage->Core.Key))
1651	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1652	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1653	\|\| rc == VERR_PAGE_NOT_PRESENT,
1654	("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
1655	# endif
1656
1657	/* Flush those PTEs that have changed. */
1658	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1659	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1660	void *pvGst;
1661	rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1662	bool fFlush;
1663	unsigned cChanges;
1664
1665	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1666	cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
1667	(PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1668	else
1669	cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
1670	(PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1671
1672	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1673	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1674	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1675	/* Note: we might want to consider keeping the dirty page active in case there were many changes. */
1676
1677	/* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1678	Assert(pPage->cModifications);
1679	if (cChanges < 4)
1680	pPage->cModifications = 1; /* must use > 0 here */
1681	else
1682	pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1683
1684	STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1685	if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
1686	pPool->idxFreeDirtyPage = idxSlot;
1687
1688	pPool->cDirtyPages--;
1689	pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1690	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1691	if (fFlush)
1692	{
1693	Assert(fAllowRemoval);
1694	Log(("Flush reused page table!\n"));
1695	pgmPoolFlushPage(pPool, pPage);
1696	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1697	}
1698	else
1699	Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
1700	}
1701
1702
1703	# ifndef IN_RING3
1704	/**
1705	* Add a new dirty page
1706	*
1707	* @param pVM The cross context VM structure.
1708	* @param pPool The pool.
1709	* @param pPage The page.
1710	*/
1711	void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1712	{
1713	PGM_LOCK_ASSERT_OWNER(pVM);
1714	AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 \|\| RT_ELEMENTS(pPool->aDirtyPages) == 16);
1715	Assert(!pPage->fDirty);
1716
1717	unsigned idxFree = pPool->idxFreeDirtyPage;
1718	Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
1719	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1720
1721	if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
1722	{
1723	STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
1724	pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
1725	}
1726	Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
1727	AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1728
1729	Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1730
1731	/*
1732	* Make a copy of the guest page table as we require valid GCPhys addresses
1733	* when removing references to physical pages.
1734	* (The HCPhys linear lookup is extremely expensive!)
1735	*/
1736	void *pvGst;
1737	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1738	memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst, (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) ? PAGE_SIZE : PAGE_SIZE/2);
1739	# ifdef VBOX_STRICT
1740	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1741	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1742	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1743	else
1744	pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
1745	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1746	# endif
1747	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1748
1749	STAM_COUNTER_INC(&pPool->StatDirtyPage);
1750	pPage->fDirty = true;
1751	pPage->idxDirtyEntry = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
1752	pPool->aidxDirtyPages[idxFree] = pPage->idx;
1753	pPool->cDirtyPages++;
1754
1755	pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1756	if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
1757	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1758	{
1759	unsigned i;
1760	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1761	{
1762	idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1763	if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1764	{
1765	pPool->idxFreeDirtyPage = idxFree;
1766	break;
1767	}
1768	}
1769	Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
1770	}
1771
1772	Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) \|\| pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1773
1774	/*
1775	* Clear all references to this shadow table. See @bugref{7298}.
1776	*/
1777	pgmPoolTrackClearPageUsers(pPool, pPage);
1778	}
1779	# endif /* !IN_RING3 */
1780
1781
1782	/**
1783	* Check if the specified page is dirty (not write monitored)
1784	*
1785	* @return dirty or not
1786	* @param pVM The cross context VM structure.
1787	* @param GCPhys Guest physical address
1788	*/
1789	bool pgmPoolIsDirtyPageSlow(PVMCC pVM, RTGCPHYS GCPhys)
1790	{
1791	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1792	PGM_LOCK_ASSERT_OWNER(pVM);
1793	if (!pPool->cDirtyPages)
1794	return false;
1795
1796	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1797
1798	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1799	{
1800	unsigned idxPage = pPool->aidxDirtyPages[i];
1801	if (idxPage != NIL_PGMPOOL_IDX)
1802	{
1803	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1804	if (pPage->GCPhys == GCPhys)
1805	return true;
1806	}
1807	}
1808	return false;
1809	}
1810
1811
1812	/**
1813	* Reset all dirty pages by reinstating page monitoring.
1814	*
1815	* @param pVM The cross context VM structure.
1816	*/
1817	void pgmPoolResetDirtyPages(PVMCC pVM)
1818	{
1819	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1820	PGM_LOCK_ASSERT_OWNER(pVM);
1821	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1822
1823	if (!pPool->cDirtyPages)
1824	return;
1825
1826	Log(("pgmPoolResetDirtyPages\n"));
1827	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1828	pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
1829
1830	pPool->idxFreeDirtyPage = 0;
1831	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
1832	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1833	{
1834	unsigned i;
1835	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1836	{
1837	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
1838	{
1839	pPool->idxFreeDirtyPage = i;
1840	break;
1841	}
1842	}
1843	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
1844	}
1845
1846	Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX \|\| pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
1847	return;
1848	}
1849
1850
1851	/**
1852	* Invalidate the PT entry for the specified page
1853	*
1854	* @param pVM The cross context VM structure.
1855	* @param GCPtrPage Guest page to invalidate
1856	*/
1857	void pgmPoolResetDirtyPage(PVMCC pVM, RTGCPTR GCPtrPage)
1858	{
1859	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1860	PGM_LOCK_ASSERT_OWNER(pVM);
1861	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1862
1863	if (!pPool->cDirtyPages)
1864	return;
1865
1866	Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
1867	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1868	{
1869	/** @todo What was intended here??? This looks incomplete... */
1870	}
1871	}
1872
1873
1874	/**
1875	* Reset all dirty pages by reinstating page monitoring.
1876	*
1877	* @param pVM The cross context VM structure.
1878	* @param GCPhysPT Physical address of the page table
1879	*/
1880	void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT)
1881	{
1882	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1883	PGM_LOCK_ASSERT_OWNER(pVM);
1884	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1885	unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
1886
1887	if (!pPool->cDirtyPages)
1888	return;
1889
1890	GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1891
1892	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1893	{
1894	unsigned idxPage = pPool->aidxDirtyPages[i];
1895	if (idxPage != NIL_PGMPOOL_IDX)
1896	{
1897	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1898	if (pPage->GCPhys == GCPhysPT)
1899	{
1900	idxDirtyPage = i;
1901	break;
1902	}
1903	}
1904	}
1905
1906	if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
1907	{
1908	pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
1909	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
1910	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1911	{
1912	unsigned i;
1913	for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1914	{
1915	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
1916	{
1917	pPool->idxFreeDirtyPage = i;
1918	break;
1919	}
1920	}
1921	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
1922	}
1923	}
1924	}
1925
1926	#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1927
1928	/**
1929	* Inserts a page into the GCPhys hash table.
1930	*
1931	* @param pPool The pool.
1932	* @param pPage The page.
1933	*/
1934	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1935	{
1936	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1937	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1938	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1939	pPage->iNext = pPool->aiHash[iHash];
1940	pPool->aiHash[iHash] = pPage->idx;
1941	}
1942
1943
1944	/**
1945	* Removes a page from the GCPhys hash table.
1946	*
1947	* @param pPool The pool.
1948	* @param pPage The page.
1949	*/
1950	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1951	{
1952	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1953	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1954	if (pPool->aiHash[iHash] == pPage->idx)
1955	pPool->aiHash[iHash] = pPage->iNext;
1956	else
1957	{
1958	uint16_t iPrev = pPool->aiHash[iHash];
1959	for (;;)
1960	{
1961	const int16_t i = pPool->aPages[iPrev].iNext;
1962	if (i == pPage->idx)
1963	{
1964	pPool->aPages[iPrev].iNext = pPage->iNext;
1965	break;
1966	}
1967	if (i == NIL_PGMPOOL_IDX)
1968	{
1969	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
1970	break;
1971	}
1972	iPrev = i;
1973	}
1974	}
1975	pPage->iNext = NIL_PGMPOOL_IDX;
1976	}
1977
1978
1979	/**
1980	* Frees up one cache page.
1981	*
1982	* @returns VBox status code.
1983	* @retval VINF_SUCCESS on success.
1984	* @param pPool The pool.
1985	* @param iUser The user index.
1986	*/
1987	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1988	{
1989	const PVMCC pVM = pPool->CTX_SUFF(pVM);
1990	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1991	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1992
1993	/*
1994	* Select one page from the tail of the age list.
1995	*/
1996	PPGMPOOLPAGE pPage;
1997	for (unsigned iLoop = 0; ; iLoop++)
1998	{
1999	uint16_t iToFree = pPool->iAgeTail;
2000	if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
2001	iToFree = pPool->aPages[iToFree].iAgePrev;
2002	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
2003	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
2004	{
2005	uint16_t i = pPool->aPages[iToFree].iAgePrev;
2006	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
2007	{
2008	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
2009	continue;
2010	iToFree = i;
2011	break;
2012	}
2013	}
2014	*/
2015	Assert(iToFree != iUser);
2016	AssertReleaseMsg(iToFree != NIL_PGMPOOL_IDX,
2017	("iToFree=%#x (iAgeTail=%#x) iUser=%#x iLoop=%u - pPool=%p LB %#zx\n",
2018	iToFree, pPool->iAgeTail, iUser, iLoop, pPool,
2019	RT_UOFFSETOF_DYN(PGMPOOL, aPages[pPool->cMaxPages])
2020	+ pPool->cMaxUsers * sizeof(PGMPOOLUSER)
2021	+ pPool->cMaxPhysExts * sizeof(PGMPOOLPHYSEXT) ));
2022
2023	pPage = &pPool->aPages[iToFree];
2024
2025	/*
2026	* Reject any attempts at flushing the currently active shadow CR3 mapping.
2027	* Call pgmPoolCacheUsed to move the page to the head of the age list.
2028	*/
2029	if ( !pgmPoolIsPageLocked(pPage)
2030	&& pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
2031	break;
2032	LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
2033	pgmPoolCacheUsed(pPool, pPage);
2034	AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
2035	}
2036
2037	/*
2038	* Found a usable page, flush it and return.
2039	*/
2040	int rc = pgmPoolFlushPage(pPool, pPage);
2041	/* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
2042	/** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
2043	if (rc == VINF_SUCCESS)
2044	PGM_INVL_ALL_VCPU_TLBS(pVM);
2045	return rc;
2046	}
2047
2048
2049	/**
2050	* Checks if a kind mismatch is really a page being reused
2051	* or if it's just normal remappings.
2052	*
2053	* @returns true if reused and the cached page (enmKind1) should be flushed
2054	* @returns false if not reused.
2055	* @param enmKind1 The kind of the cached page.
2056	* @param enmKind2 The kind of the requested page.
2057	*/
2058	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
2059	{
2060	switch (enmKind1)
2061	{
2062	/*
2063	* Never reuse them. There is no remapping in non-paging mode.
2064	*/
2065	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2066	case PGMPOOLKIND_32BIT_PD_PHYS:
2067	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2068	case PGMPOOLKIND_PAE_PD_PHYS:
2069	case PGMPOOLKIND_PAE_PDPT_PHYS:
2070	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2071	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2072	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2073	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2074	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2075	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
2076	return false;
2077
2078	/*
2079	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2080	*/
2081	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2082	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2083	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2084	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2085	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2086	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2087	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2088	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2089	case PGMPOOLKIND_32BIT_PD:
2090	case PGMPOOLKIND_PAE_PDPT:
2091	switch (enmKind2)
2092	{
2093	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2094	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2095	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2096	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2097	case PGMPOOLKIND_64BIT_PML4:
2098	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2099	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2100	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2101	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2102	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2103	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2104	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2105	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2106	return true;
2107	default:
2108	return false;
2109	}
2110
2111	/*
2112	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2113	*/
2114	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2115	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2116	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2117	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2118	case PGMPOOLKIND_64BIT_PML4:
2119	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2120	switch (enmKind2)
2121	{
2122	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2123	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2124	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2125	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2126	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2127	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2128	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2129	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2130	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2131	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2132	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2133	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2134	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2135	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2136	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2137	return true;
2138	default:
2139	return false;
2140	}
2141
2142	/*
2143	* These cannot be flushed, and it's common to reuse the PDs as PTs.
2144	*/
2145	case PGMPOOLKIND_ROOT_NESTED:
2146	return false;
2147
2148	default:
2149	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
2150	}
2151	}
2152
2153
2154	/**
2155	* Attempts to satisfy a pgmPoolAlloc request from the cache.
2156	*
2157	* @returns VBox status code.
2158	* @retval VINF_PGM_CACHED_PAGE on success.
2159	* @retval VERR_FILE_NOT_FOUND if not found.
2160	* @param pPool The pool.
2161	* @param GCPhys The GC physical address of the page we're gonna shadow.
2162	* @param enmKind The kind of mapping.
2163	* @param enmAccess Access type for the mapping (only relevant for big pages)
2164	* @param fA20Enabled Whether the CPU has the A20 gate enabled.
2165	* @param iUser The shadow page pool index of the user table. This is
2166	* NIL_PGMPOOL_IDX for root pages.
2167	* @param iUserTable The index into the user table (shadowed). Ignored if
2168	* root page
2169	* @param ppPage Where to store the pointer to the page.
2170	*/
2171	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
2172	uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
2173	{
2174	/*
2175	* Look up the GCPhys in the hash.
2176	*/
2177	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2178	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
2179	if (i != NIL_PGMPOOL_IDX)
2180	{
2181	do
2182	{
2183	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2184	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
2185	if (pPage->GCPhys == GCPhys)
2186	{
2187	if ( (PGMPOOLKIND)pPage->enmKind == enmKind
2188	&& (PGMPOOLACCESS)pPage->enmAccess == enmAccess
2189	&& pPage->fA20Enabled == fA20Enabled)
2190	{
2191	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
2192	* doesn't flush it in case there are no more free use records.
2193	*/
2194	pgmPoolCacheUsed(pPool, pPage);
2195
2196	int rc = VINF_SUCCESS;
2197	if (iUser != NIL_PGMPOOL_IDX)
2198	rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
2199	if (RT_SUCCESS(rc))
2200	{
2201	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
2202	*ppPage = pPage;
2203	if (pPage->cModifications)
2204	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
2205	STAM_COUNTER_INC(&pPool->StatCacheHits);
2206	return VINF_PGM_CACHED_PAGE;
2207	}
2208	return rc;
2209	}
2210
2211	if ((PGMPOOLKIND)pPage->enmKind != enmKind)
2212	{
2213	/*
2214	* The kind is different. In some cases we should now flush the page
2215	* as it has been reused, but in most cases this is normal remapping
2216	* of PDs as PT or big pages using the GCPhys field in a slightly
2217	* different way than the other kinds.
2218	*/
2219	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
2220	{
2221	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
2222	pgmPoolFlushPage(pPool, pPage);
2223	break;
2224	}
2225	}
2226	}
2227
2228	/* next */
2229	i = pPage->iNext;
2230	} while (i != NIL_PGMPOOL_IDX);
2231	}
2232
2233	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
2234	STAM_COUNTER_INC(&pPool->StatCacheMisses);
2235	return VERR_FILE_NOT_FOUND;
2236	}
2237
2238
2239	/**
2240	* Inserts a page into the cache.
2241	*
2242	* @param pPool The pool.
2243	* @param pPage The cached page.
2244	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
2245	*/
2246	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
2247	{
2248	/*
2249	* Insert into the GCPhys hash if the page is fit for that.
2250	*/
2251	Assert(!pPage->fCached);
2252	if (fCanBeCached)
2253	{
2254	pPage->fCached = true;
2255	pgmPoolHashInsert(pPool, pPage);
2256	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2257	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2258	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
2259	}
2260	else
2261	{
2262	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2263	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2264	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
2265	}
2266
2267	/*
2268	* Insert at the head of the age list.
2269	*/
2270	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2271	pPage->iAgeNext = pPool->iAgeHead;
2272	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
2273	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
2274	else
2275	pPool->iAgeTail = pPage->idx;
2276	pPool->iAgeHead = pPage->idx;
2277	}
2278
2279
2280	/**
2281	* Flushes a cached page.
2282	*
2283	* @param pPool The pool.
2284	* @param pPage The cached page.
2285	*/
2286	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2287	{
2288	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
2289
2290	/*
2291	* Remove the page from the hash.
2292	*/
2293	if (pPage->fCached)
2294	{
2295	pPage->fCached = false;
2296	pgmPoolHashRemove(pPool, pPage);
2297	}
2298	else
2299	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2300
2301	/*
2302	* Remove it from the age list.
2303	*/
2304	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
2305	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
2306	else
2307	pPool->iAgeTail = pPage->iAgePrev;
2308	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
2309	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
2310	else
2311	pPool->iAgeHead = pPage->iAgeNext;
2312	pPage->iAgeNext = NIL_PGMPOOL_IDX;
2313	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2314	}
2315
2316
2317	/**
2318	* Looks for pages sharing the monitor.
2319	*
2320	* @returns Pointer to the head page.
2321	* @returns NULL if not found.
2322	* @param pPool The Pool
2323	* @param pNewPage The page which is going to be monitored.
2324	*/
2325	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
2326	{
2327	/*
2328	* Look up the GCPhys in the hash.
2329	*/
2330	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2331	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2332	if (i == NIL_PGMPOOL_IDX)
2333	return NULL;
2334	do
2335	{
2336	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2337	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
2338	&& pPage != pNewPage)
2339	{
2340	switch (pPage->enmKind)
2341	{
2342	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2343	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2344	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2345	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2346	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2347	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2348	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2349	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2350	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2351	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2352	case PGMPOOLKIND_64BIT_PML4:
2353	case PGMPOOLKIND_32BIT_PD:
2354	case PGMPOOLKIND_PAE_PDPT:
2355	{
2356	/* find the head */
2357	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2358	{
2359	Assert(pPage->iMonitoredPrev != pPage->idx);
2360	pPage = &pPool->aPages[pPage->iMonitoredPrev];
2361	}
2362	return pPage;
2363	}
2364
2365	/* ignore, no monitoring. */
2366	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2367	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2368	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2369	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2370	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2371	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2372	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2373	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2374	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2375	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2376	case PGMPOOLKIND_ROOT_NESTED:
2377	case PGMPOOLKIND_PAE_PD_PHYS:
2378	case PGMPOOLKIND_PAE_PDPT_PHYS:
2379	case PGMPOOLKIND_32BIT_PD_PHYS:
2380	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2381	break;
2382	default:
2383	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2384	}
2385	}
2386
2387	/* next */
2388	i = pPage->iNext;
2389	} while (i != NIL_PGMPOOL_IDX);
2390	return NULL;
2391	}
2392
2393
2394	/**
2395	* Enabled write monitoring of a guest page.
2396	*
2397	* @returns VBox status code.
2398	* @retval VINF_SUCCESS on success.
2399	* @param pPool The pool.
2400	* @param pPage The cached page.
2401	*/
2402	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2403	{
2404	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
2405
2406	/*
2407	* Filter out the relevant kinds.
2408	*/
2409	switch (pPage->enmKind)
2410	{
2411	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2412	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2413	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2414	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2415	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2416	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2417	case PGMPOOLKIND_64BIT_PML4:
2418	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2419	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2420	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2421	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2422	case PGMPOOLKIND_32BIT_PD:
2423	case PGMPOOLKIND_PAE_PDPT:
2424	break;
2425
2426	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2427	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2428	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2429	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2430	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2431	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2432	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2433	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2434	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2435	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2436	case PGMPOOLKIND_ROOT_NESTED:
2437	/* Nothing to monitor here. */
2438	return VINF_SUCCESS;
2439
2440	case PGMPOOLKIND_32BIT_PD_PHYS:
2441	case PGMPOOLKIND_PAE_PDPT_PHYS:
2442	case PGMPOOLKIND_PAE_PD_PHYS:
2443	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2444	/* Nothing to monitor here. */
2445	return VINF_SUCCESS;
2446	default:
2447	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2448	}
2449
2450	/*
2451	* Install handler.
2452	*/
2453	int rc;
2454	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2455	if (pPageHead)
2456	{
2457	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2458	Assert(pPageHead->iMonitoredPrev != pPage->idx);
2459
2460	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2461	if (pPageHead->fDirty)
2462	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
2463	#endif
2464
2465	pPage->iMonitoredPrev = pPageHead->idx;
2466	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2467	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2468	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2469	pPageHead->iMonitoredNext = pPage->idx;
2470	rc = VINF_SUCCESS;
2471	}
2472	else
2473	{
2474	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2475	PVMCC pVM = pPool->CTX_SUFF(pVM);
2476	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2477	rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
2478	MMHyperCCToR3(pVM, pPage), MMHyperCCToR0(pVM, pPage), NIL_RTRCPTR, NIL_RTR3PTR /pszDesc/);
2479	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2480	* the heap size should suffice. */
2481	AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2482	PVMCPU pVCpu = VMMGetCpu(pVM);
2483	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
2484	}
2485	pPage->fMonitored = true;
2486	return rc;
2487	}
2488
2489
2490	/**
2491	* Disables write monitoring of a guest page.
2492	*
2493	* @returns VBox status code.
2494	* @retval VINF_SUCCESS on success.
2495	* @param pPool The pool.
2496	* @param pPage The cached page.
2497	*/
2498	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2499	{
2500	/*
2501	* Filter out the relevant kinds.
2502	*/
2503	switch (pPage->enmKind)
2504	{
2505	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2506	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2507	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2508	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2509	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2510	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2511	case PGMPOOLKIND_64BIT_PML4:
2512	case PGMPOOLKIND_32BIT_PD:
2513	case PGMPOOLKIND_PAE_PDPT:
2514	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2515	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2516	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2517	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2518	break;
2519
2520	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2521	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2522	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2523	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2524	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2525	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2526	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2527	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2528	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2529	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2530	case PGMPOOLKIND_ROOT_NESTED:
2531	case PGMPOOLKIND_PAE_PD_PHYS:
2532	case PGMPOOLKIND_PAE_PDPT_PHYS:
2533	case PGMPOOLKIND_32BIT_PD_PHYS:
2534	/* Nothing to monitor here. */
2535	Assert(!pPage->fMonitored);
2536	return VINF_SUCCESS;
2537
2538	default:
2539	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2540	}
2541	Assert(pPage->fMonitored);
2542
2543	/*
2544	* Remove the page from the monitored list or uninstall it if last.
2545	*/
2546	const PVMCC pVM = pPool->CTX_SUFF(pVM);
2547	int rc;
2548	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2549	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2550	{
2551	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2552	{
2553	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2554	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2555	rc = PGMHandlerPhysicalChangeUserArgs(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK,
2556	MMHyperCCToR3(pVM, pNewHead), MMHyperCCToR0(pVM, pNewHead));
2557
2558	AssertFatalRCSuccess(rc);
2559	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2560	}
2561	else
2562	{
2563	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2564	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2565	{
2566	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2567	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2568	}
2569	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2570	rc = VINF_SUCCESS;
2571	}
2572	}
2573	else
2574	{
2575	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
2576	AssertFatalRC(rc);
2577	PVMCPU pVCpu = VMMGetCpu(pVM);
2578	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2579	("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2580	}
2581	pPage->fMonitored = false;
2582
2583	/*
2584	* Remove it from the list of modified pages (if in it).
2585	*/
2586	pgmPoolMonitorModifiedRemove(pPool, pPage);
2587
2588	return rc;
2589	}
2590
2591
2592	/**
2593	* Inserts the page into the list of modified pages.
2594	*
2595	* @param pPool The pool.
2596	* @param pPage The page.
2597	*/
2598	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2599	{
2600	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2601	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2602	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2603	&& pPool->iModifiedHead != pPage->idx,
2604	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2605	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2606	pPool->iModifiedHead, pPool->cModifiedPages));
2607
2608	pPage->iModifiedNext = pPool->iModifiedHead;
2609	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2610	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2611	pPool->iModifiedHead = pPage->idx;
2612	pPool->cModifiedPages++;
2613	#ifdef VBOX_WITH_STATISTICS
2614	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2615	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2616	#endif
2617	}
2618
2619
2620	/**
2621	* Removes the page from the list of modified pages and resets the
2622	* modification counter.
2623	*
2624	* @param pPool The pool.
2625	* @param pPage The page which is believed to be in the list of modified pages.
2626	*/
2627	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2628	{
2629	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2630	if (pPool->iModifiedHead == pPage->idx)
2631	{
2632	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2633	pPool->iModifiedHead = pPage->iModifiedNext;
2634	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2635	{
2636	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2637	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2638	}
2639	pPool->cModifiedPages--;
2640	}
2641	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2642	{
2643	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2644	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2645	{
2646	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2647	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2648	}
2649	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2650	pPool->cModifiedPages--;
2651	}
2652	else
2653	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2654	pPage->cModifications = 0;
2655	}
2656
2657
2658	/**
2659	* Zaps the list of modified pages, resetting their modification counters in the process.
2660	*
2661	* @param pVM The cross context VM structure.
2662	*/
2663	static void pgmPoolMonitorModifiedClearAll(PVMCC pVM)
2664	{
2665	PGM_LOCK_VOID(pVM);
2666	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2667	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2668
2669	unsigned cPages = 0; NOREF(cPages);
2670
2671	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2672	pgmPoolResetDirtyPages(pVM);
2673	#endif
2674
2675	uint16_t idx = pPool->iModifiedHead;
2676	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2677	while (idx != NIL_PGMPOOL_IDX)
2678	{
2679	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2680	idx = pPage->iModifiedNext;
2681	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2682	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2683	pPage->cModifications = 0;
2684	Assert(++cPages);
2685	}
2686	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2687	pPool->cModifiedPages = 0;
2688	PGM_UNLOCK(pVM);
2689	}
2690
2691
2692	/**
2693	* Handle SyncCR3 pool tasks
2694	*
2695	* @returns VBox status code.
2696	* @retval VINF_SUCCESS if successfully added.
2697	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2698	* @param pVCpu The cross context virtual CPU structure.
2699	* @remark Should only be used when monitoring is available, thus placed in
2700	* the PGMPOOL_WITH_MONITORING \#ifdef.
2701	*/
2702	int pgmPoolSyncCR3(PVMCPUCC pVCpu)
2703	{
2704	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2705	LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
2706
2707	/*
2708	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2709	* Occasionally we will have to clear all the shadow page tables because we wanted
2710	* to monitor a page which was mapped by too many shadowed page tables. This operation
2711	* sometimes referred to as a 'lightweight flush'.
2712	*/
2713	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2714	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2715	pgmR3PoolClearAll(pVM, false /fFlushRemTlb/);
2716	# else /* !IN_RING3 */
2717	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2718	{
2719	Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2720	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2721
2722	/* Make sure all other VCPUs return to ring 3. */
2723	if (pVM->cCpus > 1)
2724	{
2725	VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
2726	PGM_INVL_ALL_VCPU_TLBS(pVM);
2727	}
2728	return VINF_PGM_SYNC_CR3;
2729	}
2730	# endif /* !IN_RING3 */
2731	else
2732	{
2733	pgmPoolMonitorModifiedClearAll(pVM);
2734
2735	/* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
2736	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2737	{
2738	Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
2739	return pgmPoolSyncCR3(pVCpu);
2740	}
2741	}
2742	return VINF_SUCCESS;
2743	}
2744
2745
2746	/**
2747	* Frees up at least one user entry.
2748	*
2749	* @returns VBox status code.
2750	* @retval VINF_SUCCESS if successfully added.
2751	*
2752	* @param pPool The pool.
2753	* @param iUser The user index.
2754	*/
2755	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2756	{
2757	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2758	/*
2759	* Just free cached pages in a braindead fashion.
2760	*/
2761	/** @todo walk the age list backwards and free the first with usage. */
2762	int rc = VINF_SUCCESS;
2763	do
2764	{
2765	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2766	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2767	rc = rc2;
2768	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2769	return rc;
2770	}
2771
2772
2773	/**
2774	* Inserts a page into the cache.
2775	*
2776	* This will create user node for the page, insert it into the GCPhys
2777	* hash, and insert it into the age list.
2778	*
2779	* @returns VBox status code.
2780	* @retval VINF_SUCCESS if successfully added.
2781	*
2782	* @param pPool The pool.
2783	* @param pPage The cached page.
2784	* @param GCPhys The GC physical address of the page we're gonna shadow.
2785	* @param iUser The user index.
2786	* @param iUserTable The user table index.
2787	*/
2788	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2789	{
2790	int rc = VINF_SUCCESS;
2791	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2792
2793	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
2794
2795	if (iUser != NIL_PGMPOOL_IDX)
2796	{
2797	#ifdef VBOX_STRICT
2798	/*
2799	* Check that the entry doesn't already exists.
2800	*/
2801	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2802	{
2803	uint16_t i = pPage->iUserHead;
2804	do
2805	{
2806	Assert(i < pPool->cMaxUsers);
2807	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2808	i = paUsers[i].iNext;
2809	} while (i != NIL_PGMPOOL_USER_INDEX);
2810	}
2811	#endif
2812
2813	/*
2814	* Find free a user node.
2815	*/
2816	uint16_t i = pPool->iUserFreeHead;
2817	if (i == NIL_PGMPOOL_USER_INDEX)
2818	{
2819	rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2820	if (RT_FAILURE(rc))
2821	return rc;
2822	i = pPool->iUserFreeHead;
2823	}
2824
2825	/*
2826	* Unlink the user node from the free list,
2827	* initialize and insert it into the user list.
2828	*/
2829	pPool->iUserFreeHead = paUsers[i].iNext;
2830	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2831	paUsers[i].iUser = iUser;
2832	paUsers[i].iUserTable = iUserTable;
2833	pPage->iUserHead = i;
2834	}
2835	else
2836	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2837
2838
2839	/*
2840	* Insert into cache and enable monitoring of the guest page if enabled.
2841	*
2842	* Until we implement caching of all levels, including the CR3 one, we'll
2843	* have to make sure we don't try monitor & cache any recursive reuse of
2844	* a monitored CR3 page. Because all windows versions are doing this we'll
2845	* have to be able to do combined access monitoring, CR3 + PT and
2846	* PD + PT (guest PAE).
2847	*
2848	* Update:
2849	* We're now cooperating with the CR3 monitor if an uncachable page is found.
2850	*/
2851	const bool fCanBeMonitored = true;
2852	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2853	if (fCanBeMonitored)
2854	{
2855	rc = pgmPoolMonitorInsert(pPool, pPage);
2856	AssertRC(rc);
2857	}
2858	return rc;
2859	}
2860
2861
2862	/**
2863	* Adds a user reference to a page.
2864	*
2865	* This will move the page to the head of the
2866	*
2867	* @returns VBox status code.
2868	* @retval VINF_SUCCESS if successfully added.
2869	*
2870	* @param pPool The pool.
2871	* @param pPage The cached page.
2872	* @param iUser The user index.
2873	* @param iUserTable The user table.
2874	*/
2875	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2876	{
2877	Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
2878	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2879	Assert(iUser != NIL_PGMPOOL_IDX);
2880
2881	# ifdef VBOX_STRICT
2882	/*
2883	* Check that the entry doesn't already exists. We only allow multiple
2884	* users of top-level paging structures (SHW_POOL_ROOT_IDX).
2885	*/
2886	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2887	{
2888	uint16_t i = pPage->iUserHead;
2889	do
2890	{
2891	Assert(i < pPool->cMaxUsers);
2892	/** @todo this assertion looks odd... Shouldn't it be && here? */
2893	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2894	i = paUsers[i].iNext;
2895	} while (i != NIL_PGMPOOL_USER_INDEX);
2896	}
2897	# endif
2898
2899	/*
2900	* Allocate a user node.
2901	*/
2902	uint16_t i = pPool->iUserFreeHead;
2903	if (i == NIL_PGMPOOL_USER_INDEX)
2904	{
2905	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2906	if (RT_FAILURE(rc))
2907	return rc;
2908	i = pPool->iUserFreeHead;
2909	}
2910	pPool->iUserFreeHead = paUsers[i].iNext;
2911
2912	/*
2913	* Initialize the user node and insert it.
2914	*/
2915	paUsers[i].iNext = pPage->iUserHead;
2916	paUsers[i].iUser = iUser;
2917	paUsers[i].iUserTable = iUserTable;
2918	pPage->iUserHead = i;
2919
2920	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2921	if (pPage->fDirty)
2922	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
2923	# endif
2924
2925	/*
2926	* Tell the cache to update its replacement stats for this page.
2927	*/
2928	pgmPoolCacheUsed(pPool, pPage);
2929	return VINF_SUCCESS;
2930	}
2931
2932
2933	/**
2934	* Frees a user record associated with a page.
2935	*
2936	* This does not clear the entry in the user table, it simply replaces the
2937	* user record to the chain of free records.
2938	*
2939	* @param pPool The pool.
2940	* @param pPage The shadow page.
2941	* @param iUser The shadow page pool index of the user table.
2942	* @param iUserTable The index into the user table (shadowed).
2943	*
2944	* @remarks Don't call this for root pages.
2945	*/
2946	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2947	{
2948	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
2949	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2950	Assert(iUser != NIL_PGMPOOL_IDX);
2951
2952	/*
2953	* Unlink and free the specified user entry.
2954	*/
2955
2956	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2957	uint16_t i = pPage->iUserHead;
2958	if ( i != NIL_PGMPOOL_USER_INDEX
2959	&& paUsers[i].iUser == iUser
2960	&& paUsers[i].iUserTable == iUserTable)
2961	{
2962	pPage->iUserHead = paUsers[i].iNext;
2963
2964	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2965	paUsers[i].iNext = pPool->iUserFreeHead;
2966	pPool->iUserFreeHead = i;
2967	return;
2968	}
2969
2970	/* General: Linear search. */
2971	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2972	while (i != NIL_PGMPOOL_USER_INDEX)
2973	{
2974	if ( paUsers[i].iUser == iUser
2975	&& paUsers[i].iUserTable == iUserTable)
2976	{
2977	if (iPrev != NIL_PGMPOOL_USER_INDEX)
2978	paUsers[iPrev].iNext = paUsers[i].iNext;
2979	else
2980	pPage->iUserHead = paUsers[i].iNext;
2981
2982	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2983	paUsers[i].iNext = pPool->iUserFreeHead;
2984	pPool->iUserFreeHead = i;
2985	return;
2986	}
2987	iPrev = i;
2988	i = paUsers[i].iNext;
2989	}
2990
2991	/* Fatal: didn't find it */
2992	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
2993	iUser, iUserTable, pPage->GCPhys));
2994	}
2995
2996
2997	#if 0 /* unused */
2998	/**
2999	* Gets the entry size of a shadow table.
3000	*
3001	* @param enmKind The kind of page.
3002	*
3003	* @returns The size of the entry in bytes. That is, 4 or 8.
3004	* @returns If the kind is not for a table, an assertion is raised and 0 is
3005	* returned.
3006	*/
3007	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
3008	{
3009	switch (enmKind)
3010	{
3011	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3012	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3013	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3014	case PGMPOOLKIND_32BIT_PD:
3015	case PGMPOOLKIND_32BIT_PD_PHYS:
3016	return 4;
3017
3018	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3019	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3020	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3021	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3022	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3023	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3024	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3025	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3026	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3027	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3028	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3029	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3030	case PGMPOOLKIND_64BIT_PML4:
3031	case PGMPOOLKIND_PAE_PDPT:
3032	case PGMPOOLKIND_ROOT_NESTED:
3033	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3034	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3035	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3036	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3037	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3038	case PGMPOOLKIND_PAE_PD_PHYS:
3039	case PGMPOOLKIND_PAE_PDPT_PHYS:
3040	return 8;
3041
3042	default:
3043	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3044	}
3045	}
3046	#endif /* unused */
3047
3048	#if 0 /* unused */
3049	/**
3050	* Gets the entry size of a guest table.
3051	*
3052	* @param enmKind The kind of page.
3053	*
3054	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
3055	* @returns If the kind is not for a table, an assertion is raised and 0 is
3056	* returned.
3057	*/
3058	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
3059	{
3060	switch (enmKind)
3061	{
3062	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3063	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3064	case PGMPOOLKIND_32BIT_PD:
3065	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3066	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3067	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3068	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3069	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3070	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3071	return 4;
3072
3073	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3074	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3075	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3076	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3077	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3078	case PGMPOOLKIND_64BIT_PML4:
3079	case PGMPOOLKIND_PAE_PDPT:
3080	return 8;
3081
3082	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3083	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3084	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3085	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3086	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3087	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3088	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3089	case PGMPOOLKIND_ROOT_NESTED:
3090	case PGMPOOLKIND_PAE_PD_PHYS:
3091	case PGMPOOLKIND_PAE_PDPT_PHYS:
3092	case PGMPOOLKIND_32BIT_PD_PHYS:
3093	/** @todo can we return 0? (nobody is calling this...) */
3094	AssertFailed();
3095	return 0;
3096
3097	default:
3098	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3099	}
3100	}
3101	#endif /* unused */
3102
3103
3104	/**
3105	* Checks one shadow page table entry for a mapping of a physical page.
3106	*
3107	* @returns true / false indicating removal of all relevant PTEs
3108	*
3109	* @param pVM The cross context VM structure.
3110	* @param pPhysPage The guest page in question.
3111	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3112	* @param iShw The shadow page table.
3113	* @param iPte Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
3114	*/
3115	static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
3116	{
3117	LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
3118	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3119	bool fRet = false;
3120
3121	/*
3122	* Assert sanity.
3123	*/
3124	Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
3125	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
3126	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
3127
3128	/*
3129	* Then, clear the actual mappings to the page in the shadow PT.
3130	*/
3131	switch (pPage->enmKind)
3132	{
3133	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3134	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3135	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3136	{
3137	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3138	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3139	uint32_t u32AndMask = 0;
3140	uint32_t u32OrMask = 0;
3141
3142	if (!fFlushPTEs)
3143	{
3144	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3145	{
3146	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3147	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3148	u32OrMask = X86_PTE_RW;
3149	u32AndMask = UINT32_MAX;
3150	fRet = true;
3151	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3152	break;
3153
3154	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3155	u32OrMask = 0;
3156	u32AndMask = ~X86_PTE_RW;
3157	fRet = true;
3158	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3159	break;
3160	default:
3161	/* (shouldn't be here, will assert below) */
3162	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3163	break;
3164	}
3165	}
3166	else
3167	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3168
3169	/* Update the counter if we're removing references. */
3170	if (!u32AndMask)
3171	{
3172	Assert(pPage->cPresent);
3173	Assert(pPool->cPresent);
3174	pPage->cPresent--;
3175	pPool->cPresent--;
3176	}
3177
3178	if ((pPT->a[iPte].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3179	{
3180	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
3181	X86PTE Pte;
3182	Pte.u = (pPT->a[iPte].u & u32AndMask) \| u32OrMask;
3183	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3184	Pte.u &= ~(X86PGUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3185	ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
3186	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3187	return fRet;
3188	}
3189	#ifdef LOG_ENABLED
3190	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3191	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3192	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3193	{
3194	Log(("i=%d cFound=%d\n", i, ++cFound));
3195	}
3196	#endif
3197	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
3198	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3199	break;
3200	}
3201
3202	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3203	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3204	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3205	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3206	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3207	case PGMPOOLKIND_EPT_PT_FOR_PHYS: /* physical mask the same as PAE; RW bit as well; be careful! */
3208	{
3209	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3210	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3211	uint64_t u64OrMask = 0;
3212	uint64_t u64AndMask = 0;
3213
3214	if (!fFlushPTEs)
3215	{
3216	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3217	{
3218	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3219	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3220	u64OrMask = X86_PTE_RW;
3221	u64AndMask = UINT64_MAX;
3222	fRet = true;
3223	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3224	break;
3225
3226	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3227	u64OrMask = 0;
3228	u64AndMask = ~(uint64_t)X86_PTE_RW;
3229	fRet = true;
3230	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3231	break;
3232
3233	default:
3234	/* (shouldn't be here, will assert below) */
3235	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3236	break;
3237	}
3238	}
3239	else
3240	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3241
3242	/* Update the counter if we're removing references. */
3243	if (!u64AndMask)
3244	{
3245	Assert(pPage->cPresent);
3246	Assert(pPool->cPresent);
3247	pPage->cPresent--;
3248	pPool->cPresent--;
3249	}
3250
3251	if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3252	{
3253	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3254	X86PTEPAE Pte;
3255	Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) \| u64OrMask;
3256	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3257	Pte.u &= ~(X86PGPAEUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3258
3259	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
3260	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3261	return fRet;
3262	}
3263	#ifdef LOG_ENABLED
3264	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3265	Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX), u64));
3266	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3267	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3268	Log(("i=%d cFound=%d\n", i, ++cFound));
3269	#endif
3270	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3271	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3272	break;
3273	}
3274
3275	#ifdef PGM_WITH_LARGE_PAGES
3276	/* Large page case only. */
3277	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3278	{
3279	Assert(pVM->pgm.s.fNestedPaging);
3280
3281	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3282	PEPTPD pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3283
3284	if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3285	{
3286	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3287	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3288	pPD->a[iPte].u = 0;
3289	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3290
3291	/* Update the counter as we're removing references. */
3292	Assert(pPage->cPresent);
3293	Assert(pPool->cPresent);
3294	pPage->cPresent--;
3295	pPool->cPresent--;
3296
3297	return fRet;
3298	}
3299	# ifdef LOG_ENABLED
3300	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3301	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3302	if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3303	Log(("i=%d cFound=%d\n", i, ++cFound));
3304	# endif
3305	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3306	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3307	break;
3308	}
3309
3310	/* AMD-V nested paging / /* @todo merge with EPT as we only check the parts that are identical. */
3311	case PGMPOOLKIND_PAE_PD_PHYS:
3312	{
3313	Assert(pVM->pgm.s.fNestedPaging);
3314
3315	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3316	PX86PDPAE pPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3317
3318	if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3319	{
3320	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3321	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3322	pPD->a[iPte].u = 0;
3323	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3324
3325	/* Update the counter as we're removing references. */
3326	Assert(pPage->cPresent);
3327	Assert(pPool->cPresent);
3328	pPage->cPresent--;
3329	pPool->cPresent--;
3330	return fRet;
3331	}
3332	# ifdef LOG_ENABLED
3333	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3334	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3335	if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3336	Log(("i=%d cFound=%d\n", i, ++cFound));
3337	# endif
3338	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3339	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3340	break;
3341	}
3342	#endif /* PGM_WITH_LARGE_PAGES */
3343
3344	default:
3345	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3346	}
3347
3348	/* not reached. */
3349	#ifndef _MSC_VER
3350	return fRet;
3351	#endif
3352	}
3353
3354
3355	/**
3356	* Scans one shadow page table for mappings of a physical page.
3357	*
3358	* @param pVM The cross context VM structure.
3359	* @param pPhysPage The guest page in question.
3360	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3361	* @param iShw The shadow page table.
3362	*/
3363	static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
3364	{
3365	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3366
3367	/* We should only come here with when there's only one reference to this physical page. */
3368	Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
3369
3370	Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
3371	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3372	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
3373	if (!fKeptPTEs)
3374	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3375	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3376	}
3377
3378
3379	/**
3380	* Flushes a list of shadow page tables mapping the same physical page.
3381	*
3382	* @param pVM The cross context VM structure.
3383	* @param pPhysPage The guest page in question.
3384	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3385	* @param iPhysExt The physical cross reference extent list to flush.
3386	*/
3387	static void pgmPoolTrackFlushGCPhysPTs(PVMCC pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
3388	{
3389	PGM_LOCK_ASSERT_OWNER(pVM);
3390	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3391	bool fKeepList = false;
3392
3393	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3394	Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
3395
3396	const uint16_t iPhysExtStart = iPhysExt;
3397	PPGMPOOLPHYSEXT pPhysExt;
3398	do
3399	{
3400	Assert(iPhysExt < pPool->cMaxPhysExts);
3401	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3402	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3403	{
3404	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3405	{
3406	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
3407	if (!fKeptPTEs)
3408	{
3409	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3410	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3411	}
3412	else
3413	fKeepList = true;
3414	}
3415	}
3416	/* next */
3417	iPhysExt = pPhysExt->iNext;
3418	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3419
3420	if (!fKeepList)
3421	{
3422	/* insert the list into the free list and clear the ram range entry. */
3423	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3424	pPool->iPhysExtFreeHead = iPhysExtStart;
3425	/* Invalidate the tracking data. */
3426	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3427	}
3428
3429	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3430	}
3431
3432
3433	/**
3434	* Flushes all shadow page table mappings of the given guest page.
3435	*
3436	* This is typically called when the host page backing the guest one has been
3437	* replaced or when the page protection was changed due to a guest access
3438	* caught by the monitoring.
3439	*
3440	* @returns VBox status code.
3441	* @retval VINF_SUCCESS if all references has been successfully cleared.
3442	* @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3443	* pool cleaning. FF and sync flags are set.
3444	*
3445	* @param pVM The cross context VM structure.
3446	* @param GCPhysPage GC physical address of the page in question
3447	* @param pPhysPage The guest page in question.
3448	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3449	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3450	* flushed, it is NOT touched if this isn't necessary.
3451	* The caller MUST initialized this to @a false.
3452	*/
3453	int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
3454	{
3455	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3456	PGM_LOCK_VOID(pVM);
3457	int rc = VINF_SUCCESS;
3458
3459	#ifdef PGM_WITH_LARGE_PAGES
3460	/* Is this page part of a large page? */
3461	if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
3462	{
3463	RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
3464	GCPhysPage &= X86_PDE_PAE_PG_MASK;
3465
3466	/* Fetch the large page base. */
3467	PPGMPAGE pLargePage;
3468	if (GCPhysBase != GCPhysPage)
3469	{
3470	pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
3471	AssertFatal(pLargePage);
3472	}
3473	else
3474	pLargePage = pPhysPage;
3475
3476	Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
3477
3478	if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
3479	{
3480	/* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
3481	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3482	pVM->pgm.s.cLargePagesDisabled++;
3483
3484	/* Update the base as that only that one has a reference and there's only one PDE to clear. */
3485	rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
3486
3487	*pfFlushTLBs = true;
3488	PGM_UNLOCK(pVM);
3489	return rc;
3490	}
3491	}
3492	#else
3493	NOREF(GCPhysPage);
3494	#endif /* PGM_WITH_LARGE_PAGES */
3495
3496	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3497	if (u16)
3498	{
3499	/*
3500	* The zero page is currently screwing up the tracking and we'll
3501	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3502	* is defined, zero pages won't normally be mapped. Some kind of solution
3503	* will be needed for this problem of course, but it will have to wait...
3504	*/
3505	if ( PGM_PAGE_IS_ZERO(pPhysPage)
3506	\|\| PGM_PAGE_IS_BALLOONED(pPhysPage))
3507	rc = VINF_PGM_GCPHYS_ALIASED;
3508	else
3509	{
3510	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3511	{
3512	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3513	pgmPoolTrackFlushGCPhysPT(pVM,
3514	pPhysPage,
3515	fFlushPTEs,
3516	PGMPOOL_TD_GET_IDX(u16));
3517	}
3518	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3519	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
3520	else
3521	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3522	*pfFlushTLBs = true;
3523	}
3524	}
3525
3526	if (rc == VINF_PGM_GCPHYS_ALIASED)
3527	{
3528	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3529	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3530	rc = VINF_PGM_SYNC_CR3;
3531	}
3532	PGM_UNLOCK(pVM);
3533	return rc;
3534	}
3535
3536
3537	/**
3538	* Scans all shadow page tables for mappings of a physical page.
3539	*
3540	* This may be slow, but it's most likely more efficient than cleaning
3541	* out the entire page pool / cache.
3542	*
3543	* @returns VBox status code.
3544	* @retval VINF_SUCCESS if all references has been successfully cleared.
3545	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3546	* a page pool cleaning.
3547	*
3548	* @param pVM The cross context VM structure.
3549	* @param pPhysPage The guest page in question.
3550	*/
3551	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage)
3552	{
3553	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3554	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3555	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3556	pPool->cUsedPages, pPool->cPresent, pPhysPage));
3557
3558	/*
3559	* There is a limit to what makes sense.
3560	*/
3561	if ( pPool->cPresent > 1024
3562	&& pVM->cCpus == 1)
3563	{
3564	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3565	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3566	return VINF_PGM_GCPHYS_ALIASED;
3567	}
3568
3569	/*
3570	* Iterate all the pages until we've encountered all that in use.
3571	* This is simple but not quite optimal solution.
3572	*/
3573	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage);
3574	unsigned cLeft = pPool->cUsedPages;
3575	unsigned iPage = pPool->cCurPages;
3576	while (--iPage >= PGMPOOL_IDX_FIRST)
3577	{
3578	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3579	if ( pPage->GCPhys != NIL_RTGCPHYS
3580	&& pPage->cPresent)
3581	{
3582	switch (pPage->enmKind)
3583	{
3584	/*
3585	* We only care about shadow page tables.
3586	*/
3587	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3588	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3589	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3590	{
3591	const uint32_t u32 = (uint32_t)u64;
3592	unsigned cPresent = pPage->cPresent;
3593	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3594	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3595	{
3596	const X86PGUINT uPte = pPT->a[i].u;
3597	if (uPte & X86_PTE_P)
3598	{
3599	if ((uPte & X86_PTE_PG_MASK) == u32)
3600	{
3601	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3602	ASMAtomicWriteU32(&pPT->a[i].u, 0);
3603
3604	/* Update the counter as we're removing references. */
3605	Assert(pPage->cPresent);
3606	Assert(pPool->cPresent);
3607	pPage->cPresent--;
3608	pPool->cPresent--;
3609	}
3610	if (!--cPresent)
3611	break;
3612	}
3613	}
3614	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3615	break;
3616	}
3617
3618	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3619	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3620	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3621	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3622	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3623	{
3624	unsigned cPresent = pPage->cPresent;
3625	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3626	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3627	if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
3628	{
3629	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_PAE_PG_MASK) == u64)
3630	{
3631	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3632	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[i], 0); /// @todo why not atomic?
3633
3634	/* Update the counter as we're removing references. */
3635	Assert(pPage->cPresent);
3636	Assert(pPool->cPresent);
3637	pPage->cPresent--;
3638	pPool->cPresent--;
3639	}
3640	if (!--cPresent)
3641	break;
3642	}
3643	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3644	break;
3645	}
3646
3647	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3648	{
3649	unsigned cPresent = pPage->cPresent;
3650	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3651	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3652	{
3653	X86PGPAEUINT const uPte = pPT->a[i].u;
3654	if (uPte & EPT_E_READ)
3655	{
3656	if ((uPte & EPT_PTE_PG_MASK) == u64)
3657	{
3658	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3659	ASMAtomicWriteU64(&pPT->a[i].u, 0);
3660
3661	/* Update the counter as we're removing references. */
3662	Assert(pPage->cPresent);
3663	Assert(pPool->cPresent);
3664	pPage->cPresent--;
3665	pPool->cPresent--;
3666	}
3667	if (!--cPresent)
3668	break;
3669	}
3670	}
3671	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3672	break;
3673	}
3674	}
3675
3676	if (!--cLeft)
3677	break;
3678	}
3679	}
3680
3681	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3682	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3683
3684	/*
3685	* There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
3686	*/
3687	if (pPool->cPresent > 1024)
3688	{
3689	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3690	return VINF_PGM_GCPHYS_ALIASED;
3691	}
3692
3693	return VINF_SUCCESS;
3694	}
3695
3696
3697	/**
3698	* Clears the user entry in a user table.
3699	*
3700	* This is used to remove all references to a page when flushing it.
3701	*/
3702	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3703	{
3704	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3705	Assert(pUser->iUser < pPool->cCurPages);
3706	uint32_t iUserTable = pUser->iUserTable;
3707
3708	/*
3709	* Map the user page. Ignore references made by fictitious pages.
3710	*/
3711	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3712	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3713	union
3714	{
3715	uint64_t *pau64;
3716	uint32_t *pau32;
3717	} u;
3718	if (pUserPage->idx < PGMPOOL_IDX_FIRST)
3719	{
3720	Assert(!pUserPage->pvPageR3);
3721	return;
3722	}
3723	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3724
3725
3726	/* Safety precaution in case we change the paging for other modes too in the future. */
3727	Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
3728
3729	#ifdef VBOX_STRICT
3730	/*
3731	* Some sanity checks.
3732	*/
3733	switch (pUserPage->enmKind)
3734	{
3735	case PGMPOOLKIND_32BIT_PD:
3736	case PGMPOOLKIND_32BIT_PD_PHYS:
3737	Assert(iUserTable < X86_PG_ENTRIES);
3738	break;
3739	case PGMPOOLKIND_PAE_PDPT:
3740	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3741	case PGMPOOLKIND_PAE_PDPT_PHYS:
3742	Assert(iUserTable < 4);
3743	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3744	break;
3745	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3746	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3747	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3748	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3749	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3750	case PGMPOOLKIND_PAE_PD_PHYS:
3751	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3752	break;
3753	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3754	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3755	break;
3756	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3757	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3758	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3759	break;
3760	case PGMPOOLKIND_64BIT_PML4:
3761	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3762	/* GCPhys >> PAGE_SHIFT is the index here */
3763	break;
3764	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3765	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3766	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3767	break;
3768
3769	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3770	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3771	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3772	break;
3773
3774	case PGMPOOLKIND_ROOT_NESTED:
3775	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3776	break;
3777
3778	default:
3779	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3780	break;
3781	}
3782	#endif /* VBOX_STRICT */
3783
3784	/*
3785	* Clear the entry in the user page.
3786	*/
3787	switch (pUserPage->enmKind)
3788	{
3789	/* 32-bit entries */
3790	case PGMPOOLKIND_32BIT_PD:
3791	case PGMPOOLKIND_32BIT_PD_PHYS:
3792	ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
3793	break;
3794
3795	/* 64-bit entries */
3796	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3797	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3798	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3799	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3800	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3801	case PGMPOOLKIND_PAE_PD_PHYS:
3802	case PGMPOOLKIND_PAE_PDPT_PHYS:
3803	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3804	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3805	case PGMPOOLKIND_64BIT_PML4:
3806	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3807	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3808	case PGMPOOLKIND_PAE_PDPT:
3809	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3810	case PGMPOOLKIND_ROOT_NESTED:
3811	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3812	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3813	ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
3814	break;
3815
3816	default:
3817	AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
3818	}
3819	PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
3820	}
3821
3822
3823	/**
3824	* Clears all users of a page.
3825	*/
3826	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3827	{
3828	/*
3829	* Free all the user records.
3830	*/
3831	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
3832
3833	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3834	uint16_t i = pPage->iUserHead;
3835	while (i != NIL_PGMPOOL_USER_INDEX)
3836	{
3837	/* Clear enter in user table. */
3838	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
3839
3840	/* Free it. */
3841	const uint16_t iNext = paUsers[i].iNext;
3842	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3843	paUsers[i].iNext = pPool->iUserFreeHead;
3844	pPool->iUserFreeHead = i;
3845
3846	/* Next. */
3847	i = iNext;
3848	}
3849	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3850	}
3851
3852
3853	/**
3854	* Allocates a new physical cross reference extent.
3855	*
3856	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
3857	* @param pVM The cross context VM structure.
3858	* @param piPhysExt Where to store the phys ext index.
3859	*/
3860	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt)
3861	{
3862	PGM_LOCK_ASSERT_OWNER(pVM);
3863	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3864	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3865	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3866	{
3867	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3868	return NULL;
3869	}
3870	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3871	pPool->iPhysExtFreeHead = pPhysExt->iNext;
3872	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3873	*piPhysExt = iPhysExt;
3874	return pPhysExt;
3875	}
3876
3877
3878	/**
3879	* Frees a physical cross reference extent.
3880	*
3881	* @param pVM The cross context VM structure.
3882	* @param iPhysExt The extent to free.
3883	*/
3884	void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt)
3885	{
3886	PGM_LOCK_ASSERT_OWNER(pVM);
3887	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3888	Assert(iPhysExt < pPool->cMaxPhysExts);
3889	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3890	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3891	{
3892	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3893	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3894	}
3895	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3896	pPool->iPhysExtFreeHead = iPhysExt;
3897	}
3898
3899
3900	/**
3901	* Frees a physical cross reference extent.
3902	*
3903	* @param pVM The cross context VM structure.
3904	* @param iPhysExt The extent to free.
3905	*/
3906	void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt)
3907	{
3908	PGM_LOCK_ASSERT_OWNER(pVM);
3909	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3910
3911	const uint16_t iPhysExtStart = iPhysExt;
3912	PPGMPOOLPHYSEXT pPhysExt;
3913	do
3914	{
3915	Assert(iPhysExt < pPool->cMaxPhysExts);
3916	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3917	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3918	{
3919	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3920	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3921	}
3922
3923	/* next */
3924	iPhysExt = pPhysExt->iNext;
3925	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3926
3927	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3928	pPool->iPhysExtFreeHead = iPhysExtStart;
3929	}
3930
3931
3932	/**
3933	* Insert a reference into a list of physical cross reference extents.
3934	*
3935	* @returns The new tracking data for PGMPAGE.
3936	*
3937	* @param pVM The cross context VM structure.
3938	* @param iPhysExt The physical extent index of the list head.
3939	* @param iShwPT The shadow page table index.
3940	* @param iPte Page table entry
3941	*
3942	*/
3943	static uint16_t pgmPoolTrackPhysExtInsert(PVMCC pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
3944	{
3945	PGM_LOCK_ASSERT_OWNER(pVM);
3946	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3947	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3948
3949	/*
3950	* Special common cases.
3951	*/
3952	if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
3953	{
3954	paPhysExts[iPhysExt].aidx[1] = iShwPT;
3955	paPhysExts[iPhysExt].apte[1] = iPte;
3956	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
3957	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
3958	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3959	}
3960	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3961	{
3962	paPhysExts[iPhysExt].aidx[2] = iShwPT;
3963	paPhysExts[iPhysExt].apte[2] = iPte;
3964	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
3965	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
3966	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3967	}
3968	AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
3969
3970	/*
3971	* General treatment.
3972	*/
3973	const uint16_t iPhysExtStart = iPhysExt;
3974	unsigned cMax = 15;
3975	for (;;)
3976	{
3977	Assert(iPhysExt < pPool->cMaxPhysExts);
3978	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3979	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3980	{
3981	paPhysExts[iPhysExt].aidx[i] = iShwPT;
3982	paPhysExts[iPhysExt].apte[i] = iPte;
3983	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
3984	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
3985	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
3986	}
3987	if (!--cMax)
3988	{
3989	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackOverflows);
3990	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3991	LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
3992	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3993	}
3994
3995	/* advance */
3996	iPhysExt = paPhysExts[iPhysExt].iNext;
3997	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3998	break;
3999	}
4000
4001	/*
4002	* Add another extent to the list.
4003	*/
4004	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4005	if (!pNew)
4006	{
4007	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackNoExtentsLeft);
4008	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4009	LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
4010	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4011	}
4012	pNew->iNext = iPhysExtStart;
4013	pNew->aidx[0] = iShwPT;
4014	pNew->apte[0] = iPte;
4015	LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
4016	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4017	}
4018
4019
4020	/**
4021	* Add a reference to guest physical page where extents are in use.
4022	*
4023	* @returns The new tracking data for PGMPAGE.
4024	*
4025	* @param pVM The cross context VM structure.
4026	* @param pPhysPage Pointer to the aPages entry in the ram range.
4027	* @param u16 The ram range flags (top 16-bits).
4028	* @param iShwPT The shadow page table index.
4029	* @param iPte Page table entry
4030	*/
4031	uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
4032	{
4033	PGM_LOCK_VOID(pVM);
4034	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
4035	{
4036	/*
4037	* Convert to extent list.
4038	*/
4039	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
4040	uint16_t iPhysExt;
4041	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4042	if (pPhysExt)
4043	{
4044	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
4045	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliased);
4046	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
4047	pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
4048	pPhysExt->aidx[1] = iShwPT;
4049	pPhysExt->apte[1] = iPte;
4050	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4051	}
4052	else
4053	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4054	}
4055	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
4056	{
4057	/*
4058	* Insert into the extent list.
4059	*/
4060	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
4061	}
4062	else
4063	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedLots);
4064	PGM_UNLOCK(pVM);
4065	return u16;
4066	}
4067
4068
4069	/**
4070	* Clear references to guest physical memory.
4071	*
4072	* @param pPool The pool.
4073	* @param pPage The page.
4074	* @param pPhysPage Pointer to the aPages entry in the ram range.
4075	* @param iPte Shadow PTE index
4076	*/
4077	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
4078	{
4079	PVMCC pVM = pPool->CTX_SUFF(pVM);
4080	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4081	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4082
4083	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
4084	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
4085	{
4086	PGM_LOCK_VOID(pVM);
4087
4088	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
4089	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4090	do
4091	{
4092	Assert(iPhysExt < pPool->cMaxPhysExts);
4093
4094	/*
4095	* Look for the shadow page and check if it's all freed.
4096	*/
4097	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4098	{
4099	if ( paPhysExts[iPhysExt].aidx[i] == pPage->idx
4100	&& paPhysExts[iPhysExt].apte[i] == iPte)
4101	{
4102	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
4103	paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4104
4105	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4106	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
4107	{
4108	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
4109	PGM_UNLOCK(pVM);
4110	return;
4111	}
4112
4113	/* we can free the node. */
4114	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
4115	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
4116	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
4117	{
4118	/* lonely node */
4119	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4120	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
4121	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
4122	}
4123	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
4124	{
4125	/* head */
4126	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
4127	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
4128	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4129	}
4130	else
4131	{
4132	/* in list */
4133	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
4134	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
4135	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4136	}
4137	iPhysExt = iPhysExtNext;
4138	PGM_UNLOCK(pVM);
4139	return;
4140	}
4141	}
4142
4143	/* next */
4144	iPhysExtPrev = iPhysExt;
4145	iPhysExt = paPhysExts[iPhysExt].iNext;
4146	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4147
4148	PGM_UNLOCK(pVM);
4149	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4150	}
4151	else /* nothing to do */
4152	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
4153	}
4154
4155	/**
4156	* Clear references to guest physical memory.
4157	*
4158	* This is the same as pgmPoolTracDerefGCPhysHint except that the guest
4159	* physical address is assumed to be correct, so the linear search can be
4160	* skipped and we can assert at an earlier point.
4161	*
4162	* @param pPool The pool.
4163	* @param pPage The page.
4164	* @param HCPhys The host physical address corresponding to the guest page.
4165	* @param GCPhys The guest physical address corresponding to HCPhys.
4166	* @param iPte Shadow PTE index
4167	*/
4168	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
4169	{
4170	/*
4171	* Lookup the page and check if it checks out before derefing it.
4172	*/
4173	PVMCC pVM = pPool->CTX_SUFF(pVM);
4174	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
4175	if (pPhysPage)
4176	{
4177	Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
4178	#ifdef LOG_ENABLED
4179	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
4180	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
4181	#endif
4182	if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
4183	{
4184	Assert(pPage->cPresent);
4185	Assert(pPool->cPresent);
4186	pPage->cPresent--;
4187	pPool->cPresent--;
4188	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4189	return;
4190	}
4191
4192	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp\n",
4193	HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage)));
4194	}
4195	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
4196	}
4197
4198
4199	/**
4200	* Clear references to guest physical memory.
4201	*
4202	* @param pPool The pool.
4203	* @param pPage The page.
4204	* @param HCPhys The host physical address corresponding to the guest page.
4205	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
4206	* @param iPte Shadow pte index
4207	*/
4208	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
4209	{
4210	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
4211
4212	/*
4213	* Try the hint first.
4214	*/
4215	RTHCPHYS HCPhysHinted;
4216	PVMCC pVM = pPool->CTX_SUFF(pVM);
4217	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
4218	if (pPhysPage)
4219	{
4220	HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
4221	Assert(HCPhysHinted);
4222	if (HCPhysHinted == HCPhys)
4223	{
4224	Assert(pPage->cPresent);
4225	Assert(pPool->cPresent);
4226	pPage->cPresent--;
4227	pPool->cPresent--;
4228	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4229	return;
4230	}
4231	}
4232	else
4233	HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
4234
4235	/*
4236	* Damn, the hint didn't work. We'll have to do an expensive linear search.
4237	*/
4238	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
4239	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
4240	while (pRam)
4241	{
4242	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4243	while (iPage-- > 0)
4244	{
4245	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
4246	{
4247	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
4248	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
4249	Assert(pPage->cPresent);
4250	Assert(pPool->cPresent);
4251	pPage->cPresent--;
4252	pPool->cPresent--;
4253	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
4254	return;
4255	}
4256	}
4257	pRam = pRam->CTX_SUFF(pNext);
4258	}
4259
4260	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
4261	}
4262
4263
4264	/**
4265	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
4266	*
4267	* @param pPool The pool.
4268	* @param pPage The page.
4269	* @param pShwPT The shadow page table (mapping of the page).
4270	* @param pGstPT The guest page table.
4271	*/
4272	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
4273	{
4274	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4275	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4276	{
4277	const X86PGUINT uPte = pShwPT->a[i].u;
4278	Assert(!(uPte & RT_BIT_32(10)));
4279	if (uPte & X86_PTE_P)
4280	{
4281	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
4282	i, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
4283	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
4284	if (!pPage->cPresent)
4285	break;
4286	}
4287	}
4288	}
4289
4290
4291	/**
4292	* Clear references to guest physical memory in a PAE / 32-bit page table.
4293	*
4294	* @param pPool The pool.
4295	* @param pPage The page.
4296	* @param pShwPT The shadow page table (mapping of the page).
4297	* @param pGstPT The guest page table (just a half one).
4298	*/
4299	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
4300	{
4301	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4302	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4303	{
4304	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4305	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4306	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4307	{
4308	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
4309	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
4310	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4311	if (!pPage->cPresent)
4312	break;
4313	}
4314	}
4315	}
4316
4317
4318	/**
4319	* Clear references to guest physical memory in a PAE / PAE page table.
4320	*
4321	* @param pPool The pool.
4322	* @param pPage The page.
4323	* @param pShwPT The shadow page table (mapping of the page).
4324	* @param pGstPT The guest page table.
4325	*/
4326	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
4327	{
4328	RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
4329	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4330	{
4331	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4332	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4333	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4334	{
4335	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
4336	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
4337	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4338	if (!pPage->cPresent)
4339	break;
4340	}
4341	}
4342	}
4343
4344
4345	/**
4346	* Clear references to guest physical memory in a 32-bit / 4MB page table.
4347	*
4348	* @param pPool The pool.
4349	* @param pPage The page.
4350	* @param pShwPT The shadow page table (mapping of the page).
4351	*/
4352	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
4353	{
4354	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4355	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4356	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4357	{
4358	const X86PGUINT uPte = pShwPT->a[i].u;
4359	Assert(!(uPte & RT_BIT_32(10)));
4360	if (uPte & X86_PTE_P)
4361	{
4362	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
4363	i, uPte & X86_PTE_PG_MASK, GCPhys));
4364	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4365	if (!pPage->cPresent)
4366	break;
4367	}
4368	}
4369	}
4370
4371
4372	/**
4373	* Clear references to guest physical memory in a PAE / 2/4MB page table.
4374	*
4375	* @param pPool The pool.
4376	* @param pPage The page.
4377	* @param pShwPT The shadow page table (mapping of the page).
4378	*/
4379	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
4380	{
4381	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4382	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4383	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4384	{
4385	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4386	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4387	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4388	{
4389	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
4390	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
4391	pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
4392	if (!pPage->cPresent)
4393	break;
4394	}
4395	}
4396	}
4397
4398
4399	/**
4400	* Clear references to shadowed pages in an EPT page table.
4401	*
4402	* @param pPool The pool.
4403	* @param pPage The page.
4404	* @param pShwPT The shadow page directory pointer table (mapping of the
4405	* page).
4406	*/
4407	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4408	{
4409	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4410	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4411	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4412	{
4413	X86PGPAEUINT const uPte = pShwPT->a[i].u;
4414	Assert((uPte & UINT64_C(0xfff0000000000f80)) == 0);
4415	if (uPte & EPT_E_READ)
4416	{
4417	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4418	i, uPte & EPT_PTE_PG_MASK, pPage->GCPhys));
4419	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4420	if (!pPage->cPresent)
4421	break;
4422	}
4423	}
4424	}
4425
4426
4427	/**
4428	* Clear references to shadowed pages in a 32 bits page directory.
4429	*
4430	* @param pPool The pool.
4431	* @param pPage The page.
4432	* @param pShwPD The shadow page directory (mapping of the page).
4433	*/
4434	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
4435	{
4436	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4437	{
4438	X86PGUINT const uPde = pShwPD->a[i].u;
4439	if (uPde & X86_PDE_P)
4440	{
4441	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
4442	if (pSubPage)
4443	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4444	else
4445	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
4446	}
4447	}
4448	}
4449
4450
4451	/**
4452	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
4453	*
4454	* @param pPool The pool.
4455	* @param pPage The page.
4456	* @param pShwPD The shadow page directory (mapping of the page).
4457	*/
4458	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
4459	{
4460	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4461	{
4462	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4463	if (uPde & X86_PDE_P)
4464	{
4465	#ifdef PGM_WITH_LARGE_PAGES
4466	if (uPde & X86_PDE_PS)
4467	{
4468	Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
4469	i, uPde & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4470	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & X86_PDE2M_PAE_PG_MASK,
4471	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4472	i);
4473	}
4474	else
4475	#endif
4476	{
4477	Assert((uPde & (X86_PDE_PAE_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000000))) == 0);
4478	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & X86_PDE_PAE_PG_MASK);
4479	if (pSubPage)
4480	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4481	else
4482	AssertFatalMsgFailed(("%RX64\n", uPde & X86_PDE_PAE_PG_MASK));
4483	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4484	}
4485	}
4486	}
4487	}
4488
4489
4490	/**
4491	* Clear references to shadowed pages in a PAE page directory pointer table.
4492	*
4493	* @param pPool The pool.
4494	* @param pPage The page.
4495	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4496	*/
4497	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4498	{
4499	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4500	{
4501	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4502	Assert((uPdpe & (X86_PDPE_PAE_MBZ_MASK \| UINT64_C(0x7ff0000000000200))) == 0);
4503	if (uPdpe & X86_PDPE_P)
4504	{
4505	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4506	if (pSubPage)
4507	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4508	else
4509	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4510	}
4511	}
4512	}
4513
4514
4515	/**
4516	* Clear references to shadowed pages in a 64-bit page directory pointer table.
4517	*
4518	* @param pPool The pool.
4519	* @param pPage The page.
4520	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4521	*/
4522	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4523	{
4524	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4525	{
4526	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4527	Assert((uPdpe & (X86_PDPE_LM_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4528	if (uPdpe & X86_PDPE_P)
4529	{
4530	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4531	if (pSubPage)
4532	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4533	else
4534	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4535	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4536	}
4537	}
4538	}
4539
4540
4541	/**
4542	* Clear references to shadowed pages in a 64-bit level 4 page table.
4543	*
4544	* @param pPool The pool.
4545	* @param pPage The page.
4546	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4547	*/
4548	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4549	{
4550	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4551	{
4552	X86PGPAEUINT const uPml4e = pShwPML4->a[i].u;
4553	Assert((uPml4e & (X86_PML4E_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4554	if (uPml4e & X86_PML4E_P)
4555	{
4556	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & X86_PDPE_PG_MASK);
4557	if (pSubPage)
4558	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4559	else
4560	AssertFatalMsgFailed(("%RX64\n", uPml4e & X86_PML4E_PG_MASK));
4561	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4562	}
4563	}
4564	}
4565
4566
4567	/**
4568	* Clear references to shadowed pages in an EPT page directory.
4569	*
4570	* @param pPool The pool.
4571	* @param pPage The page.
4572	* @param pShwPD The shadow page directory (mapping of the page).
4573	*/
4574	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4575	{
4576	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4577	{
4578	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4579	Assert((uPde & UINT64_C(0xfff0000000000f80)) == 0);
4580	if (uPde & EPT_E_READ)
4581	{
4582	#ifdef PGM_WITH_LARGE_PAGES
4583	if (uPde & EPT_E_LEAF)
4584	{
4585	Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
4586	i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
4587	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK,
4588	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4589	i);
4590	}
4591	else
4592	#endif
4593	{
4594	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK);
4595	if (pSubPage)
4596	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4597	else
4598	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4599	}
4600	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4601	}
4602	}
4603	}
4604
4605
4606	/**
4607	* Clear references to shadowed pages in an EPT page directory pointer table.
4608	*
4609	* @param pPool The pool.
4610	* @param pPage The page.
4611	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4612	*/
4613	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4614	{
4615	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4616	{
4617	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4618	Assert((uPdpe & UINT64_C(0xfff0000000000f80)) == 0);
4619	if (uPdpe & EPT_E_READ)
4620	{
4621	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & EPT_PDPTE_PG_MASK);
4622	if (pSubPage)
4623	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4624	else
4625	AssertFatalMsgFailed(("%RX64\n", uPdpe & EPT_PDPTE_PG_MASK));
4626	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4627	}
4628	}
4629	}
4630
4631
4632	/**
4633	* Clears all references made by this page.
4634	*
4635	* This includes other shadow pages and GC physical addresses.
4636	*
4637	* @param pPool The pool.
4638	* @param pPage The page.
4639	*/
4640	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4641	{
4642	/*
4643	* Map the shadow page and take action according to the page kind.
4644	*/
4645	PVMCC pVM = pPool->CTX_SUFF(pVM);
4646	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4647	switch (pPage->enmKind)
4648	{
4649	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4650	{
4651	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4652	void *pvGst;
4653	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4654	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4655	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4656	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4657	break;
4658	}
4659
4660	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4661	{
4662	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4663	void *pvGst;
4664	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4665	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
4666	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4667	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4668	break;
4669	}
4670
4671	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4672	{
4673	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4674	void *pvGst;
4675	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4676	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
4677	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4678	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4679	break;
4680	}
4681
4682	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4683	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4684	{
4685	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4686	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4687	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4688	break;
4689	}
4690
4691	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
4692	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4693	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4694	{
4695	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4696	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
4697	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4698	break;
4699	}
4700
4701	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4702	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4703	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4704	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4705	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4706	case PGMPOOLKIND_PAE_PD_PHYS:
4707	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4708	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4709	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
4710	break;
4711
4712	case PGMPOOLKIND_32BIT_PD_PHYS:
4713	case PGMPOOLKIND_32BIT_PD:
4714	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
4715	break;
4716
4717	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4718	case PGMPOOLKIND_PAE_PDPT:
4719	case PGMPOOLKIND_PAE_PDPT_PHYS:
4720	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
4721	break;
4722
4723	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4724	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4725	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
4726	break;
4727
4728	case PGMPOOLKIND_64BIT_PML4:
4729	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
4730	break;
4731
4732	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4733	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
4734	break;
4735
4736	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4737	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
4738	break;
4739
4740	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4741	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
4742	break;
4743
4744	default:
4745	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
4746	}
4747
4748	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
4749	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4750	ASMMemZeroPage(pvShw);
4751	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4752	pPage->fZeroed = true;
4753	Assert(!pPage->cPresent);
4754	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
4755	}
4756
4757
4758	/**
4759	* Flushes a pool page.
4760	*
4761	* This moves the page to the free list after removing all user references to it.
4762	*
4763	* @returns VBox status code.
4764	* @retval VINF_SUCCESS on success.
4765	* @param pPool The pool.
4766	* @param pPage The shadow page.
4767	* @param fFlush Flush the TLBS when required (should only be false in very specific use cases!!)
4768	*/
4769	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
4770	{
4771	PVMCC pVM = pPool->CTX_SUFF(pVM);
4772	bool fFlushRequired = false;
4773
4774	int rc = VINF_SUCCESS;
4775	STAM_PROFILE_START(&pPool->StatFlushPage, f);
4776	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4777	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4778
4779	/*
4780	* Reject any attempts at flushing any of the special root pages (shall
4781	* not happen).
4782	*/
4783	AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
4784	("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
4785	pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
4786	VINF_SUCCESS);
4787
4788	PGM_LOCK_VOID(pVM);
4789
4790	/*
4791	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4792	*/
4793	if (pgmPoolIsPageLocked(pPage))
4794	{
4795	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4796	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4797	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4798	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
4799	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4800	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4801	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4802	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4803	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
4804	\|\| pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
4805	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
4806	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4807	PGM_UNLOCK(pVM);
4808	return VINF_SUCCESS;
4809	}
4810
4811	/*
4812	* Mark the page as being in need of an ASMMemZeroPage().
4813	*/
4814	pPage->fZeroed = false;
4815
4816	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4817	if (pPage->fDirty)
4818	pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
4819	#endif
4820
4821	/* If there are any users of this table, then we must issue a tlb flush on all VCPUs. */
4822	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
4823	fFlushRequired = true;
4824
4825	/*
4826	* Clear the page.
4827	*/
4828	pgmPoolTrackClearPageUsers(pPool, pPage);
4829	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
4830	pgmPoolTrackDeref(pPool, pPage);
4831	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
4832
4833	/*
4834	* Flush it from the cache.
4835	*/
4836	pgmPoolCacheFlushPage(pPool, pPage);
4837
4838	/*
4839	* Deregistering the monitoring.
4840	*/
4841	if (pPage->fMonitored)
4842	rc = pgmPoolMonitorFlush(pPool, pPage);
4843
4844	/*
4845	* Free the page.
4846	*/
4847	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
4848	pPage->iNext = pPool->iFreeHead;
4849	pPool->iFreeHead = pPage->idx;
4850	pPage->enmKind = PGMPOOLKIND_FREE;
4851	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4852	pPage->GCPhys = NIL_RTGCPHYS;
4853	pPage->fReusedFlushPending = false;
4854
4855	pPool->cUsedPages--;
4856
4857	/* Flush the TLBs of all VCPUs if required. */
4858	if ( fFlushRequired
4859	&& fFlush)
4860	{
4861	PGM_INVL_ALL_VCPU_TLBS(pVM);
4862	}
4863
4864	PGM_UNLOCK(pVM);
4865	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
4866	return rc;
4867	}
4868
4869
4870	/**
4871	* Frees a usage of a pool page.
4872	*
4873	* The caller is responsible to updating the user table so that it no longer
4874	* references the shadow page.
4875	*
4876	* @param pPool The pool.
4877	* @param pPage The shadow page.
4878	* @param iUser The shadow page pool index of the user table.
4879	* NIL_PGMPOOL_IDX for root pages.
4880	* @param iUserTable The index into the user table (shadowed). Ignored if
4881	* root page.
4882	*/
4883	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
4884	{
4885	PVMCC pVM = pPool->CTX_SUFF(pVM);
4886
4887	STAM_PROFILE_START(&pPool->StatFree, a);
4888	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
4889	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
4890	AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
4891
4892	PGM_LOCK_VOID(pVM);
4893	if (iUser != NIL_PGMPOOL_IDX)
4894	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
4895	if (!pPage->fCached)
4896	pgmPoolFlushPage(pPool, pPage);
4897	PGM_UNLOCK(pVM);
4898	STAM_PROFILE_STOP(&pPool->StatFree, a);
4899	}
4900
4901
4902	/**
4903	* Makes one or more free page free.
4904	*
4905	* @returns VBox status code.
4906	* @retval VINF_SUCCESS on success.
4907	*
4908	* @param pPool The pool.
4909	* @param enmKind Page table kind
4910	* @param iUser The user of the page.
4911	*/
4912	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
4913	{
4914	PVMCC pVM = pPool->CTX_SUFF(pVM);
4915	LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
4916	NOREF(enmKind);
4917
4918	/*
4919	* If the pool isn't full grown yet, expand it.
4920	*/
4921	if (pPool->cCurPages < pPool->cMaxPages)
4922	{
4923	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
4924	#ifdef IN_RING3
4925	int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM));
4926	#else
4927	int rc = PGMR0PoolGrow(pVM, VMMGetCpuId(pVM));
4928	#endif
4929	if (RT_FAILURE(rc))
4930	return rc;
4931	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
4932	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
4933	return VINF_SUCCESS;
4934	}
4935
4936	/*
4937	* Free one cached page.
4938	*/
4939	return pgmPoolCacheFreeOne(pPool, iUser);
4940	}
4941
4942
4943	/**
4944	* Allocates a page from the pool.
4945	*
4946	* This page may actually be a cached page and not in need of any processing
4947	* on the callers part.
4948	*
4949	* @returns VBox status code.
4950	* @retval VINF_SUCCESS if a NEW page was allocated.
4951	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4952	*
4953	* @param pVM The cross context VM structure.
4954	* @param GCPhys The GC physical address of the page we're gonna shadow.
4955	* For 4MB and 2MB PD entries, it's the first address the
4956	* shadow PT is covering.
4957	* @param enmKind The kind of mapping.
4958	* @param enmAccess Access type for the mapping (only relevant for big pages)
4959	* @param fA20Enabled Whether the A20 gate is enabled or not.
4960	* @param iUser The shadow page pool index of the user table. Root
4961	* pages should pass NIL_PGMPOOL_IDX.
4962	* @param iUserTable The index into the user table (shadowed). Ignored for
4963	* root pages (iUser == NIL_PGMPOOL_IDX).
4964	* @param fLockPage Lock the page
4965	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4966	*/
4967	int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
4968	uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
4969	{
4970	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4971	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4972	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
4973	*ppPage = NULL;
4974	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4975	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
4976	* Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
4977
4978	PGM_LOCK_VOID(pVM);
4979
4980	if (pPool->fCacheEnabled)
4981	{
4982	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
4983	if (RT_SUCCESS(rc2))
4984	{
4985	if (fLockPage)
4986	pgmPoolLockPage(pPool, *ppPage);
4987	PGM_UNLOCK(pVM);
4988	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4989	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
4990	return rc2;
4991	}
4992	}
4993
4994	/*
4995	* Allocate a new one.
4996	*/
4997	int rc = VINF_SUCCESS;
4998	uint16_t iNew = pPool->iFreeHead;
4999	if (iNew == NIL_PGMPOOL_IDX)
5000	{
5001	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
5002	if (RT_FAILURE(rc))
5003	{
5004	PGM_UNLOCK(pVM);
5005	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
5006	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5007	return rc;
5008	}
5009	iNew = pPool->iFreeHead;
5010	AssertReleaseMsgReturn(iNew != NIL_PGMPOOL_IDX, ("iNew=%#x\n", iNew), VERR_PGM_POOL_IPE);
5011	}
5012
5013	/* unlink the free head */
5014	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
5015	pPool->iFreeHead = pPage->iNext;
5016	pPage->iNext = NIL_PGMPOOL_IDX;
5017
5018	/*
5019	* Initialize it.
5020	*/
5021	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
5022	pPage->enmKind = enmKind;
5023	pPage->enmAccess = enmAccess;
5024	pPage->GCPhys = GCPhys;
5025	pPage->fA20Enabled = fA20Enabled;
5026	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
5027	pPage->fMonitored = false;
5028	pPage->fCached = false;
5029	pPage->fDirty = false;
5030	pPage->fReusedFlushPending = false;
5031	pPage->cModifications = 0;
5032	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5033	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5034	pPage->cPresent = 0;
5035	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5036	pPage->idxDirtyEntry = 0;
5037	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5038	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5039	pPage->cLastAccessHandler = 0;
5040	pPage->cLocked = 0;
5041	# ifdef VBOX_STRICT
5042	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5043	# endif
5044
5045	/*
5046	* Insert into the tracking and cache. If this fails, free the page.
5047	*/
5048	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
5049	if (RT_FAILURE(rc3))
5050	{
5051	pPool->cUsedPages--;
5052	pPage->enmKind = PGMPOOLKIND_FREE;
5053	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5054	pPage->GCPhys = NIL_RTGCPHYS;
5055	pPage->iNext = pPool->iFreeHead;
5056	pPool->iFreeHead = pPage->idx;
5057	PGM_UNLOCK(pVM);
5058	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5059	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
5060	return rc3;
5061	}
5062
5063	/*
5064	* Commit the allocation, clear the page and return.
5065	*/
5066	#ifdef VBOX_WITH_STATISTICS
5067	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
5068	pPool->cUsedPagesHigh = pPool->cUsedPages;
5069	#endif
5070
5071	if (!pPage->fZeroed)
5072	{
5073	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5074	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
5075	ASMMemZeroPage(pv);
5076	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5077	}
5078
5079	*ppPage = pPage;
5080	if (fLockPage)
5081	pgmPoolLockPage(pPool, pPage);
5082	PGM_UNLOCK(pVM);
5083	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
5084	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
5085	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5086	return rc;
5087	}
5088
5089
5090	/**
5091	* Frees a usage of a pool page.
5092	*
5093	* @param pVM The cross context VM structure.
5094	* @param HCPhys The HC physical address of the shadow page.
5095	* @param iUser The shadow page pool index of the user table.
5096	* NIL_PGMPOOL_IDX if root page.
5097	* @param iUserTable The index into the user table (shadowed). Ignored if
5098	* root page.
5099	*/
5100	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
5101	{
5102	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
5103	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5104	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
5105	}
5106
5107
5108	/**
5109	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
5110	*
5111	* @returns Pointer to the shadow page structure.
5112	* @param pPool The pool.
5113	* @param HCPhys The HC physical address of the shadow page.
5114	*/
5115	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
5116	{
5117	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5118
5119	/*
5120	* Look up the page.
5121	*/
5122	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5123
5124	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
5125	return pPage;
5126	}
5127
5128
5129	/**
5130	* Internal worker for finding a page for debugging purposes, no assertions.
5131	*
5132	* @returns Pointer to the shadow page structure. NULL on if not found.
5133	* @param pPool The pool.
5134	* @param HCPhys The HC physical address of the shadow page.
5135	*/
5136	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
5137	{
5138	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5139	return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5140	}
5141
5142
5143	/**
5144	* Internal worker for PGM_HCPHYS_2_PTR.
5145	*
5146	* @returns VBox status code.
5147	* @param pVM The cross context VM structure.
5148	* @param HCPhys The HC physical address of the shadow page.
5149	* @param ppv Where to return the address.
5150	*/
5151	int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv)
5152	{
5153	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5154	AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE,
5155	("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0),
5156	VERR_PGM_POOL_GET_PAGE_FAILED);
5157	ppv = (uint8_t )pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK);
5158	return VINF_SUCCESS;
5159	}
5160
5161	#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
5162
5163	/**
5164	* Flush the specified page if present
5165	*
5166	* @param pVM The cross context VM structure.
5167	* @param GCPhys Guest physical address of the page to flush
5168	*/
5169	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
5170	{
5171	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5172
5173	VM_ASSERT_EMT(pVM);
5174
5175	/*
5176	* Look up the GCPhys in the hash.
5177	*/
5178	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
5179	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
5180	if (i == NIL_PGMPOOL_IDX)
5181	return;
5182
5183	do
5184	{
5185	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5186	if (pPage->GCPhys - GCPhys < PAGE_SIZE)
5187	{
5188	switch (pPage->enmKind)
5189	{
5190	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5191	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5192	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5193	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5194	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5195	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5196	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5197	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5198	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5199	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5200	case PGMPOOLKIND_64BIT_PML4:
5201	case PGMPOOLKIND_32BIT_PD:
5202	case PGMPOOLKIND_PAE_PDPT:
5203	{
5204	Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
5205	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5206	if (pPage->fDirty)
5207	STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
5208	else
5209	# endif
5210	STAM_COUNTER_INC(&pPool->StatForceFlushPage);
5211	Assert(!pgmPoolIsPageLocked(pPage));
5212	pgmPoolMonitorChainFlush(pPool, pPage);
5213	return;
5214	}
5215
5216	/* ignore, no monitoring. */
5217	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5218	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5219	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5220	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5221	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5222	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5223	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5224	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5225	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5226	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5227	case PGMPOOLKIND_ROOT_NESTED:
5228	case PGMPOOLKIND_PAE_PD_PHYS:
5229	case PGMPOOLKIND_PAE_PDPT_PHYS:
5230	case PGMPOOLKIND_32BIT_PD_PHYS:
5231	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5232	break;
5233
5234	default:
5235	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
5236	}
5237	}
5238
5239	/* next */
5240	i = pPage->iNext;
5241	} while (i != NIL_PGMPOOL_IDX);
5242	return;
5243	}
5244
5245
5246	/**
5247	* Reset CPU on hot plugging.
5248	*
5249	* @param pVM The cross context VM structure.
5250	* @param pVCpu The cross context virtual CPU structure.
5251	*/
5252	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
5253	{
5254	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5255
5256	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5257	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5258	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5259	}
5260
5261
5262	/**
5263	* Flushes the entire cache.
5264	*
5265	* It will assert a global CR3 flush (FF) and assumes the caller is aware of
5266	* this and execute this CR3 flush.
5267	*
5268	* @param pVM The cross context VM structure.
5269	*/
5270	void pgmR3PoolReset(PVM pVM)
5271	{
5272	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5273
5274	PGM_LOCK_ASSERT_OWNER(pVM);
5275	STAM_PROFILE_START(&pPool->StatR3Reset, a);
5276	LogFlow(("pgmR3PoolReset:\n"));
5277
5278	/*
5279	* If there are no pages in the pool, there is nothing to do.
5280	*/
5281	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
5282	{
5283	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5284	return;
5285	}
5286
5287	/*
5288	* Exit the shadow mode since we're going to clear everything,
5289	* including the root page.
5290	*/
5291	VMCC_FOR_EACH_VMCPU(pVM)
5292	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5293	VMCC_FOR_EACH_VMCPU_END(pVM);
5294
5295
5296	/*
5297	* Nuke the free list and reinsert all pages into it.
5298	*/
5299	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
5300	{
5301	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5302
5303	if (pPage->fMonitored)
5304	pgmPoolMonitorFlush(pPool, pPage);
5305	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5306	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5307	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
5308	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
5309	pPage->GCPhys = NIL_RTGCPHYS;
5310	pPage->enmKind = PGMPOOLKIND_FREE;
5311	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5312	Assert(pPage->idx == i);
5313	pPage->iNext = i + 1;
5314	pPage->fA20Enabled = true;
5315	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
5316	pPage->fSeenNonGlobal = false;
5317	pPage->fMonitored = false;
5318	pPage->fDirty = false;
5319	pPage->fCached = false;
5320	pPage->fReusedFlushPending = false;
5321	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
5322	pPage->cPresent = 0;
5323	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5324	pPage->cModifications = 0;
5325	pPage->iAgeNext = NIL_PGMPOOL_IDX;
5326	pPage->iAgePrev = NIL_PGMPOOL_IDX;
5327	pPage->idxDirtyEntry = 0;
5328	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5329	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5330	pPage->cLastAccessHandler = 0;
5331	pPage->cLocked = 0;
5332	# ifdef VBOX_STRICT
5333	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5334	# endif
5335	}
5336	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
5337	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
5338	pPool->cUsedPages = 0;
5339
5340	/*
5341	* Zap and reinitialize the user records.
5342	*/
5343	pPool->cPresent = 0;
5344	pPool->iUserFreeHead = 0;
5345	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
5346	const unsigned cMaxUsers = pPool->cMaxUsers;
5347	for (unsigned i = 0; i < cMaxUsers; i++)
5348	{
5349	paUsers[i].iNext = i + 1;
5350	paUsers[i].iUser = NIL_PGMPOOL_IDX;
5351	paUsers[i].iUserTable = 0xfffffffe;
5352	}
5353	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
5354
5355	/*
5356	* Clear all the GCPhys links and rebuild the phys ext free list.
5357	*/
5358	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
5359	pRam;
5360	pRam = pRam->CTX_SUFF(pNext))
5361	{
5362	unsigned iPage = pRam->cb >> PAGE_SHIFT;
5363	while (iPage-- > 0)
5364	PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
5365	}
5366
5367	pPool->iPhysExtFreeHead = 0;
5368	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
5369	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
5370	for (unsigned i = 0; i < cMaxPhysExts; i++)
5371	{
5372	paPhysExts[i].iNext = i + 1;
5373	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
5374	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5375	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
5376	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5377	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
5378	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5379	}
5380	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
5381
5382	/*
5383	* Just zap the modified list.
5384	*/
5385	pPool->cModifiedPages = 0;
5386	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
5387
5388	/*
5389	* Clear the GCPhys hash and the age list.
5390	*/
5391	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
5392	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
5393	pPool->iAgeHead = NIL_PGMPOOL_IDX;
5394	pPool->iAgeTail = NIL_PGMPOOL_IDX;
5395
5396	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5397	/* Clear all dirty pages. */
5398	pPool->idxFreeDirtyPage = 0;
5399	pPool->cDirtyPages = 0;
5400	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++)
5401	pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
5402	# endif
5403
5404	/*
5405	* Reinsert active pages into the hash and ensure monitoring chains are correct.
5406	*/
5407	VMCC_FOR_EACH_VMCPU(pVM)
5408	{
5409	/*
5410	* Re-enter the shadowing mode and assert Sync CR3 FF.
5411	*/
5412	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5413	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5414	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5415	}
5416	VMCC_FOR_EACH_VMCPU_END(pVM);
5417
5418	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5419	}
5420
5421	#endif /* IN_RING3 */
5422
5423	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
5424	/**
5425	* Stringifies a PGMPOOLKIND value.
5426	*/
5427	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
5428	{
5429	switch ((PGMPOOLKIND)enmKind)
5430	{
5431	case PGMPOOLKIND_INVALID:
5432	return "PGMPOOLKIND_INVALID";
5433	case PGMPOOLKIND_FREE:
5434	return "PGMPOOLKIND_FREE";
5435	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5436	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
5437	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5438	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
5439	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5440	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
5441	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5442	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
5443	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5444	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
5445	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5446	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
5447	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5448	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
5449	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5450	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
5451	case PGMPOOLKIND_32BIT_PD:
5452	return "PGMPOOLKIND_32BIT_PD";
5453	case PGMPOOLKIND_32BIT_PD_PHYS:
5454	return "PGMPOOLKIND_32BIT_PD_PHYS";
5455	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5456	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
5457	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5458	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
5459	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5460	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
5461	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5462	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
5463	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5464	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
5465	case PGMPOOLKIND_PAE_PD_PHYS:
5466	return "PGMPOOLKIND_PAE_PD_PHYS";
5467	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5468	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
5469	case PGMPOOLKIND_PAE_PDPT:
5470	return "PGMPOOLKIND_PAE_PDPT";
5471	case PGMPOOLKIND_PAE_PDPT_PHYS:
5472	return "PGMPOOLKIND_PAE_PDPT_PHYS";
5473	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5474	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
5475	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5476	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
5477	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5478	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
5479	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5480	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
5481	case PGMPOOLKIND_64BIT_PML4:
5482	return "PGMPOOLKIND_64BIT_PML4";
5483	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5484	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
5485	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5486	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
5487	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5488	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
5489	case PGMPOOLKIND_ROOT_NESTED:
5490	return "PGMPOOLKIND_ROOT_NESTED";
5491	}
5492	return "Unknown kind!";
5493	}
5494	#endif /* LOG_ENABLED \|\| VBOX_STRICT */
5495

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 92561

Download in other formats: