PGMAllPool.cpp@ 66974

Last change on this file since 66974 was 65957, checked in by vboxsync, 8 years ago
PGMPool: Address problem where we would incorrectly try to flush an active CR3 page (or pages in PAE) because we thought it was being reused, which obviously would fail, then pretend we succeeded and restart the instruction. Happend on a STOS during linux bootup (e.g. ubuntu 15.10/amd64). Fix was to extend pgmRZPoolMonitorIsReused with a check on locked pages.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 216.5 KB

Line
1	/* $Id: PGMAllPool.cpp 65957 2017-03-06 20:41:43Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_POOL
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/mm.h>
25	#include <VBox/vmm/em.h>
26	#include <VBox/vmm/cpum.h>
27	#ifdef IN_RC
28	# include <VBox/vmm/patm.h>
29	#endif
30	#include "PGMInternal.h"
31	#include <VBox/vmm/vm.h>
32	#include "PGMInline.h"
33	#include <VBox/disopcode.h>
34	#include <VBox/vmm/hm_vmx.h>
35
36	#include <VBox/log.h>
37	#include <VBox/err.h>
38	#include <iprt/asm.h>
39	#include <iprt/asm-amd64-x86.h>
40	#include <iprt/string.h>
41
42
43	/*********************************************************************************************************************************
44	* Internal Functions *
45	*********************************************************************************************************************************/
46	RT_C_DECLS_BEGIN
47	#if 0 /* unused */
48	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
49	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
50	#endif /* unused */
51	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
52	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
53	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
54	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
55	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
56	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
57	#endif
58	#if 0 /defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)/
59	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
60	#endif
61
62	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage);
63	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
64	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
65	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
66
67	RT_C_DECLS_END
68
69
70	#if 0 /* unused */
71	/**
72	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
73	*
74	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
75	* @param enmKind The page kind.
76	*/
77	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
78	{
79	switch (enmKind)
80	{
81	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
82	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
83	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
84	return true;
85	default:
86	return false;
87	}
88	}
89	#endif /* unused */
90
91
92	/**
93	* Flushes a chain of pages sharing the same access monitor.
94	*
95	* @returns VBox status code suitable for scheduling.
96	* @param pPool The pool.
97	* @param pPage A page in the chain.
98	* @todo VBOXSTRICTRC
99	*/
100	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
101	{
102	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
103
104	/*
105	* Find the list head.
106	*/
107	uint16_t idx = pPage->idx;
108	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
109	{
110	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
111	{
112	idx = pPage->iMonitoredPrev;
113	Assert(idx != pPage->idx);
114	pPage = &pPool->aPages[idx];
115	}
116	}
117
118	/*
119	* Iterate the list flushing each shadow page.
120	*/
121	int rc = VINF_SUCCESS;
122	for (;;)
123	{
124	idx = pPage->iMonitoredNext;
125	Assert(idx != pPage->idx);
126	if (pPage->idx >= PGMPOOL_IDX_FIRST)
127	{
128	int rc2 = pgmPoolFlushPage(pPool, pPage);
129	AssertRC(rc2);
130	}
131	/* next */
132	if (idx == NIL_PGMPOOL_IDX)
133	break;
134	pPage = &pPool->aPages[idx];
135	}
136	return rc;
137	}
138
139
140	/**
141	* Wrapper for getting the current context pointer to the entry being modified.
142	*
143	* @returns VBox status code suitable for scheduling.
144	* @param pVM The cross context VM structure.
145	* @param pvDst Destination address
146	* @param pvSrc Pointer to the mapping of @a GCPhysSrc or NULL depending
147	* on the context (e.g. \#PF in R0 & RC).
148	* @param GCPhysSrc The source guest physical address.
149	* @param cb Size of data to read
150	*/
151	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVM pVM, void pvDst, void const pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
152	{
153	#if defined(IN_RING3)
154	NOREF(pVM); NOREF(GCPhysSrc);
155	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
156	return VINF_SUCCESS;
157	#else
158	/** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
159	NOREF(pvSrc);
160	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
161	#endif
162	}
163
164
165	/**
166	* Process shadow entries before they are changed by the guest.
167	*
168	* For PT entries we will clear them. For PD entries, we'll simply check
169	* for mapping conflicts and set the SyncCR3 FF if found.
170	*
171	* @param pVCpu The cross context virtual CPU structure.
172	* @param pPool The pool.
173	* @param pPage The head page.
174	* @param GCPhysFault The guest physical fault address.
175	* @param pvAddress Pointer to the mapping of @a GCPhysFault or NULL
176	* depending on the context (e.g. \#PF in R0 & RC).
177	* @param cbWrite Write size; might be zero if the caller knows we're not crossing entry boundaries
178	*/
179	static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
180	void const *pvAddress, unsigned cbWrite)
181	{
182	AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
183	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
184	PVM pVM = pPool->CTX_SUFF(pVM);
185	NOREF(pVCpu);
186
187	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
188	(RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
189
190	for (;;)
191	{
192	union
193	{
194	void *pv;
195	PX86PT pPT;
196	PPGMSHWPTPAE pPTPae;
197	PX86PD pPD;
198	PX86PDPAE pPDPae;
199	PX86PDPT pPDPT;
200	PX86PML4 pPML4;
201	} uShw;
202
203	LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
204	pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
205
206	uShw.pv = NULL;
207	switch (pPage->enmKind)
208	{
209	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
210	{
211	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
212	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
213	const unsigned iShw = off / sizeof(X86PTE);
214	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
215	if (uShw.pPT->a[iShw].n.u1Present)
216	{
217	X86PTE GstPte;
218
219	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
220	AssertRC(rc);
221	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
222	pgmPoolTracDerefGCPhysHint(pPool, pPage,
223	uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
224	GstPte.u & X86_PTE_PG_MASK,
225	iShw);
226	ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
227	}
228	break;
229	}
230
231	/* page/2 sized */
232	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
233	{
234	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
235	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
236	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
237	{
238	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
239	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
240	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
241	{
242	X86PTE GstPte;
243	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
244	AssertRC(rc);
245
246	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
247	pgmPoolTracDerefGCPhysHint(pPool, pPage,
248	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
249	GstPte.u & X86_PTE_PG_MASK,
250	iShw);
251	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
252	}
253	}
254	break;
255	}
256
257	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
258	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
259	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
260	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
261	{
262	unsigned iGst = off / sizeof(X86PDE);
263	unsigned iShwPdpt = iGst / 256;
264	unsigned iShw = (iGst % 256) * 2;
265	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
266
267	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
268	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
269	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
270	{
271	for (unsigned i = 0; i < 2; i++)
272	{
273	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
274	if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
275	{
276	Assert(pgmMapAreMappingsEnabled(pVM));
277	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
278	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
279	break;
280	}
281	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
282	if (uShw.pPDPae->a[iShw+i].n.u1Present)
283	{
284	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
285	pgmPoolFree(pVM,
286	uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
287	pPage->idx,
288	iShw + i);
289	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw+i].u, 0);
290	}
291
292	/* paranoia / a bit assumptive. */
293	if ( (off & 3)
294	&& (off & 3) + cbWrite > 4)
295	{
296	const unsigned iShw2 = iShw + 2 + i;
297	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
298	{
299	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
300	if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
301	{
302	Assert(pgmMapAreMappingsEnabled(pVM));
303	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
304	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
305	break;
306	}
307	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
308	if (uShw.pPDPae->a[iShw2].n.u1Present)
309	{
310	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
311	pgmPoolFree(pVM,
312	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
313	pPage->idx,
314	iShw2);
315	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
316	}
317	}
318	}
319	}
320	}
321	break;
322	}
323
324	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
325	{
326	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
327	const unsigned iShw = off / sizeof(X86PTEPAE);
328	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
329	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
330	{
331	X86PTEPAE GstPte;
332	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
333	AssertRC(rc);
334
335	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
336	pgmPoolTracDerefGCPhysHint(pPool, pPage,
337	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
338	GstPte.u & X86_PTE_PAE_PG_MASK,
339	iShw);
340	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
341	}
342
343	/* paranoia / a bit assumptive. */
344	if ( (off & 7)
345	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
346	{
347	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
348	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
349
350	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
351	{
352	X86PTEPAE GstPte;
353	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
354	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
355	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
356	AssertRC(rc);
357	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
358	pgmPoolTracDerefGCPhysHint(pPool, pPage,
359	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
360	GstPte.u & X86_PTE_PAE_PG_MASK,
361	iShw2);
362	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
363	}
364	}
365	break;
366	}
367
368	case PGMPOOLKIND_32BIT_PD:
369	{
370	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
371	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
372
373	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
374	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
375	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
376	if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
377	{
378	Assert(pgmMapAreMappingsEnabled(pVM));
379	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
380	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
381	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
382	break;
383	}
384	else
385	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
386	{
387	if (uShw.pPD->a[iShw].n.u1Present)
388	{
389	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
390	pgmPoolFree(pVM,
391	uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
392	pPage->idx,
393	iShw);
394	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
395	}
396	}
397	/* paranoia / a bit assumptive. */
398	if ( (off & 3)
399	&& (off & 3) + cbWrite > sizeof(X86PTE))
400	{
401	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
402	if ( iShw2 != iShw
403	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
404	{
405	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
406	if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
407	{
408	Assert(pgmMapAreMappingsEnabled(pVM));
409	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
410	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
411	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
412	break;
413	}
414	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
415	if (uShw.pPD->a[iShw2].n.u1Present)
416	{
417	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
418	pgmPoolFree(pVM,
419	uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
420	pPage->idx,
421	iShw2);
422	ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
423	}
424	}
425	}
426	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
427	if ( uShw.pPD->a[iShw].n.u1Present
428	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
429	{
430	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
431	# ifdef IN_RC /* TLB load - we're pushing things a bit... */
432	ASMProbeReadByte(pvAddress);
433	# endif
434	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
435	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
436	}
437	#endif
438	break;
439	}
440
441	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
442	{
443	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
444	const unsigned iShw = off / sizeof(X86PDEPAE);
445	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
446	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
447	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
448	{
449	Assert(pgmMapAreMappingsEnabled(pVM));
450	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
451	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
452	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
453	break;
454	}
455	#endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
456	/*
457	* Causes trouble when the guest uses a PDE to refer to the whole page table level
458	* structure. (Invalidate here; faults later on when it tries to change the page
459	* table entries -> recheck; probably only applies to the RC case.)
460	*/
461	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
462	else
463	#endif
464	{
465	if (uShw.pPDPae->a[iShw].n.u1Present)
466	{
467	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
468	pgmPoolFree(pVM,
469	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
470	pPage->idx,
471	iShw);
472	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
473	}
474	}
475	/* paranoia / a bit assumptive. */
476	if ( (off & 7)
477	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
478	{
479	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
480	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
481
482	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
483	if ( iShw2 != iShw
484	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
485	{
486	Assert(pgmMapAreMappingsEnabled(pVM));
487	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
488	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
489	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
490	break;
491	}
492	else
493	#endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
494	if (uShw.pPDPae->a[iShw2].n.u1Present)
495	{
496	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
497	pgmPoolFree(pVM,
498	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
499	pPage->idx,
500	iShw2);
501	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
502	}
503	}
504	break;
505	}
506
507	case PGMPOOLKIND_PAE_PDPT:
508	{
509	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
510	/*
511	* Hopefully this doesn't happen very often:
512	* - touching unused parts of the page
513	* - messing with the bits of pd pointers without changing the physical address
514	*/
515	/* PDPT roots are not page aligned; 32 byte only! */
516	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
517
518	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
519	const unsigned iShw = offPdpt / sizeof(X86PDPE);
520	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
521	{
522	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
523	if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
524	{
525	Assert(pgmMapAreMappingsEnabled(pVM));
526	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
527	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
528	LogFlow(("pgmPoolMonitorChainChanging: Detected pdpt conflict at iShw=%#x!\n", iShw));
529	break;
530	}
531	else
532	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
533	if (uShw.pPDPT->a[iShw].n.u1Present)
534	{
535	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
536	pgmPoolFree(pVM,
537	uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
538	pPage->idx,
539	iShw);
540	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
541	}
542
543	/* paranoia / a bit assumptive. */
544	if ( (offPdpt & 7)
545	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
546	{
547	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
548	if ( iShw2 != iShw
549	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
550	{
551	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
552	if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
553	{
554	Assert(pgmMapAreMappingsEnabled(pVM));
555	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
556	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
557	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
558	break;
559	}
560	else
561	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
562	if (uShw.pPDPT->a[iShw2].n.u1Present)
563	{
564	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
565	pgmPoolFree(pVM,
566	uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
567	pPage->idx,
568	iShw2);
569	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
570	}
571	}
572	}
573	}
574	break;
575	}
576
577	#ifndef IN_RC
578	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
579	{
580	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
581	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
582	const unsigned iShw = off / sizeof(X86PDEPAE);
583	Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
584	if (uShw.pPDPae->a[iShw].n.u1Present)
585	{
586	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
587	pgmPoolFree(pVM,
588	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
589	pPage->idx,
590	iShw);
591	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
592	}
593	/* paranoia / a bit assumptive. */
594	if ( (off & 7)
595	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
596	{
597	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
598	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
599
600	Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
601	if (uShw.pPDPae->a[iShw2].n.u1Present)
602	{
603	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
604	pgmPoolFree(pVM,
605	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
606	pPage->idx,
607	iShw2);
608	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
609	}
610	}
611	break;
612	}
613
614	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
615	{
616	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
617	/*
618	* Hopefully this doesn't happen very often:
619	* - messing with the bits of pd pointers without changing the physical address
620	*/
621	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
622	const unsigned iShw = off / sizeof(X86PDPE);
623	if (uShw.pPDPT->a[iShw].n.u1Present)
624	{
625	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
626	pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
627	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
628	}
629	/* paranoia / a bit assumptive. */
630	if ( (off & 7)
631	&& (off & 7) + cbWrite > sizeof(X86PDPE))
632	{
633	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
634	if (uShw.pPDPT->a[iShw2].n.u1Present)
635	{
636	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
637	pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
638	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
639	}
640	}
641	break;
642	}
643
644	case PGMPOOLKIND_64BIT_PML4:
645	{
646	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
647	/*
648	* Hopefully this doesn't happen very often:
649	* - messing with the bits of pd pointers without changing the physical address
650	*/
651	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
652	const unsigned iShw = off / sizeof(X86PDPE);
653	if (uShw.pPML4->a[iShw].n.u1Present)
654	{
655	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
656	pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
657	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
658	}
659	/* paranoia / a bit assumptive. */
660	if ( (off & 7)
661	&& (off & 7) + cbWrite > sizeof(X86PDPE))
662	{
663	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
664	if (uShw.pPML4->a[iShw2].n.u1Present)
665	{
666	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
667	pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
668	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
669	}
670	}
671	break;
672	}
673	#endif /* IN_RING0 */
674
675	default:
676	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
677	}
678	PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
679
680	/* next */
681	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
682	return;
683	pPage = &pPool->aPages[pPage->iMonitoredNext];
684	}
685	}
686
687	#ifndef IN_RING3
688
689	/**
690	* Checks if a access could be a fork operation in progress.
691	*
692	* Meaning, that the guest is setting up the parent process for Copy-On-Write.
693	*
694	* @returns true if it's likely that we're forking, otherwise false.
695	* @param pPool The pool.
696	* @param pDis The disassembled instruction.
697	* @param offFault The access offset.
698	*/
699	DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
700	{
701	/*
702	* i386 linux is using btr to clear X86_PTE_RW.
703	* The functions involved are (2.6.16 source inspection):
704	* clear_bit
705	* ptep_set_wrprotect
706	* copy_one_pte
707	* copy_pte_range
708	* copy_pmd_range
709	* copy_pud_range
710	* copy_page_range
711	* dup_mmap
712	* dup_mm
713	* copy_mm
714	* copy_process
715	* do_fork
716	*/
717	if ( pDis->pCurInstr->uOpcode == OP_BTR
718	&& !(offFault & 4)
719	/** @todo Validate that the bit index is X86_PTE_RW. */
720	)
721	{
722	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
723	return true;
724	}
725	return false;
726	}
727
728
729	/**
730	* Determine whether the page is likely to have been reused.
731	*
732	* @returns true if we consider the page as being reused for a different purpose.
733	* @returns false if we consider it to still be a paging page.
734	* @param pVM The cross context VM structure.
735	* @param pVCpu The cross context virtual CPU structure.
736	* @param pRegFrame Trap register frame.
737	* @param pDis The disassembly info for the faulting instruction.
738	* @param pvFault The fault address.
739	* @param pPage The pool page being accessed.
740	*
741	* @remark The REP prefix check is left to the caller because of STOSD/W.
742	*/
743	DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault,
744	PPGMPOOLPAGE pPage)
745	{
746	/* Locked (CR3, PDPTR*4) should not be reusable. Considering them as
747	such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
748	if (pPage->cLocked)
749	{
750	Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
751	return false;
752	}
753
754	# ifndef IN_RC
755	/** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
756	if ( HMHasPendingIrq(pVM)
757	&& (pRegFrame->rsp - pvFault) < 32)
758	{
759	/* Fault caused by stack writes while trying to inject an interrupt event. */
760	Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
761	return true;
762	}
763	# else
764	NOREF(pVM); NOREF(pvFault);
765	# endif
766
767	LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse, pDis->Param1.Base.idxGenReg));
768
769	/* Non-supervisor mode write means it's used for something else. */
770	if (CPUMGetGuestCPL(pVCpu) == 3)
771	return true;
772
773	switch (pDis->pCurInstr->uOpcode)
774	{
775	/* call implies the actual push of the return address faulted */
776	case OP_CALL:
777	Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
778	return true;
779	case OP_PUSH:
780	Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
781	return true;
782	case OP_PUSHF:
783	Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
784	return true;
785	case OP_PUSHA:
786	Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
787	return true;
788	case OP_FXSAVE:
789	Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
790	return true;
791	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
792	Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
793	return true;
794	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
795	Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
796	return true;
797	case OP_MOVSWD:
798	case OP_STOSWD:
799	if ( pDis->fPrefix == (DISPREFIX_REP\|DISPREFIX_REX)
800	&& pRegFrame->rcx >= 0x40
801	)
802	{
803	Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
804
805	Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
806	return true;
807	}
808	break;
809
810	default:
811	/*
812	* Anything having ESP on the left side means stack writes.
813	*/
814	if ( ( (pDis->Param1.fUse & DISUSE_REG_GEN32)
815	\|\| (pDis->Param1.fUse & DISUSE_REG_GEN64))
816	&& (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
817	{
818	Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
819	return true;
820	}
821	break;
822	}
823
824	/*
825	* Page table updates are very very unlikely to be crossing page boundraries,
826	* and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
827	*/
828	uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
829	if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
830	{
831	Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
832	return true;
833	}
834
835	/*
836	* Nobody does an unaligned 8 byte write to a page table, right.
837	*/
838	if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
839	{
840	Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
841	return true;
842	}
843
844	return false;
845	}
846
847
848	/**
849	* Flushes the page being accessed.
850	*
851	* @returns VBox status code suitable for scheduling.
852	* @param pVM The cross context VM structure.
853	* @param pVCpu The cross context virtual CPU structure.
854	* @param pPool The pool.
855	* @param pPage The pool page (head).
856	* @param pDis The disassembly of the write instruction.
857	* @param pRegFrame The trap register frame.
858	* @param GCPhysFault The fault address as guest physical address.
859	* @param pvFault The fault address.
860	* @todo VBOXSTRICTRC
861	*/
862	static int pgmRZPoolAccessPfHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
863	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
864	{
865	NOREF(pVM); NOREF(GCPhysFault);
866
867	/*
868	* First, do the flushing.
869	*/
870	int rc = pgmPoolMonitorChainFlush(pPool, pPage);
871
872	/*
873	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
874	* Must do this in raw mode (!); XP boot will fail otherwise.
875	*/
876	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
877	if (rc2 == VINF_SUCCESS)
878	{ /* do nothing */ }
879	else if (rc2 == VINF_EM_RESCHEDULE)
880	{
881	if (rc == VINF_SUCCESS)
882	rc = VBOXSTRICTRC_VAL(rc2);
883	# ifndef IN_RING3
884	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
885	# endif
886	}
887	else if (rc2 == VERR_EM_INTERPRETER)
888	{
889	# ifdef IN_RC
890	if (PATMIsPatchGCAddr(pVM, pRegFrame->eip))
891	{
892	LogFlow(("pgmRZPoolAccessPfHandlerFlush: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
893	pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->eip));
894	rc = VINF_SUCCESS;
895	STAM_COUNTER_INC(&pPool->StatMonitorPfRZIntrFailPatch2);
896	}
897	else
898	# endif
899	{
900	rc = VINF_EM_RAW_EMULATE_INSTR;
901	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
902	}
903	}
904	else if (RT_FAILURE_NP(rc2))
905	rc = VBOXSTRICTRC_VAL(rc2);
906	else
907	AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
908
909	LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
910	return rc;
911	}
912
913
914	/**
915	* Handles the STOSD write accesses.
916	*
917	* @returns VBox status code suitable for scheduling.
918	* @param pVM The cross context VM structure.
919	* @param pPool The pool.
920	* @param pPage The pool page (head).
921	* @param pDis The disassembly of the write instruction.
922	* @param pRegFrame The trap register frame.
923	* @param GCPhysFault The fault address as guest physical address.
924	* @param pvFault The fault address.
925	*/
926	DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
927	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
928	{
929	unsigned uIncrement = pDis->Param1.cb;
930	NOREF(pVM);
931
932	Assert(pDis->uCpuMode == DISCPUMODE_32BIT \|\| pDis->uCpuMode == DISCPUMODE_64BIT);
933	Assert(pRegFrame->rcx <= 0x20);
934
935	# ifdef VBOX_STRICT
936	if (pDis->uOpMode == DISCPUMODE_32BIT)
937	Assert(uIncrement == 4);
938	else
939	Assert(uIncrement == 8);
940	# endif
941
942	Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
943
944	/*
945	* Increment the modification counter and insert it into the list
946	* of modified pages the first time.
947	*/
948	if (!pPage->cModifications++)
949	pgmPoolMonitorModifiedInsert(pPool, pPage);
950
951	/*
952	* Execute REP STOSD.
953	*
954	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
955	* write situation, meaning that it's safe to write here.
956	*/
957	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
958	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
959	while (pRegFrame->rcx)
960	{
961	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
962	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
963	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
964	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
965	# else
966	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
967	# endif
968	# ifdef IN_RC
969	(uint32_t )(uintptr_t)pu32 = pRegFrame->eax;
970	# else
971	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
972	# endif
973	pu32 += uIncrement;
974	GCPhysFault += uIncrement;
975	pRegFrame->rdi += uIncrement;
976	pRegFrame->rcx--;
977	}
978	pRegFrame->rip += pDis->cbInstr;
979
980	LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
981	return VINF_SUCCESS;
982	}
983
984
985	/**
986	* Handles the simple write accesses.
987	*
988	* @returns VBox status code suitable for scheduling.
989	* @param pVM The cross context VM structure.
990	* @param pVCpu The cross context virtual CPU structure.
991	* @param pPool The pool.
992	* @param pPage The pool page (head).
993	* @param pDis The disassembly of the write instruction.
994	* @param pRegFrame The trap register frame.
995	* @param GCPhysFault The fault address as guest physical address.
996	* @param pvFault The fault address.
997	* @param pfReused Reused state (in/out)
998	*/
999	DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
1000	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
1001	{
1002	Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
1003	NOREF(pVM);
1004	NOREF(pfReused); /* initialized by caller */
1005
1006	/*
1007	* Increment the modification counter and insert it into the list
1008	* of modified pages the first time.
1009	*/
1010	if (!pPage->cModifications++)
1011	pgmPoolMonitorModifiedInsert(pPool, pPage);
1012
1013	/*
1014	* Clear all the pages. ASSUMES that pvFault is readable.
1015	*/
1016	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1017	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
1018	# endif
1019
1020	uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
1021	if (cbWrite <= 8)
1022	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
1023	else if (cbWrite <= 16)
1024	{
1025	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
1026	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
1027	}
1028	else
1029	{
1030	Assert(cbWrite <= 32);
1031	for (uint32_t off = 0; off < cbWrite; off += 8)
1032	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
1033	}
1034
1035	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1036	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
1037	# endif
1038
1039	/*
1040	* Interpret the instruction.
1041	*/
1042	VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
1043	if (RT_SUCCESS(rc))
1044	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
1045	else if (rc == VERR_EM_INTERPRETER)
1046	{
1047	LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
1048	pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode));
1049	rc = VINF_EM_RAW_EMULATE_INSTR;
1050	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
1051	}
1052
1053	# if 0 /* experimental code */
1054	if (rc == VINF_SUCCESS)
1055	{
1056	switch (pPage->enmKind)
1057	{
1058	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1059	{
1060	X86PTEPAE GstPte;
1061	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
1062	AssertRC(rc);
1063
1064	/* Check the new value written by the guest. If present and with a bogus physical address, then
1065	* it's fairly safe to assume the guest is reusing the PT.
1066	*/
1067	if (GstPte.n.u1Present)
1068	{
1069	RTHCPHYS HCPhys = -1;
1070	int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
1071	if (rc != VINF_SUCCESS)
1072	{
1073	*pfReused = true;
1074	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1075	}
1076	}
1077	break;
1078	}
1079	}
1080	}
1081	# endif
1082
1083	LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
1084	return VBOXSTRICTRC_VAL(rc);
1085	}
1086
1087
1088	/**
1089	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
1090	* \#PF access handler callback for page table pages.}
1091	*
1092	* @remarks The @a pvUser argument points to the PGMPOOLPAGE.
1093	*/
1094	DECLEXPORT(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
1095	RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1096	{
1097	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
1098	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1099	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1100	unsigned cMaxModifications;
1101	bool fForcedFlush = false;
1102	NOREF(uErrorCode);
1103
1104	LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1105
1106	pgmLock(pVM);
1107	if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1108	{
1109	/* Pool page changed while we were waiting for the lock; ignore. */
1110	Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1111	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1112	pgmUnlock(pVM);
1113	return VINF_SUCCESS;
1114	}
1115	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1116	if (pPage->fDirty)
1117	{
1118	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
1119	pgmUnlock(pVM);
1120	return VINF_SUCCESS; /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
1121	}
1122	# endif
1123
1124	# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
1125	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1126	{
1127	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
1128	void *pvGst;
1129	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1130	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1131	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1132	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1133	}
1134	# endif
1135
1136	/*
1137	* Disassemble the faulting instruction.
1138	*/
1139	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
1140	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
1141	if (RT_UNLIKELY(rc != VINF_SUCCESS))
1142	{
1143	AssertMsg(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
1144	pgmUnlock(pVM);
1145	return rc;
1146	}
1147
1148	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1149
1150	/*
1151	* We should ALWAYS have the list head as user parameter. This
1152	* is because we use that page to record the changes.
1153	*/
1154	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1155
1156	# ifdef IN_RING0
1157	/* Maximum nr of modifications depends on the page type. */
1158	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1159	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1160	cMaxModifications = 4;
1161	else
1162	cMaxModifications = 24;
1163	# else
1164	cMaxModifications = 48;
1165	# endif
1166
1167	/*
1168	* Incremental page table updates should weigh more than random ones.
1169	* (Only applies when started from offset 0)
1170	*/
1171	pVCpu->pgm.s.cPoolAccessHandler++;
1172	if ( pPage->GCPtrLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1173	&& pPage->GCPtrLastAccessHandlerRip < pRegFrame->rip + 0x40
1174	&& pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb)
1175	&& pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
1176	{
1177	Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1178	Assert(pPage->cModifications < 32000);
1179	pPage->cModifications = pPage->cModifications * 2;
1180	pPage->GCPtrLastAccessHandlerFault = pvFault;
1181	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1182	if (pPage->cModifications >= cMaxModifications)
1183	{
1184	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
1185	fForcedFlush = true;
1186	}
1187	}
1188
1189	if (pPage->cModifications >= cMaxModifications)
1190	Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1191
1192	/*
1193	* Check if it's worth dealing with.
1194	*/
1195	bool fReused = false;
1196	bool fNotReusedNotForking = false;
1197	if ( ( pPage->cModifications < cMaxModifications /** @todo \#define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1198	\|\| pgmPoolIsPageLocked(pPage)
1199	)
1200	&& !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage))
1201	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1202	{
1203	/*
1204	* Simple instructions, no REP prefix.
1205	*/
1206	if (!(pDis->fPrefix & (DISPREFIX_REP \| DISPREFIX_REPNE)))
1207	{
1208	rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
1209	if (fReused)
1210	goto flushPage;
1211
1212	/* A mov instruction to change the first page table entry will be remembered so we can detect
1213	* full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1214	*/
1215	if ( rc == VINF_SUCCESS
1216	&& !pPage->cLocked /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
1217	&& pDis->pCurInstr->uOpcode == OP_MOV
1218	&& (pvFault & PAGE_OFFSET_MASK) == 0)
1219	{
1220	pPage->GCPtrLastAccessHandlerFault = pvFault;
1221	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1222	pPage->GCPtrLastAccessHandlerRip = pRegFrame->rip;
1223	/* Make sure we don't kick out a page too quickly. */
1224	if (pPage->cModifications > 8)
1225	pPage->cModifications = 2;
1226	}
1227	else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
1228	{
1229	/* ignore the 2nd write to this page table entry. */
1230	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1231	}
1232	else
1233	{
1234	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
1235	pPage->GCPtrLastAccessHandlerRip = 0;
1236	}
1237
1238	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1239	pgmUnlock(pVM);
1240	return rc;
1241	}
1242
1243	/*
1244	* Windows is frequently doing small memset() operations (netio test 4k+).
1245	* We have to deal with these or we'll kill the cache and performance.
1246	*/
1247	if ( pDis->pCurInstr->uOpcode == OP_STOSWD
1248	&& !pRegFrame->eflags.Bits.u1DF
1249	&& pDis->uOpMode == pDis->uCpuMode
1250	&& pDis->uAddrMode == pDis->uCpuMode)
1251	{
1252	bool fValidStosd = false;
1253
1254	if ( pDis->uCpuMode == DISCPUMODE_32BIT
1255	&& pDis->fPrefix == DISPREFIX_REP
1256	&& pRegFrame->ecx <= 0x20
1257	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1258	&& !((uintptr_t)pvFault & 3)
1259	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1260	)
1261	{
1262	fValidStosd = true;
1263	pRegFrame->rcx &= 0xffffffff; /* paranoia */
1264	}
1265	else
1266	if ( pDis->uCpuMode == DISCPUMODE_64BIT
1267	&& pDis->fPrefix == (DISPREFIX_REP \| DISPREFIX_REX)
1268	&& pRegFrame->rcx <= 0x20
1269	&& pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1270	&& !((uintptr_t)pvFault & 7)
1271	&& (pRegFrame->rax == 0 \|\| pRegFrame->rax == 0x80) /* the two values observed. */
1272	)
1273	{
1274	fValidStosd = true;
1275	}
1276
1277	if (fValidStosd)
1278	{
1279	rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1280	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
1281	pgmUnlock(pVM);
1282	return rc;
1283	}
1284	}
1285
1286	/* REP prefix, don't bother. */
1287	STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
1288	Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1289	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
1290	fNotReusedNotForking = true;
1291	}
1292
1293	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
1294	/* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1295	* leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1296	*/
1297	if ( pPage->cModifications >= cMaxModifications
1298	&& !fForcedFlush
1299	&& (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT \|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1300	&& ( fNotReusedNotForking
1301	\|\| ( !pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage)
1302	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1303	)
1304	)
1305	{
1306	Assert(!pgmPoolIsPageLocked(pPage));
1307	Assert(pPage->fDirty == false);
1308
1309	/* Flush any monitored duplicates as we will disable write protection. */
1310	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1311	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1312	{
1313	PPGMPOOLPAGE pPageHead = pPage;
1314
1315	/* Find the monitor head. */
1316	while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1317	pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1318
1319	while (pPageHead)
1320	{
1321	unsigned idxNext = pPageHead->iMonitoredNext;
1322
1323	if (pPageHead != pPage)
1324	{
1325	STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1326	Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1327	int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1328	AssertRC(rc2);
1329	}
1330
1331	if (idxNext == NIL_PGMPOOL_IDX)
1332	break;
1333
1334	pPageHead = &pPool->aPages[idxNext];
1335	}
1336	}
1337
1338	/* The flushing above might fail for locked pages, so double check. */
1339	if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX
1340	&& pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1341	{
1342	pgmPoolAddDirtyPage(pVM, pPool, pPage);
1343
1344	/* Temporarily allow write access to the page table again. */
1345	rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK, pPage->GCPhys & PAGE_BASE_GC_MASK);
1346	if (rc == VINF_SUCCESS)
1347	{
1348	rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
1349	AssertMsg(rc == VINF_SUCCESS
1350	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1351	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1352	\|\| rc == VERR_PAGE_NOT_PRESENT,
1353	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1354	# ifdef VBOX_STRICT
1355	pPage->GCPtrDirtyFault = pvFault;
1356	# endif
1357
1358	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
1359	pgmUnlock(pVM);
1360	return rc;
1361	}
1362	}
1363	}
1364	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1365
1366	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
1367	flushPage:
1368	/*
1369	* Not worth it, so flush it.
1370	*
1371	* If we considered it to be reused, don't go back to ring-3
1372	* to emulate failed instructions since we usually cannot
1373	* interpret then. This may be a bit risky, in which case
1374	* the reuse detection must be fixed.
1375	*/
1376	rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1377	if ( rc == VINF_EM_RAW_EMULATE_INSTR
1378	&& fReused)
1379	{
1380	/* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
1381	if (PGMShwGetPage(pVCpu, pRegFrame->rip, NULL, NULL) == VINF_SUCCESS)
1382	rc = VINF_SUCCESS; /* safe to restart the instruction. */
1383	}
1384	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
1385	pgmUnlock(pVM);
1386	return rc;
1387	}
1388
1389	#endif /* !IN_RING3 */
1390
1391	/**
1392	* @callback_method_impl{FNPGMPHYSHANDLER,
1393	* Access handler for shadowed page table pages.}
1394	*
1395	* @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
1396	*/
1397	PGM_ALL_CB2_DECL(VBOXSTRICTRC)
1398	pgmPoolAccessHandler(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
1399	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
1400	{
1401	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1402	STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
1403	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1404	LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
1405	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
1406
1407	NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
1408
1409	pgmLock(pVM);
1410
1411	#ifdef VBOX_WITH_STATISTICS
1412	/*
1413	* Collect stats on the access.
1414	*/
1415	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
1416	if (cbBuf <= 16 && cbBuf > 0)
1417	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
1418	else if (cbBuf >= 17 && cbBuf < 32)
1419	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
1420	else if (cbBuf >= 32 && cbBuf < 64)
1421	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
1422	else if (cbBuf >= 64)
1423	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
1424
1425	uint8_t cbAlign;
1426	switch (pPage->enmKind)
1427	{
1428	default:
1429	cbAlign = 7;
1430	break;
1431	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1432	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1433	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1434	case PGMPOOLKIND_32BIT_PD:
1435	case PGMPOOLKIND_32BIT_PD_PHYS:
1436	cbAlign = 3;
1437	break;
1438	}
1439	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
1440	if ((uint8_t)GCPhys & cbAlign)
1441	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
1442	#endif
1443
1444	/*
1445	* Make sure the pool page wasn't modified by a different CPU.
1446	*/
1447	if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
1448	{
1449	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1450
1451	/* The max modification count before flushing depends on the context and page type. */
1452	#ifdef IN_RING3
1453	uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
1454	#else
1455	uint16_t cMaxModifications;
1456	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1457	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1458	cMaxModifications = 4;
1459	else
1460	cMaxModifications = 24;
1461	# ifdef IN_RC
1462	cMaxModifications = 2; / traps are cheaper than exists. */
1463	# endif
1464	#endif
1465
1466	/*
1467	* We don't have to be very sophisticated about this since there are relativly few calls here.
1468	* However, we must try our best to detect any non-cpu accesses (disk / networking).
1469	*/
1470	if ( ( pPage->cModifications < cMaxModifications
1471	\|\| pgmPoolIsPageLocked(pPage) )
1472	&& enmOrigin != PGMACCESSORIGIN_DEVICE
1473	&& cbBuf <= 16)
1474	{
1475	/* Clear the shadow entry. */
1476	if (!pPage->cModifications++)
1477	pgmPoolMonitorModifiedInsert(pPool, pPage);
1478
1479	if (cbBuf <= 8)
1480	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
1481	else
1482	{
1483	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
1484	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
1485	}
1486	}
1487	else
1488	{
1489	/* ASSUME that VERR_PGM_POOL_CLEARED can be ignored here and that FFs will deal with it in due time. */
1490	pgmPoolMonitorChainFlush(pPool, pPage);
1491	}
1492
1493	STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1494	}
1495	else
1496	Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1497	pgmUnlock(pVM);
1498	return VINF_PGM_HANDLER_DO_DEFAULT;
1499	}
1500
1501
1502	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1503
1504	# if defined(VBOX_STRICT) && !defined(IN_RING3)
1505
1506	/**
1507	* Check references to guest physical memory in a PAE / PAE page table.
1508	*
1509	* @param pPool The pool.
1510	* @param pPage The page.
1511	* @param pShwPT The shadow page table (mapping of the page).
1512	* @param pGstPT The guest page table.
1513	*/
1514	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
1515	{
1516	unsigned cErrors = 0;
1517	int LastRc = -1; /* initialized to shut up gcc */
1518	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1519	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1520	PVM pVM = pPool->CTX_SUFF(pVM);
1521
1522	#ifdef VBOX_STRICT
1523	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1524	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1525	#endif
1526	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1527	{
1528	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1529	{
1530	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1531	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1532	if ( rc != VINF_SUCCESS
1533	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1534	{
1535	Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1536	LastPTE = i;
1537	LastRc = rc;
1538	LastHCPhys = HCPhys;
1539	cErrors++;
1540
1541	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1542	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1543	AssertRC(rc);
1544
1545	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1546	{
1547	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1548
1549	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1550	{
1551	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1552
1553	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1554	{
1555	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1556	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1557	{
1558	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1559	}
1560	}
1561
1562	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1563	}
1564	}
1565	}
1566	}
1567	}
1568	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1569	}
1570
1571
1572	/**
1573	* Check references to guest physical memory in a PAE / 32-bit page table.
1574	*
1575	* @param pPool The pool.
1576	* @param pPage The page.
1577	* @param pShwPT The shadow page table (mapping of the page).
1578	* @param pGstPT The guest page table.
1579	*/
1580	static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
1581	{
1582	unsigned cErrors = 0;
1583	int LastRc = -1; /* initialized to shut up gcc */
1584	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1585	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1586	PVM pVM = pPool->CTX_SUFF(pVM);
1587
1588	#ifdef VBOX_STRICT
1589	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1590	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1591	#endif
1592	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1593	{
1594	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1595	{
1596	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1597	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1598	if ( rc != VINF_SUCCESS
1599	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1600	{
1601	Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1602	LastPTE = i;
1603	LastRc = rc;
1604	LastHCPhys = HCPhys;
1605	cErrors++;
1606
1607	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1608	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1609	AssertRC(rc);
1610
1611	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1612	{
1613	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1614
1615	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1616	{
1617	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1618
1619	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1620	{
1621	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1622	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1623	{
1624	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1625	}
1626	}
1627
1628	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1629	}
1630	}
1631	}
1632	}
1633	}
1634	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1635	}
1636
1637	# endif /* VBOX_STRICT && !IN_RING3 */
1638
1639	/**
1640	* Clear references to guest physical memory in a PAE / PAE page table.
1641	*
1642	* @returns nr of changed PTEs
1643	* @param pPool The pool.
1644	* @param pPage The page.
1645	* @param pShwPT The shadow page table (mapping of the page).
1646	* @param pGstPT The guest page table.
1647	* @param pOldGstPT The old cached guest page table.
1648	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1649	* @param pfFlush Flush reused page table (out)
1650	*/
1651	DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
1652	PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1653	{
1654	unsigned cChanged = 0;
1655
1656	#ifdef VBOX_STRICT
1657	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1658	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1659	#endif
1660	*pfFlush = false;
1661
1662	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1663	{
1664	/* Check the new value written by the guest. If present and with a bogus physical address, then
1665	* it's fairly safe to assume the guest is reusing the PT.
1666	*/
1667	if ( fAllowRemoval
1668	&& pGstPT->a[i].n.u1Present)
1669	{
1670	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1671	{
1672	*pfFlush = true;
1673	return ++cChanged;
1674	}
1675	}
1676	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1677	{
1678	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1679	if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1680	{
1681	#ifdef VBOX_STRICT
1682	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1683	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1684	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1685	#endif
1686	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1687	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1688	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1689	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1690
1691	if ( uHostAttr == uGuestAttr
1692	&& fHostRW <= fGuestRW)
1693	continue;
1694	}
1695	cChanged++;
1696	/* Something was changed, so flush it. */
1697	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
1698	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1699	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
1700	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1701	}
1702	}
1703	return cChanged;
1704	}
1705
1706
1707	/**
1708	* Clear references to guest physical memory in a PAE / PAE page table.
1709	*
1710	* @returns nr of changed PTEs
1711	* @param pPool The pool.
1712	* @param pPage The page.
1713	* @param pShwPT The shadow page table (mapping of the page).
1714	* @param pGstPT The guest page table.
1715	* @param pOldGstPT The old cached guest page table.
1716	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1717	* @param pfFlush Flush reused page table (out)
1718	*/
1719	DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
1720	PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1721	{
1722	unsigned cChanged = 0;
1723
1724	#ifdef VBOX_STRICT
1725	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1726	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1727	#endif
1728	*pfFlush = false;
1729
1730	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1731	{
1732	/* Check the new value written by the guest. If present and with a bogus physical address, then
1733	* it's fairly safe to assume the guest is reusing the PT.
1734	*/
1735	if ( fAllowRemoval
1736	&& pGstPT->a[i].n.u1Present)
1737	{
1738	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK))
1739	{
1740	*pfFlush = true;
1741	return ++cChanged;
1742	}
1743	}
1744	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1745	{
1746	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1747	if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
1748	{
1749	#ifdef VBOX_STRICT
1750	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1751	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1752	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1753	#endif
1754	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1755	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1756	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1757	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1758
1759	if ( uHostAttr == uGuestAttr
1760	&& fHostRW <= fGuestRW)
1761	continue;
1762	}
1763	cChanged++;
1764	/* Something was changed, so flush it. */
1765	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
1766	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
1767	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
1768	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1769	}
1770	}
1771	return cChanged;
1772	}
1773
1774
1775	/**
1776	* Flush a dirty page
1777	*
1778	* @param pVM The cross context VM structure.
1779	* @param pPool The pool.
1780	* @param idxSlot Dirty array slot index
1781	* @param fAllowRemoval Allow a reused page table to be removed
1782	*/
1783	static void pgmPoolFlushDirtyPage(PVM pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
1784	{
1785	PPGMPOOLPAGE pPage;
1786	unsigned idxPage;
1787
1788	Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
1789	if (pPool->aDirtyPages[idxSlot].uIdx == NIL_PGMPOOL_IDX)
1790	return;
1791
1792	idxPage = pPool->aDirtyPages[idxSlot].uIdx;
1793	AssertRelease(idxPage != NIL_PGMPOOL_IDX);
1794	pPage = &pPool->aPages[idxPage];
1795	Assert(pPage->idx == idxPage);
1796	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1797
1798	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1799	Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1800
1801	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1802	PVMCPU pVCpu = VMMGetCpu(pVM);
1803	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
1804	#endif
1805
1806	/* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
1807	int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
1808	Assert(rc == VINF_SUCCESS);
1809	pPage->fDirty = false;
1810
1811	#ifdef VBOX_STRICT
1812	uint64_t fFlags = 0;
1813	RTHCPHYS HCPhys;
1814	rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
1815	AssertMsg( ( rc == VINF_SUCCESS
1816	&& (!(fFlags & X86_PTE_RW) \|\| HCPhys != pPage->Core.Key))
1817	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1818	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1819	\|\| rc == VERR_PAGE_NOT_PRESENT,
1820	("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
1821	#endif
1822
1823	/* Flush those PTEs that have changed. */
1824	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1825	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1826	void *pvGst;
1827	rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1828	bool fFlush;
1829	unsigned cChanges;
1830
1831	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1832	cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
1833	(PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1834	else
1835	cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
1836	(PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1837
1838	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1839	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1840	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1841	/* Note: we might want to consider keeping the dirty page active in case there were many changes. */
1842
1843	/* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1844	Assert(pPage->cModifications);
1845	if (cChanges < 4)
1846	pPage->cModifications = 1; /* must use > 0 here */
1847	else
1848	pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1849
1850	STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1851	if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
1852	pPool->idxFreeDirtyPage = idxSlot;
1853
1854	pPool->cDirtyPages--;
1855	pPool->aDirtyPages[idxSlot].uIdx = NIL_PGMPOOL_IDX;
1856	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1857	if (fFlush)
1858	{
1859	Assert(fAllowRemoval);
1860	Log(("Flush reused page table!\n"));
1861	pgmPoolFlushPage(pPool, pPage);
1862	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1863	}
1864	else
1865	Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
1866
1867	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1868	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
1869	#endif
1870	}
1871
1872
1873	# ifndef IN_RING3
1874	/**
1875	* Add a new dirty page
1876	*
1877	* @param pVM The cross context VM structure.
1878	* @param pPool The pool.
1879	* @param pPage The page.
1880	*/
1881	void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1882	{
1883	unsigned idxFree;
1884
1885	PGM_LOCK_ASSERT_OWNER(pVM);
1886	AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 \|\| RT_ELEMENTS(pPool->aDirtyPages) == 16);
1887	Assert(!pPage->fDirty);
1888
1889	idxFree = pPool->idxFreeDirtyPage;
1890	Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
1891	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1892
1893	if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
1894	{
1895	STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
1896	pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
1897	}
1898	Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
1899	AssertMsg(pPool->aDirtyPages[idxFree].uIdx == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1900
1901	Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1902
1903	/*
1904	* Make a copy of the guest page table as we require valid GCPhys addresses
1905	* when removing references to physical pages.
1906	* (The HCPhys linear lookup is extremely expensive!)
1907	*/
1908	void *pvGst;
1909	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1910	memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst, (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) ? PAGE_SIZE : PAGE_SIZE/2);
1911	# ifdef VBOX_STRICT
1912	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1913	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1914	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1915	else
1916	pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
1917	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1918	# endif
1919	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1920
1921	STAM_COUNTER_INC(&pPool->StatDirtyPage);
1922	pPage->fDirty = true;
1923	pPage->idxDirtyEntry = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
1924	pPool->aDirtyPages[idxFree].uIdx = pPage->idx;
1925	pPool->cDirtyPages++;
1926
1927	pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1928	if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
1929	&& pPool->aDirtyPages[pPool->idxFreeDirtyPage].uIdx != NIL_PGMPOOL_IDX)
1930	{
1931	unsigned i;
1932	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1933	{
1934	idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1935	if (pPool->aDirtyPages[idxFree].uIdx == NIL_PGMPOOL_IDX)
1936	{
1937	pPool->idxFreeDirtyPage = idxFree;
1938	break;
1939	}
1940	}
1941	Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
1942	}
1943
1944	Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) \|\| pPool->aDirtyPages[pPool->idxFreeDirtyPage].uIdx == NIL_PGMPOOL_IDX);
1945
1946	/*
1947	* Clear all references to this shadow table. See @bugref{7298}.
1948	*/
1949	pgmPoolTrackClearPageUsers(pPool, pPage);
1950	}
1951	# endif /* !IN_RING3 */
1952
1953
1954	/**
1955	* Check if the specified page is dirty (not write monitored)
1956	*
1957	* @return dirty or not
1958	* @param pVM The cross context VM structure.
1959	* @param GCPhys Guest physical address
1960	*/
1961	bool pgmPoolIsDirtyPage(PVM pVM, RTGCPHYS GCPhys)
1962	{
1963	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1964	PGM_LOCK_ASSERT_OWNER(pVM);
1965	if (!pPool->cDirtyPages)
1966	return false;
1967
1968	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1969
1970	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1971	{
1972	if (pPool->aDirtyPages[i].uIdx != NIL_PGMPOOL_IDX)
1973	{
1974	PPGMPOOLPAGE pPage;
1975	unsigned idxPage = pPool->aDirtyPages[i].uIdx;
1976
1977	pPage = &pPool->aPages[idxPage];
1978	if (pPage->GCPhys == GCPhys)
1979	return true;
1980	}
1981	}
1982	return false;
1983	}
1984
1985
1986	/**
1987	* Reset all dirty pages by reinstating page monitoring.
1988	*
1989	* @param pVM The cross context VM structure.
1990	*/
1991	void pgmPoolResetDirtyPages(PVM pVM)
1992	{
1993	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1994	PGM_LOCK_ASSERT_OWNER(pVM);
1995	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1996
1997	if (!pPool->cDirtyPages)
1998	return;
1999
2000	Log(("pgmPoolResetDirtyPages\n"));
2001	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2002	pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
2003
2004	pPool->idxFreeDirtyPage = 0;
2005	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
2006	&& pPool->aDirtyPages[pPool->idxFreeDirtyPage].uIdx != NIL_PGMPOOL_IDX)
2007	{
2008	unsigned i;
2009	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2010	{
2011	if (pPool->aDirtyPages[i].uIdx == NIL_PGMPOOL_IDX)
2012	{
2013	pPool->idxFreeDirtyPage = i;
2014	break;
2015	}
2016	}
2017	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2018	}
2019
2020	Assert(pPool->aDirtyPages[pPool->idxFreeDirtyPage].uIdx == NIL_PGMPOOL_IDX \|\| pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
2021	return;
2022	}
2023
2024
2025	/**
2026	* Invalidate the PT entry for the specified page
2027	*
2028	* @param pVM The cross context VM structure.
2029	* @param GCPtrPage Guest page to invalidate
2030	*/
2031	void pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage)
2032	{
2033	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2034	PGM_LOCK_ASSERT_OWNER(pVM);
2035	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2036
2037	if (!pPool->cDirtyPages)
2038	return;
2039
2040	Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2041	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2042	{
2043	}
2044	}
2045
2046
2047	/**
2048	* Reset all dirty pages by reinstating page monitoring.
2049	*
2050	* @param pVM The cross context VM structure.
2051	* @param GCPhysPT Physical address of the page table
2052	*/
2053	void pgmPoolInvalidateDirtyPage(PVM pVM, RTGCPHYS GCPhysPT)
2054	{
2055	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2056	PGM_LOCK_ASSERT_OWNER(pVM);
2057	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2058	unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
2059
2060	if (!pPool->cDirtyPages)
2061	return;
2062
2063	GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2064
2065	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2066	{
2067	if (pPool->aDirtyPages[i].uIdx != NIL_PGMPOOL_IDX)
2068	{
2069	unsigned idxPage = pPool->aDirtyPages[i].uIdx;
2070
2071	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
2072	if (pPage->GCPhys == GCPhysPT)
2073	{
2074	idxDirtyPage = i;
2075	break;
2076	}
2077	}
2078	}
2079
2080	if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
2081	{
2082	pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
2083	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
2084	&& pPool->aDirtyPages[pPool->idxFreeDirtyPage].uIdx != NIL_PGMPOOL_IDX)
2085	{
2086	unsigned i;
2087	for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2088	{
2089	if (pPool->aDirtyPages[i].uIdx == NIL_PGMPOOL_IDX)
2090	{
2091	pPool->idxFreeDirtyPage = i;
2092	break;
2093	}
2094	}
2095	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2096	}
2097	}
2098	}
2099
2100	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
2101
2102	/**
2103	* Inserts a page into the GCPhys hash table.
2104	*
2105	* @param pPool The pool.
2106	* @param pPage The page.
2107	*/
2108	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2109	{
2110	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
2111	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2112	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2113	pPage->iNext = pPool->aiHash[iHash];
2114	pPool->aiHash[iHash] = pPage->idx;
2115	}
2116
2117
2118	/**
2119	* Removes a page from the GCPhys hash table.
2120	*
2121	* @param pPool The pool.
2122	* @param pPage The page.
2123	*/
2124	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2125	{
2126	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
2127	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2128	if (pPool->aiHash[iHash] == pPage->idx)
2129	pPool->aiHash[iHash] = pPage->iNext;
2130	else
2131	{
2132	uint16_t iPrev = pPool->aiHash[iHash];
2133	for (;;)
2134	{
2135	const int16_t i = pPool->aPages[iPrev].iNext;
2136	if (i == pPage->idx)
2137	{
2138	pPool->aPages[iPrev].iNext = pPage->iNext;
2139	break;
2140	}
2141	if (i == NIL_PGMPOOL_IDX)
2142	{
2143	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
2144	break;
2145	}
2146	iPrev = i;
2147	}
2148	}
2149	pPage->iNext = NIL_PGMPOOL_IDX;
2150	}
2151
2152
2153	/**
2154	* Frees up one cache page.
2155	*
2156	* @returns VBox status code.
2157	* @retval VINF_SUCCESS on success.
2158	* @param pPool The pool.
2159	* @param iUser The user index.
2160	*/
2161	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
2162	{
2163	#ifndef IN_RC
2164	const PVM pVM = pPool->CTX_SUFF(pVM);
2165	#endif
2166	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
2167	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
2168
2169	/*
2170	* Select one page from the tail of the age list.
2171	*/
2172	PPGMPOOLPAGE pPage;
2173	for (unsigned iLoop = 0; ; iLoop++)
2174	{
2175	uint16_t iToFree = pPool->iAgeTail;
2176	if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
2177	iToFree = pPool->aPages[iToFree].iAgePrev;
2178	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
2179	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
2180	{
2181	uint16_t i = pPool->aPages[iToFree].iAgePrev;
2182	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
2183	{
2184	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
2185	continue;
2186	iToFree = i;
2187	break;
2188	}
2189	}
2190	*/
2191	Assert(iToFree != iUser);
2192	AssertRelease(iToFree != NIL_PGMPOOL_IDX);
2193	pPage = &pPool->aPages[iToFree];
2194
2195	/*
2196	* Reject any attempts at flushing the currently active shadow CR3 mapping.
2197	* Call pgmPoolCacheUsed to move the page to the head of the age list.
2198	*/
2199	if ( !pgmPoolIsPageLocked(pPage)
2200	&& pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
2201	break;
2202	LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
2203	pgmPoolCacheUsed(pPool, pPage);
2204	AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
2205	}
2206
2207	/*
2208	* Found a usable page, flush it and return.
2209	*/
2210	int rc = pgmPoolFlushPage(pPool, pPage);
2211	/* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
2212	/** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
2213	if (rc == VINF_SUCCESS)
2214	PGM_INVL_ALL_VCPU_TLBS(pVM);
2215	return rc;
2216	}
2217
2218
2219	/**
2220	* Checks if a kind mismatch is really a page being reused
2221	* or if it's just normal remappings.
2222	*
2223	* @returns true if reused and the cached page (enmKind1) should be flushed
2224	* @returns false if not reused.
2225	* @param enmKind1 The kind of the cached page.
2226	* @param enmKind2 The kind of the requested page.
2227	*/
2228	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
2229	{
2230	switch (enmKind1)
2231	{
2232	/*
2233	* Never reuse them. There is no remapping in non-paging mode.
2234	*/
2235	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2236	case PGMPOOLKIND_32BIT_PD_PHYS:
2237	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2238	case PGMPOOLKIND_PAE_PD_PHYS:
2239	case PGMPOOLKIND_PAE_PDPT_PHYS:
2240	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2241	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2242	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2243	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2244	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2245	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
2246	return false;
2247
2248	/*
2249	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2250	*/
2251	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2252	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2253	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2254	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2255	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2256	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2257	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2258	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2259	case PGMPOOLKIND_32BIT_PD:
2260	case PGMPOOLKIND_PAE_PDPT:
2261	switch (enmKind2)
2262	{
2263	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2264	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2265	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2266	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2267	case PGMPOOLKIND_64BIT_PML4:
2268	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2269	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2270	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2271	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2272	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2273	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2274	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2275	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2276	return true;
2277	default:
2278	return false;
2279	}
2280
2281	/*
2282	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2283	*/
2284	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2285	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2286	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2287	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2288	case PGMPOOLKIND_64BIT_PML4:
2289	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2290	switch (enmKind2)
2291	{
2292	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2293	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2294	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2295	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2296	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2297	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2298	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2299	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2300	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2301	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2302	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2303	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2304	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2305	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2306	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2307	return true;
2308	default:
2309	return false;
2310	}
2311
2312	/*
2313	* These cannot be flushed, and it's common to reuse the PDs as PTs.
2314	*/
2315	case PGMPOOLKIND_ROOT_NESTED:
2316	return false;
2317
2318	default:
2319	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
2320	}
2321	}
2322
2323
2324	/**
2325	* Attempts to satisfy a pgmPoolAlloc request from the cache.
2326	*
2327	* @returns VBox status code.
2328	* @retval VINF_PGM_CACHED_PAGE on success.
2329	* @retval VERR_FILE_NOT_FOUND if not found.
2330	* @param pPool The pool.
2331	* @param GCPhys The GC physical address of the page we're gonna shadow.
2332	* @param enmKind The kind of mapping.
2333	* @param enmAccess Access type for the mapping (only relevant for big pages)
2334	* @param fA20Enabled Whether the CPU has the A20 gate enabled.
2335	* @param iUser The shadow page pool index of the user table. This is
2336	* NIL_PGMPOOL_IDX for root pages.
2337	* @param iUserTable The index into the user table (shadowed). Ignored if
2338	* root page
2339	* @param ppPage Where to store the pointer to the page.
2340	*/
2341	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
2342	uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
2343	{
2344	/*
2345	* Look up the GCPhys in the hash.
2346	*/
2347	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2348	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
2349	if (i != NIL_PGMPOOL_IDX)
2350	{
2351	do
2352	{
2353	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2354	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
2355	if (pPage->GCPhys == GCPhys)
2356	{
2357	if ( (PGMPOOLKIND)pPage->enmKind == enmKind
2358	&& (PGMPOOLACCESS)pPage->enmAccess == enmAccess
2359	&& pPage->fA20Enabled == fA20Enabled)
2360	{
2361	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
2362	* doesn't flush it in case there are no more free use records.
2363	*/
2364	pgmPoolCacheUsed(pPool, pPage);
2365
2366	int rc = VINF_SUCCESS;
2367	if (iUser != NIL_PGMPOOL_IDX)
2368	rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
2369	if (RT_SUCCESS(rc))
2370	{
2371	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
2372	*ppPage = pPage;
2373	if (pPage->cModifications)
2374	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
2375	STAM_COUNTER_INC(&pPool->StatCacheHits);
2376	return VINF_PGM_CACHED_PAGE;
2377	}
2378	return rc;
2379	}
2380
2381	if ((PGMPOOLKIND)pPage->enmKind != enmKind)
2382	{
2383	/*
2384	* The kind is different. In some cases we should now flush the page
2385	* as it has been reused, but in most cases this is normal remapping
2386	* of PDs as PT or big pages using the GCPhys field in a slightly
2387	* different way than the other kinds.
2388	*/
2389	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
2390	{
2391	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
2392	pgmPoolFlushPage(pPool, pPage);
2393	break;
2394	}
2395	}
2396	}
2397
2398	/* next */
2399	i = pPage->iNext;
2400	} while (i != NIL_PGMPOOL_IDX);
2401	}
2402
2403	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
2404	STAM_COUNTER_INC(&pPool->StatCacheMisses);
2405	return VERR_FILE_NOT_FOUND;
2406	}
2407
2408
2409	/**
2410	* Inserts a page into the cache.
2411	*
2412	* @param pPool The pool.
2413	* @param pPage The cached page.
2414	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
2415	*/
2416	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
2417	{
2418	/*
2419	* Insert into the GCPhys hash if the page is fit for that.
2420	*/
2421	Assert(!pPage->fCached);
2422	if (fCanBeCached)
2423	{
2424	pPage->fCached = true;
2425	pgmPoolHashInsert(pPool, pPage);
2426	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2427	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2428	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
2429	}
2430	else
2431	{
2432	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2433	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2434	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
2435	}
2436
2437	/*
2438	* Insert at the head of the age list.
2439	*/
2440	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2441	pPage->iAgeNext = pPool->iAgeHead;
2442	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
2443	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
2444	else
2445	pPool->iAgeTail = pPage->idx;
2446	pPool->iAgeHead = pPage->idx;
2447	}
2448
2449
2450	/**
2451	* Flushes a cached page.
2452	*
2453	* @param pPool The pool.
2454	* @param pPage The cached page.
2455	*/
2456	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2457	{
2458	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
2459
2460	/*
2461	* Remove the page from the hash.
2462	*/
2463	if (pPage->fCached)
2464	{
2465	pPage->fCached = false;
2466	pgmPoolHashRemove(pPool, pPage);
2467	}
2468	else
2469	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2470
2471	/*
2472	* Remove it from the age list.
2473	*/
2474	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
2475	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
2476	else
2477	pPool->iAgeTail = pPage->iAgePrev;
2478	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
2479	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
2480	else
2481	pPool->iAgeHead = pPage->iAgeNext;
2482	pPage->iAgeNext = NIL_PGMPOOL_IDX;
2483	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2484	}
2485
2486
2487	/**
2488	* Looks for pages sharing the monitor.
2489	*
2490	* @returns Pointer to the head page.
2491	* @returns NULL if not found.
2492	* @param pPool The Pool
2493	* @param pNewPage The page which is going to be monitored.
2494	*/
2495	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
2496	{
2497	/*
2498	* Look up the GCPhys in the hash.
2499	*/
2500	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2501	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2502	if (i == NIL_PGMPOOL_IDX)
2503	return NULL;
2504	do
2505	{
2506	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2507	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
2508	&& pPage != pNewPage)
2509	{
2510	switch (pPage->enmKind)
2511	{
2512	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2513	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2514	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2515	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2516	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2517	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2518	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2519	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2520	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2521	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2522	case PGMPOOLKIND_64BIT_PML4:
2523	case PGMPOOLKIND_32BIT_PD:
2524	case PGMPOOLKIND_PAE_PDPT:
2525	{
2526	/* find the head */
2527	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2528	{
2529	Assert(pPage->iMonitoredPrev != pPage->idx);
2530	pPage = &pPool->aPages[pPage->iMonitoredPrev];
2531	}
2532	return pPage;
2533	}
2534
2535	/* ignore, no monitoring. */
2536	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2537	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2538	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2539	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2540	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2541	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2542	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2543	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2544	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2545	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2546	case PGMPOOLKIND_ROOT_NESTED:
2547	case PGMPOOLKIND_PAE_PD_PHYS:
2548	case PGMPOOLKIND_PAE_PDPT_PHYS:
2549	case PGMPOOLKIND_32BIT_PD_PHYS:
2550	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2551	break;
2552	default:
2553	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2554	}
2555	}
2556
2557	/* next */
2558	i = pPage->iNext;
2559	} while (i != NIL_PGMPOOL_IDX);
2560	return NULL;
2561	}
2562
2563
2564	/**
2565	* Enabled write monitoring of a guest page.
2566	*
2567	* @returns VBox status code.
2568	* @retval VINF_SUCCESS on success.
2569	* @param pPool The pool.
2570	* @param pPage The cached page.
2571	*/
2572	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2573	{
2574	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
2575
2576	/*
2577	* Filter out the relevant kinds.
2578	*/
2579	switch (pPage->enmKind)
2580	{
2581	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2582	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2583	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2584	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2585	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2586	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2587	case PGMPOOLKIND_64BIT_PML4:
2588	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2589	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2590	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2591	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2592	case PGMPOOLKIND_32BIT_PD:
2593	case PGMPOOLKIND_PAE_PDPT:
2594	break;
2595
2596	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2597	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2598	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2599	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2600	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2601	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2602	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2603	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2604	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2605	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2606	case PGMPOOLKIND_ROOT_NESTED:
2607	/* Nothing to monitor here. */
2608	return VINF_SUCCESS;
2609
2610	case PGMPOOLKIND_32BIT_PD_PHYS:
2611	case PGMPOOLKIND_PAE_PDPT_PHYS:
2612	case PGMPOOLKIND_PAE_PD_PHYS:
2613	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2614	/* Nothing to monitor here. */
2615	return VINF_SUCCESS;
2616	default:
2617	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2618	}
2619
2620	/*
2621	* Install handler.
2622	*/
2623	int rc;
2624	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2625	if (pPageHead)
2626	{
2627	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2628	Assert(pPageHead->iMonitoredPrev != pPage->idx);
2629
2630	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2631	if (pPageHead->fDirty)
2632	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
2633	#endif
2634
2635	pPage->iMonitoredPrev = pPageHead->idx;
2636	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2637	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2638	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2639	pPageHead->iMonitoredNext = pPage->idx;
2640	rc = VINF_SUCCESS;
2641	}
2642	else
2643	{
2644	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2645	PVM pVM = pPool->CTX_SUFF(pVM);
2646	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2647	rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
2648	MMHyperCCToR3(pVM, pPage), MMHyperCCToR0(pVM, pPage), MMHyperCCToRC(pVM, pPage),
2649	NIL_RTR3PTR /pszDesc/);
2650	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2651	* the heap size should suffice. */
2652	AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2653	PVMCPU pVCpu = VMMGetCpu(pVM);
2654	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
2655	}
2656	pPage->fMonitored = true;
2657	return rc;
2658	}
2659
2660
2661	/**
2662	* Disables write monitoring of a guest page.
2663	*
2664	* @returns VBox status code.
2665	* @retval VINF_SUCCESS on success.
2666	* @param pPool The pool.
2667	* @param pPage The cached page.
2668	*/
2669	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2670	{
2671	/*
2672	* Filter out the relevant kinds.
2673	*/
2674	switch (pPage->enmKind)
2675	{
2676	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2677	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2678	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2679	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2680	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2681	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2682	case PGMPOOLKIND_64BIT_PML4:
2683	case PGMPOOLKIND_32BIT_PD:
2684	case PGMPOOLKIND_PAE_PDPT:
2685	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2686	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2687	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2688	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2689	break;
2690
2691	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2692	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2693	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2694	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2695	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2696	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2697	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2698	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2699	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2700	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2701	case PGMPOOLKIND_ROOT_NESTED:
2702	case PGMPOOLKIND_PAE_PD_PHYS:
2703	case PGMPOOLKIND_PAE_PDPT_PHYS:
2704	case PGMPOOLKIND_32BIT_PD_PHYS:
2705	/* Nothing to monitor here. */
2706	Assert(!pPage->fMonitored);
2707	return VINF_SUCCESS;
2708
2709	default:
2710	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2711	}
2712	Assert(pPage->fMonitored);
2713
2714	/*
2715	* Remove the page from the monitored list or uninstall it if last.
2716	*/
2717	const PVM pVM = pPool->CTX_SUFF(pVM);
2718	int rc;
2719	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2720	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2721	{
2722	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2723	{
2724	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2725	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2726	rc = PGMHandlerPhysicalChangeUserArgs(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK, MMHyperCCToR3(pVM, pNewHead),
2727	MMHyperCCToR0(pVM, pNewHead), MMHyperCCToRC(pVM, pNewHead));
2728
2729	AssertFatalRCSuccess(rc);
2730	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2731	}
2732	else
2733	{
2734	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2735	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2736	{
2737	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2738	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2739	}
2740	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2741	rc = VINF_SUCCESS;
2742	}
2743	}
2744	else
2745	{
2746	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
2747	AssertFatalRC(rc);
2748	PVMCPU pVCpu = VMMGetCpu(pVM);
2749	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2750	("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2751	}
2752	pPage->fMonitored = false;
2753
2754	/*
2755	* Remove it from the list of modified pages (if in it).
2756	*/
2757	pgmPoolMonitorModifiedRemove(pPool, pPage);
2758
2759	return rc;
2760	}
2761
2762
2763	/**
2764	* Inserts the page into the list of modified pages.
2765	*
2766	* @param pPool The pool.
2767	* @param pPage The page.
2768	*/
2769	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2770	{
2771	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2772	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2773	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2774	&& pPool->iModifiedHead != pPage->idx,
2775	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2776	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2777	pPool->iModifiedHead, pPool->cModifiedPages));
2778
2779	pPage->iModifiedNext = pPool->iModifiedHead;
2780	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2781	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2782	pPool->iModifiedHead = pPage->idx;
2783	pPool->cModifiedPages++;
2784	#ifdef VBOX_WITH_STATISTICS
2785	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2786	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2787	#endif
2788	}
2789
2790
2791	/**
2792	* Removes the page from the list of modified pages and resets the
2793	* modification counter.
2794	*
2795	* @param pPool The pool.
2796	* @param pPage The page which is believed to be in the list of modified pages.
2797	*/
2798	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2799	{
2800	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2801	if (pPool->iModifiedHead == pPage->idx)
2802	{
2803	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2804	pPool->iModifiedHead = pPage->iModifiedNext;
2805	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2806	{
2807	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2808	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2809	}
2810	pPool->cModifiedPages--;
2811	}
2812	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2813	{
2814	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2815	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2816	{
2817	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2818	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2819	}
2820	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2821	pPool->cModifiedPages--;
2822	}
2823	else
2824	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2825	pPage->cModifications = 0;
2826	}
2827
2828
2829	/**
2830	* Zaps the list of modified pages, resetting their modification counters in the process.
2831	*
2832	* @param pVM The cross context VM structure.
2833	*/
2834	static void pgmPoolMonitorModifiedClearAll(PVM pVM)
2835	{
2836	pgmLock(pVM);
2837	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2838	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2839
2840	unsigned cPages = 0; NOREF(cPages);
2841
2842	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2843	pgmPoolResetDirtyPages(pVM);
2844	#endif
2845
2846	uint16_t idx = pPool->iModifiedHead;
2847	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2848	while (idx != NIL_PGMPOOL_IDX)
2849	{
2850	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2851	idx = pPage->iModifiedNext;
2852	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2853	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2854	pPage->cModifications = 0;
2855	Assert(++cPages);
2856	}
2857	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2858	pPool->cModifiedPages = 0;
2859	pgmUnlock(pVM);
2860	}
2861
2862
2863	/**
2864	* Handle SyncCR3 pool tasks
2865	*
2866	* @returns VBox status code.
2867	* @retval VINF_SUCCESS if successfully added.
2868	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2869	* @param pVCpu The cross context virtual CPU structure.
2870	* @remark Should only be used when monitoring is available, thus placed in
2871	* the PGMPOOL_WITH_MONITORING \#ifdef.
2872	*/
2873	int pgmPoolSyncCR3(PVMCPU pVCpu)
2874	{
2875	PVM pVM = pVCpu->CTX_SUFF(pVM);
2876	LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
2877
2878	/*
2879	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2880	* Occasionally we will have to clear all the shadow page tables because we wanted
2881	* to monitor a page which was mapped by too many shadowed page tables. This operation
2882	* sometimes referred to as a 'lightweight flush'.
2883	*/
2884	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2885	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2886	pgmR3PoolClearAll(pVM, false /fFlushRemTlb/);
2887	# else /* !IN_RING3 */
2888	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2889	{
2890	Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2891	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2892
2893	/* Make sure all other VCPUs return to ring 3. */
2894	if (pVM->cCpus > 1)
2895	{
2896	VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
2897	PGM_INVL_ALL_VCPU_TLBS(pVM);
2898	}
2899	return VINF_PGM_SYNC_CR3;
2900	}
2901	# endif /* !IN_RING3 */
2902	else
2903	{
2904	pgmPoolMonitorModifiedClearAll(pVM);
2905
2906	/* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
2907	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2908	{
2909	Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
2910	return pgmPoolSyncCR3(pVCpu);
2911	}
2912	}
2913	return VINF_SUCCESS;
2914	}
2915
2916
2917	/**
2918	* Frees up at least one user entry.
2919	*
2920	* @returns VBox status code.
2921	* @retval VINF_SUCCESS if successfully added.
2922	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2923	* @param pPool The pool.
2924	* @param iUser The user index.
2925	*/
2926	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2927	{
2928	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2929	/*
2930	* Just free cached pages in a braindead fashion.
2931	*/
2932	/** @todo walk the age list backwards and free the first with usage. */
2933	int rc = VINF_SUCCESS;
2934	do
2935	{
2936	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2937	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2938	rc = rc2;
2939	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2940	return rc;
2941	}
2942
2943
2944	/**
2945	* Inserts a page into the cache.
2946	*
2947	* This will create user node for the page, insert it into the GCPhys
2948	* hash, and insert it into the age list.
2949	*
2950	* @returns VBox status code.
2951	* @retval VINF_SUCCESS if successfully added.
2952	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2953	* @param pPool The pool.
2954	* @param pPage The cached page.
2955	* @param GCPhys The GC physical address of the page we're gonna shadow.
2956	* @param iUser The user index.
2957	* @param iUserTable The user table index.
2958	*/
2959	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2960	{
2961	int rc = VINF_SUCCESS;
2962	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2963
2964	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
2965
2966	if (iUser != NIL_PGMPOOL_IDX)
2967	{
2968	#ifdef VBOX_STRICT
2969	/*
2970	* Check that the entry doesn't already exists.
2971	*/
2972	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2973	{
2974	uint16_t i = pPage->iUserHead;
2975	do
2976	{
2977	Assert(i < pPool->cMaxUsers);
2978	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2979	i = paUsers[i].iNext;
2980	} while (i != NIL_PGMPOOL_USER_INDEX);
2981	}
2982	#endif
2983
2984	/*
2985	* Find free a user node.
2986	*/
2987	uint16_t i = pPool->iUserFreeHead;
2988	if (i == NIL_PGMPOOL_USER_INDEX)
2989	{
2990	rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2991	if (RT_FAILURE(rc))
2992	return rc;
2993	i = pPool->iUserFreeHead;
2994	}
2995
2996	/*
2997	* Unlink the user node from the free list,
2998	* initialize and insert it into the user list.
2999	*/
3000	pPool->iUserFreeHead = paUsers[i].iNext;
3001	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
3002	paUsers[i].iUser = iUser;
3003	paUsers[i].iUserTable = iUserTable;
3004	pPage->iUserHead = i;
3005	}
3006	else
3007	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3008
3009
3010	/*
3011	* Insert into cache and enable monitoring of the guest page if enabled.
3012	*
3013	* Until we implement caching of all levels, including the CR3 one, we'll
3014	* have to make sure we don't try monitor & cache any recursive reuse of
3015	* a monitored CR3 page. Because all windows versions are doing this we'll
3016	* have to be able to do combined access monitoring, CR3 + PT and
3017	* PD + PT (guest PAE).
3018	*
3019	* Update:
3020	* We're now cooperating with the CR3 monitor if an uncachable page is found.
3021	*/
3022	const bool fCanBeMonitored = true;
3023	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
3024	if (fCanBeMonitored)
3025	{
3026	rc = pgmPoolMonitorInsert(pPool, pPage);
3027	AssertRC(rc);
3028	}
3029	return rc;
3030	}
3031
3032
3033	/**
3034	* Adds a user reference to a page.
3035	*
3036	* This will move the page to the head of the
3037	*
3038	* @returns VBox status code.
3039	* @retval VINF_SUCCESS if successfully added.
3040	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3041	* @param pPool The pool.
3042	* @param pPage The cached page.
3043	* @param iUser The user index.
3044	* @param iUserTable The user table.
3045	*/
3046	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3047	{
3048	Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
3049	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3050	Assert(iUser != NIL_PGMPOOL_IDX);
3051
3052	# ifdef VBOX_STRICT
3053	/*
3054	* Check that the entry doesn't already exists. We only allow multiple
3055	* users of top-level paging structures (SHW_POOL_ROOT_IDX).
3056	*/
3057	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
3058	{
3059	uint16_t i = pPage->iUserHead;
3060	do
3061	{
3062	Assert(i < pPool->cMaxUsers);
3063	/** @todo this assertion looks odd... Shouldn't it be && here? */
3064	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
3065	i = paUsers[i].iNext;
3066	} while (i != NIL_PGMPOOL_USER_INDEX);
3067	}
3068	# endif
3069
3070	/*
3071	* Allocate a user node.
3072	*/
3073	uint16_t i = pPool->iUserFreeHead;
3074	if (i == NIL_PGMPOOL_USER_INDEX)
3075	{
3076	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
3077	if (RT_FAILURE(rc))
3078	return rc;
3079	i = pPool->iUserFreeHead;
3080	}
3081	pPool->iUserFreeHead = paUsers[i].iNext;
3082
3083	/*
3084	* Initialize the user node and insert it.
3085	*/
3086	paUsers[i].iNext = pPage->iUserHead;
3087	paUsers[i].iUser = iUser;
3088	paUsers[i].iUserTable = iUserTable;
3089	pPage->iUserHead = i;
3090
3091	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3092	if (pPage->fDirty)
3093	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
3094	# endif
3095
3096	/*
3097	* Tell the cache to update its replacement stats for this page.
3098	*/
3099	pgmPoolCacheUsed(pPool, pPage);
3100	return VINF_SUCCESS;
3101	}
3102
3103
3104	/**
3105	* Frees a user record associated with a page.
3106	*
3107	* This does not clear the entry in the user table, it simply replaces the
3108	* user record to the chain of free records.
3109	*
3110	* @param pPool The pool.
3111	* @param pPage The shadow page.
3112	* @param iUser The shadow page pool index of the user table.
3113	* @param iUserTable The index into the user table (shadowed).
3114	*
3115	* @remarks Don't call this for root pages.
3116	*/
3117	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3118	{
3119	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
3120	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3121	Assert(iUser != NIL_PGMPOOL_IDX);
3122
3123	/*
3124	* Unlink and free the specified user entry.
3125	*/
3126
3127	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
3128	uint16_t i = pPage->iUserHead;
3129	if ( i != NIL_PGMPOOL_USER_INDEX
3130	&& paUsers[i].iUser == iUser
3131	&& paUsers[i].iUserTable == iUserTable)
3132	{
3133	pPage->iUserHead = paUsers[i].iNext;
3134
3135	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3136	paUsers[i].iNext = pPool->iUserFreeHead;
3137	pPool->iUserFreeHead = i;
3138	return;
3139	}
3140
3141	/* General: Linear search. */
3142	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
3143	while (i != NIL_PGMPOOL_USER_INDEX)
3144	{
3145	if ( paUsers[i].iUser == iUser
3146	&& paUsers[i].iUserTable == iUserTable)
3147	{
3148	if (iPrev != NIL_PGMPOOL_USER_INDEX)
3149	paUsers[iPrev].iNext = paUsers[i].iNext;
3150	else
3151	pPage->iUserHead = paUsers[i].iNext;
3152
3153	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3154	paUsers[i].iNext = pPool->iUserFreeHead;
3155	pPool->iUserFreeHead = i;
3156	return;
3157	}
3158	iPrev = i;
3159	i = paUsers[i].iNext;
3160	}
3161
3162	/* Fatal: didn't find it */
3163	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
3164	iUser, iUserTable, pPage->GCPhys));
3165	}
3166
3167
3168	#if 0 /* unused */
3169	/**
3170	* Gets the entry size of a shadow table.
3171	*
3172	* @param enmKind The kind of page.
3173	*
3174	* @returns The size of the entry in bytes. That is, 4 or 8.
3175	* @returns If the kind is not for a table, an assertion is raised and 0 is
3176	* returned.
3177	*/
3178	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
3179	{
3180	switch (enmKind)
3181	{
3182	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3183	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3184	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3185	case PGMPOOLKIND_32BIT_PD:
3186	case PGMPOOLKIND_32BIT_PD_PHYS:
3187	return 4;
3188
3189	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3190	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3191	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3192	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3193	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3194	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3195	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3196	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3197	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3198	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3199	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3200	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3201	case PGMPOOLKIND_64BIT_PML4:
3202	case PGMPOOLKIND_PAE_PDPT:
3203	case PGMPOOLKIND_ROOT_NESTED:
3204	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3205	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3206	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3207	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3208	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3209	case PGMPOOLKIND_PAE_PD_PHYS:
3210	case PGMPOOLKIND_PAE_PDPT_PHYS:
3211	return 8;
3212
3213	default:
3214	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3215	}
3216	}
3217	#endif /* unused */
3218
3219	#if 0 /* unused */
3220	/**
3221	* Gets the entry size of a guest table.
3222	*
3223	* @param enmKind The kind of page.
3224	*
3225	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
3226	* @returns If the kind is not for a table, an assertion is raised and 0 is
3227	* returned.
3228	*/
3229	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
3230	{
3231	switch (enmKind)
3232	{
3233	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3234	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3235	case PGMPOOLKIND_32BIT_PD:
3236	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3237	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3238	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3239	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3240	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3241	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3242	return 4;
3243
3244	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3245	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3246	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3247	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3248	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3249	case PGMPOOLKIND_64BIT_PML4:
3250	case PGMPOOLKIND_PAE_PDPT:
3251	return 8;
3252
3253	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3254	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3255	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3256	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3257	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3258	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3259	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3260	case PGMPOOLKIND_ROOT_NESTED:
3261	case PGMPOOLKIND_PAE_PD_PHYS:
3262	case PGMPOOLKIND_PAE_PDPT_PHYS:
3263	case PGMPOOLKIND_32BIT_PD_PHYS:
3264	/** @todo can we return 0? (nobody is calling this...) */
3265	AssertFailed();
3266	return 0;
3267
3268	default:
3269	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3270	}
3271	}
3272	#endif /* unused */
3273
3274
3275	/**
3276	* Checks one shadow page table entry for a mapping of a physical page.
3277	*
3278	* @returns true / false indicating removal of all relevant PTEs
3279	*
3280	* @param pVM The cross context VM structure.
3281	* @param pPhysPage The guest page in question.
3282	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3283	* @param iShw The shadow page table.
3284	* @param iPte Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
3285	*/
3286	static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
3287	{
3288	LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
3289	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3290	bool fRet = false;
3291
3292	/*
3293	* Assert sanity.
3294	*/
3295	Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
3296	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
3297	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
3298
3299	/*
3300	* Then, clear the actual mappings to the page in the shadow PT.
3301	*/
3302	switch (pPage->enmKind)
3303	{
3304	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3305	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3306	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3307	{
3308	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3309	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3310	uint32_t u32AndMask = 0;
3311	uint32_t u32OrMask = 0;
3312
3313	if (!fFlushPTEs)
3314	{
3315	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3316	{
3317	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /** No handler installed. */
3318	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /** Monitoring is temporarily disabled. */
3319	u32OrMask = X86_PTE_RW;
3320	u32AndMask = UINT32_MAX;
3321	fRet = true;
3322	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3323	break;
3324
3325	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /** Write access is monitored. */
3326	u32OrMask = 0;
3327	u32AndMask = ~X86_PTE_RW;
3328	fRet = true;
3329	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3330	break;
3331	default:
3332	/* (shouldn't be here, will assert below) */
3333	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3334	break;
3335	}
3336	}
3337	else
3338	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3339
3340	/* Update the counter if we're removing references. */
3341	if (!u32AndMask)
3342	{
3343	Assert(pPage->cPresent);
3344	Assert(pPool->cPresent);
3345	pPage->cPresent--;
3346	pPool->cPresent--;
3347	}
3348
3349	if ((pPT->a[iPte].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3350	{
3351	X86PTE Pte;
3352
3353	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
3354	Pte.u = (pPT->a[iPte].u & u32AndMask) \| u32OrMask;
3355	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3356	Pte.n.u1Write = 0; /* need to disallow writes when dirty bit tracking is still active. */
3357
3358	ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
3359	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3360	return fRet;
3361	}
3362	#ifdef LOG_ENABLED
3363	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3364	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3365	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3366	{
3367	Log(("i=%d cFound=%d\n", i, ++cFound));
3368	}
3369	#endif
3370	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
3371	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3372	break;
3373	}
3374
3375	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3376	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3377	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3378	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3379	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3380	case PGMPOOLKIND_EPT_PT_FOR_PHYS: /* physical mask the same as PAE; RW bit as well; be careful! */
3381	{
3382	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3383	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3384	uint64_t u64OrMask = 0;
3385	uint64_t u64AndMask = 0;
3386
3387	if (!fFlushPTEs)
3388	{
3389	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3390	{
3391	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3392	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3393	u64OrMask = X86_PTE_RW;
3394	u64AndMask = UINT64_MAX;
3395	fRet = true;
3396	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3397	break;
3398
3399	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3400	u64OrMask = 0;
3401	u64AndMask = ~(uint64_t)X86_PTE_RW;
3402	fRet = true;
3403	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3404	break;
3405
3406	default:
3407	/* (shouldn't be here, will assert below) */
3408	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3409	break;
3410	}
3411	}
3412	else
3413	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3414
3415	/* Update the counter if we're removing references. */
3416	if (!u64AndMask)
3417	{
3418	Assert(pPage->cPresent);
3419	Assert(pPool->cPresent);
3420	pPage->cPresent--;
3421	pPool->cPresent--;
3422	}
3423
3424	if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3425	{
3426	X86PTEPAE Pte;
3427
3428	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3429	Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) \| u64OrMask;
3430	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3431	Pte.n.u1Write = 0; /* need to disallow writes when dirty bit tracking is still active. */
3432
3433	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
3434	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3435	return fRet;
3436	}
3437	#ifdef LOG_ENABLED
3438	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3439	Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX), u64));
3440	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3441	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3442	Log(("i=%d cFound=%d\n", i, ++cFound));
3443	#endif
3444	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3445	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3446	break;
3447	}
3448
3449	#ifdef PGM_WITH_LARGE_PAGES
3450	/* Large page case only. */
3451	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3452	{
3453	Assert(pVM->pgm.s.fNestedPaging);
3454
3455	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3456	PEPTPD pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3457
3458	if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3459	{
3460	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3461	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3462	pPD->a[iPte].u = 0;
3463	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3464
3465	/* Update the counter as we're removing references. */
3466	Assert(pPage->cPresent);
3467	Assert(pPool->cPresent);
3468	pPage->cPresent--;
3469	pPool->cPresent--;
3470
3471	return fRet;
3472	}
3473	# ifdef LOG_ENABLED
3474	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3475	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3476	if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3477	Log(("i=%d cFound=%d\n", i, ++cFound));
3478	# endif
3479	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3480	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3481	break;
3482	}
3483
3484	/* AMD-V nested paging / /* @todo merge with EPT as we only check the parts that are identical. */
3485	case PGMPOOLKIND_PAE_PD_PHYS:
3486	{
3487	Assert(pVM->pgm.s.fNestedPaging);
3488
3489	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3490	PX86PD pPD = (PX86PD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3491
3492	if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3493	{
3494	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3495	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3496	pPD->a[iPte].u = 0;
3497	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3498
3499	/* Update the counter as we're removing references. */
3500	Assert(pPage->cPresent);
3501	Assert(pPool->cPresent);
3502	pPage->cPresent--;
3503	pPool->cPresent--;
3504	return fRet;
3505	}
3506	# ifdef LOG_ENABLED
3507	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3508	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3509	if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3510	Log(("i=%d cFound=%d\n", i, ++cFound));
3511	# endif
3512	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3513	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3514	break;
3515	}
3516	#endif /* PGM_WITH_LARGE_PAGES */
3517
3518	default:
3519	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3520	}
3521
3522	/* not reached. */
3523	#ifndef _MSC_VER
3524	return fRet;
3525	#endif
3526	}
3527
3528
3529	/**
3530	* Scans one shadow page table for mappings of a physical page.
3531	*
3532	* @param pVM The cross context VM structure.
3533	* @param pPhysPage The guest page in question.
3534	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3535	* @param iShw The shadow page table.
3536	*/
3537	static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
3538	{
3539	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3540
3541	/* We should only come here with when there's only one reference to this physical page. */
3542	Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
3543
3544	Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
3545	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3546	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
3547	if (!fKeptPTEs)
3548	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3549	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3550	}
3551
3552
3553	/**
3554	* Flushes a list of shadow page tables mapping the same physical page.
3555	*
3556	* @param pVM The cross context VM structure.
3557	* @param pPhysPage The guest page in question.
3558	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3559	* @param iPhysExt The physical cross reference extent list to flush.
3560	*/
3561	static void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
3562	{
3563	PGM_LOCK_ASSERT_OWNER(pVM);
3564	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3565	bool fKeepList = false;
3566
3567	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3568	Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
3569
3570	const uint16_t iPhysExtStart = iPhysExt;
3571	PPGMPOOLPHYSEXT pPhysExt;
3572	do
3573	{
3574	Assert(iPhysExt < pPool->cMaxPhysExts);
3575	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3576	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3577	{
3578	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3579	{
3580	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
3581	if (!fKeptPTEs)
3582	{
3583	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3584	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3585	}
3586	else
3587	fKeepList = true;
3588	}
3589	}
3590	/* next */
3591	iPhysExt = pPhysExt->iNext;
3592	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3593
3594	if (!fKeepList)
3595	{
3596	/* insert the list into the free list and clear the ram range entry. */
3597	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3598	pPool->iPhysExtFreeHead = iPhysExtStart;
3599	/* Invalidate the tracking data. */
3600	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3601	}
3602
3603	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3604	}
3605
3606
3607	/**
3608	* Flushes all shadow page table mappings of the given guest page.
3609	*
3610	* This is typically called when the host page backing the guest one has been
3611	* replaced or when the page protection was changed due to a guest access
3612	* caught by the monitoring.
3613	*
3614	* @returns VBox status code.
3615	* @retval VINF_SUCCESS if all references has been successfully cleared.
3616	* @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3617	* pool cleaning. FF and sync flags are set.
3618	*
3619	* @param pVM The cross context VM structure.
3620	* @param GCPhysPage GC physical address of the page in question
3621	* @param pPhysPage The guest page in question.
3622	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3623	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3624	* flushed, it is NOT touched if this isn't necessary.
3625	* The caller MUST initialized this to @a false.
3626	*/
3627	int pgmPoolTrackUpdateGCPhys(PVM pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
3628	{
3629	PVMCPU pVCpu = VMMGetCpu(pVM);
3630	pgmLock(pVM);
3631	int rc = VINF_SUCCESS;
3632
3633	#ifdef PGM_WITH_LARGE_PAGES
3634	/* Is this page part of a large page? */
3635	if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
3636	{
3637	RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
3638	GCPhysPage &= X86_PDE_PAE_PG_MASK;
3639
3640	/* Fetch the large page base. */
3641	PPGMPAGE pLargePage;
3642	if (GCPhysBase != GCPhysPage)
3643	{
3644	pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
3645	AssertFatal(pLargePage);
3646	}
3647	else
3648	pLargePage = pPhysPage;
3649
3650	Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
3651
3652	if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
3653	{
3654	/* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
3655	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3656	pVM->pgm.s.cLargePagesDisabled++;
3657
3658	/* Update the base as that only that one has a reference and there's only one PDE to clear. */
3659	rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
3660
3661	*pfFlushTLBs = true;
3662	pgmUnlock(pVM);
3663	return rc;
3664	}
3665	}
3666	#else
3667	NOREF(GCPhysPage);
3668	#endif /* PGM_WITH_LARGE_PAGES */
3669
3670	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3671	if (u16)
3672	{
3673	/*
3674	* The zero page is currently screwing up the tracking and we'll
3675	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3676	* is defined, zero pages won't normally be mapped. Some kind of solution
3677	* will be needed for this problem of course, but it will have to wait...
3678	*/
3679	if ( PGM_PAGE_IS_ZERO(pPhysPage)
3680	\|\| PGM_PAGE_IS_BALLOONED(pPhysPage))
3681	rc = VINF_PGM_GCPHYS_ALIASED;
3682	else
3683	{
3684	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC) /** @todo we can drop this now. */
3685	/* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
3686	pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
3687	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
3688	# endif
3689
3690	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3691	{
3692	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3693	pgmPoolTrackFlushGCPhysPT(pVM,
3694	pPhysPage,
3695	fFlushPTEs,
3696	PGMPOOL_TD_GET_IDX(u16));
3697	}
3698	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3699	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
3700	else
3701	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3702	*pfFlushTLBs = true;
3703
3704	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
3705	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
3706	# endif
3707	}
3708	}
3709
3710	if (rc == VINF_PGM_GCPHYS_ALIASED)
3711	{
3712	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3713	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3714	rc = VINF_PGM_SYNC_CR3;
3715	}
3716	pgmUnlock(pVM);
3717	return rc;
3718	}
3719
3720
3721	/**
3722	* Scans all shadow page tables for mappings of a physical page.
3723	*
3724	* This may be slow, but it's most likely more efficient than cleaning
3725	* out the entire page pool / cache.
3726	*
3727	* @returns VBox status code.
3728	* @retval VINF_SUCCESS if all references has been successfully cleared.
3729	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3730	* a page pool cleaning.
3731	*
3732	* @param pVM The cross context VM structure.
3733	* @param pPhysPage The guest page in question.
3734	*/
3735	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
3736	{
3737	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3738	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3739	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3740	pPool->cUsedPages, pPool->cPresent, pPhysPage));
3741
3742	/*
3743	* There is a limit to what makes sense.
3744	*/
3745	if ( pPool->cPresent > 1024
3746	&& pVM->cCpus == 1)
3747	{
3748	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3749	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3750	return VINF_PGM_GCPHYS_ALIASED;
3751	}
3752
3753	/*
3754	* Iterate all the pages until we've encountered all that in use.
3755	* This is simple but not quite optimal solution.
3756	*/
3757	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P; /** @todo drop X86_PTE_P here as we always test if present separately, anyway. */
3758	const uint32_t u32 = u64; /** @todo move into the 32BIT_PT_xx case */
3759	unsigned cLeft = pPool->cUsedPages;
3760	unsigned iPage = pPool->cCurPages;
3761	while (--iPage >= PGMPOOL_IDX_FIRST)
3762	{
3763	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3764	if ( pPage->GCPhys != NIL_RTGCPHYS
3765	&& pPage->cPresent)
3766	{
3767	switch (pPage->enmKind)
3768	{
3769	/*
3770	* We only care about shadow page tables.
3771	*/
3772	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3773	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3774	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3775	{
3776	unsigned cPresent = pPage->cPresent;
3777	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3778	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3779	if (pPT->a[i].n.u1Present)
3780	{
3781	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3782	{
3783	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3784	pPT->a[i].u = 0;
3785
3786	/* Update the counter as we're removing references. */
3787	Assert(pPage->cPresent);
3788	Assert(pPool->cPresent);
3789	pPage->cPresent--;
3790	pPool->cPresent--;
3791	}
3792	if (!--cPresent)
3793	break;
3794	}
3795	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3796	break;
3797	}
3798
3799	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3800	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3801	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3802	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3803	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3804	{
3805	unsigned cPresent = pPage->cPresent;
3806	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3807	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3808	if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
3809	{
3810	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
3811	{
3812	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3813	PGMSHWPTEPAE_SET(pPT->a[i], 0); /// @todo why not atomic?
3814
3815	/* Update the counter as we're removing references. */
3816	Assert(pPage->cPresent);
3817	Assert(pPool->cPresent);
3818	pPage->cPresent--;
3819	pPool->cPresent--;
3820	}
3821	if (!--cPresent)
3822	break;
3823	}
3824	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3825	break;
3826	}
3827	#ifndef IN_RC
3828	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3829	{
3830	unsigned cPresent = pPage->cPresent;
3831	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3832	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3833	if (pPT->a[i].n.u1Present)
3834	{
3835	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
3836	{
3837	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3838	pPT->a[i].u = 0;
3839
3840	/* Update the counter as we're removing references. */
3841	Assert(pPage->cPresent);
3842	Assert(pPool->cPresent);
3843	pPage->cPresent--;
3844	pPool->cPresent--;
3845	}
3846	if (!--cPresent)
3847	break;
3848	}
3849	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3850	break;
3851	}
3852	#endif
3853	}
3854	if (!--cLeft)
3855	break;
3856	}
3857	}
3858
3859	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3860	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3861
3862	/*
3863	* There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
3864	*/
3865	if (pPool->cPresent > 1024)
3866	{
3867	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3868	return VINF_PGM_GCPHYS_ALIASED;
3869	}
3870
3871	return VINF_SUCCESS;
3872	}
3873
3874
3875	/**
3876	* Clears the user entry in a user table.
3877	*
3878	* This is used to remove all references to a page when flushing it.
3879	*/
3880	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3881	{
3882	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3883	Assert(pUser->iUser < pPool->cCurPages);
3884	uint32_t iUserTable = pUser->iUserTable;
3885
3886	/*
3887	* Map the user page. Ignore references made by fictitious pages.
3888	*/
3889	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3890	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3891	union
3892	{
3893	uint64_t *pau64;
3894	uint32_t *pau32;
3895	} u;
3896	if (pUserPage->idx < PGMPOOL_IDX_FIRST)
3897	{
3898	Assert(!pUserPage->pvPageR3);
3899	return;
3900	}
3901	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3902
3903
3904	/* Safety precaution in case we change the paging for other modes too in the future. */
3905	Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
3906
3907	#ifdef VBOX_STRICT
3908	/*
3909	* Some sanity checks.
3910	*/
3911	switch (pUserPage->enmKind)
3912	{
3913	case PGMPOOLKIND_32BIT_PD:
3914	case PGMPOOLKIND_32BIT_PD_PHYS:
3915	Assert(iUserTable < X86_PG_ENTRIES);
3916	break;
3917	case PGMPOOLKIND_PAE_PDPT:
3918	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3919	case PGMPOOLKIND_PAE_PDPT_PHYS:
3920	Assert(iUserTable < 4);
3921	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3922	break;
3923	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3924	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3925	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3926	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3927	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3928	case PGMPOOLKIND_PAE_PD_PHYS:
3929	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3930	break;
3931	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3932	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3933	Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
3934	break;
3935	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3936	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3937	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3938	break;
3939	case PGMPOOLKIND_64BIT_PML4:
3940	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3941	/* GCPhys >> PAGE_SHIFT is the index here */
3942	break;
3943	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3944	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3945	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3946	break;
3947
3948	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3949	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3950	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3951	break;
3952
3953	case PGMPOOLKIND_ROOT_NESTED:
3954	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3955	break;
3956
3957	default:
3958	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3959	break;
3960	}
3961	#endif /* VBOX_STRICT */
3962
3963	/*
3964	* Clear the entry in the user page.
3965	*/
3966	switch (pUserPage->enmKind)
3967	{
3968	/* 32-bit entries */
3969	case PGMPOOLKIND_32BIT_PD:
3970	case PGMPOOLKIND_32BIT_PD_PHYS:
3971	ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
3972	break;
3973
3974	/* 64-bit entries */
3975	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3976	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3977	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3978	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3979	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3980	#ifdef IN_RC
3981	/*
3982	* In 32 bits PAE mode we must invalidate the TLB when changing a
3983	* PDPT entry; the CPU fetches them only during cr3 load, so any
3984	* non-present PDPT will continue to cause page faults.
3985	*/
3986	ASMReloadCR3();
3987	#endif
3988	/* fall thru */
3989	case PGMPOOLKIND_PAE_PD_PHYS:
3990	case PGMPOOLKIND_PAE_PDPT_PHYS:
3991	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3992	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3993	case PGMPOOLKIND_64BIT_PML4:
3994	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3995	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3996	case PGMPOOLKIND_PAE_PDPT:
3997	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3998	case PGMPOOLKIND_ROOT_NESTED:
3999	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4000	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4001	ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
4002	break;
4003
4004	default:
4005	AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
4006	}
4007	PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
4008	}
4009
4010
4011	/**
4012	* Clears all users of a page.
4013	*/
4014	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4015	{
4016	/*
4017	* Free all the user records.
4018	*/
4019	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
4020
4021	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4022	uint16_t i = pPage->iUserHead;
4023	while (i != NIL_PGMPOOL_USER_INDEX)
4024	{
4025	/* Clear enter in user table. */
4026	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
4027
4028	/* Free it. */
4029	const uint16_t iNext = paUsers[i].iNext;
4030	paUsers[i].iUser = NIL_PGMPOOL_IDX;
4031	paUsers[i].iNext = pPool->iUserFreeHead;
4032	pPool->iUserFreeHead = i;
4033
4034	/* Next. */
4035	i = iNext;
4036	}
4037	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4038	}
4039
4040
4041	/**
4042	* Allocates a new physical cross reference extent.
4043	*
4044	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
4045	* @param pVM The cross context VM structure.
4046	* @param piPhysExt Where to store the phys ext index.
4047	*/
4048	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
4049	{
4050	PGM_LOCK_ASSERT_OWNER(pVM);
4051	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4052	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
4053	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4054	{
4055	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
4056	return NULL;
4057	}
4058	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4059	pPool->iPhysExtFreeHead = pPhysExt->iNext;
4060	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4061	*piPhysExt = iPhysExt;
4062	return pPhysExt;
4063	}
4064
4065
4066	/**
4067	* Frees a physical cross reference extent.
4068	*
4069	* @param pVM The cross context VM structure.
4070	* @param iPhysExt The extent to free.
4071	*/
4072	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
4073	{
4074	PGM_LOCK_ASSERT_OWNER(pVM);
4075	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4076	Assert(iPhysExt < pPool->cMaxPhysExts);
4077	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4078	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4079	{
4080	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4081	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4082	}
4083	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4084	pPool->iPhysExtFreeHead = iPhysExt;
4085	}
4086
4087
4088	/**
4089	* Frees a physical cross reference extent.
4090	*
4091	* @param pVM The cross context VM structure.
4092	* @param iPhysExt The extent to free.
4093	*/
4094	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
4095	{
4096	PGM_LOCK_ASSERT_OWNER(pVM);
4097	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4098
4099	const uint16_t iPhysExtStart = iPhysExt;
4100	PPGMPOOLPHYSEXT pPhysExt;
4101	do
4102	{
4103	Assert(iPhysExt < pPool->cMaxPhysExts);
4104	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4105	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4106	{
4107	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4108	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4109	}
4110
4111	/* next */
4112	iPhysExt = pPhysExt->iNext;
4113	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4114
4115	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4116	pPool->iPhysExtFreeHead = iPhysExtStart;
4117	}
4118
4119
4120	/**
4121	* Insert a reference into a list of physical cross reference extents.
4122	*
4123	* @returns The new tracking data for PGMPAGE.
4124	*
4125	* @param pVM The cross context VM structure.
4126	* @param iPhysExt The physical extent index of the list head.
4127	* @param iShwPT The shadow page table index.
4128	* @param iPte Page table entry
4129	*
4130	*/
4131	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
4132	{
4133	PGM_LOCK_ASSERT_OWNER(pVM);
4134	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4135	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4136
4137	/*
4138	* Special common cases.
4139	*/
4140	if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
4141	{
4142	paPhysExts[iPhysExt].aidx[1] = iShwPT;
4143	paPhysExts[iPhysExt].apte[1] = iPte;
4144	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
4145	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
4146	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4147	}
4148	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
4149	{
4150	paPhysExts[iPhysExt].aidx[2] = iShwPT;
4151	paPhysExts[iPhysExt].apte[2] = iPte;
4152	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
4153	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
4154	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4155	}
4156	AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
4157
4158	/*
4159	* General treatment.
4160	*/
4161	const uint16_t iPhysExtStart = iPhysExt;
4162	unsigned cMax = 15;
4163	for (;;)
4164	{
4165	Assert(iPhysExt < pPool->cMaxPhysExts);
4166	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4167	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
4168	{
4169	paPhysExts[iPhysExt].aidx[i] = iShwPT;
4170	paPhysExts[iPhysExt].apte[i] = iPte;
4171	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
4172	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
4173	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
4174	}
4175	if (!--cMax)
4176	{
4177	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackOverflows);
4178	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4179	LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
4180	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4181	}
4182
4183	/* advance */
4184	iPhysExt = paPhysExts[iPhysExt].iNext;
4185	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4186	break;
4187	}
4188
4189	/*
4190	* Add another extent to the list.
4191	*/
4192	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4193	if (!pNew)
4194	{
4195	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackNoExtentsLeft);
4196	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4197	LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
4198	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4199	}
4200	pNew->iNext = iPhysExtStart;
4201	pNew->aidx[0] = iShwPT;
4202	pNew->apte[0] = iPte;
4203	LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
4204	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4205	}
4206
4207
4208	/**
4209	* Add a reference to guest physical page where extents are in use.
4210	*
4211	* @returns The new tracking data for PGMPAGE.
4212	*
4213	* @param pVM The cross context VM structure.
4214	* @param pPhysPage Pointer to the aPages entry in the ram range.
4215	* @param u16 The ram range flags (top 16-bits).
4216	* @param iShwPT The shadow page table index.
4217	* @param iPte Page table entry
4218	*/
4219	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
4220	{
4221	pgmLock(pVM);
4222	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
4223	{
4224	/*
4225	* Convert to extent list.
4226	*/
4227	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
4228	uint16_t iPhysExt;
4229	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4230	if (pPhysExt)
4231	{
4232	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
4233	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliased);
4234	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
4235	pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
4236	pPhysExt->aidx[1] = iShwPT;
4237	pPhysExt->apte[1] = iPte;
4238	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4239	}
4240	else
4241	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4242	}
4243	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
4244	{
4245	/*
4246	* Insert into the extent list.
4247	*/
4248	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
4249	}
4250	else
4251	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedLots);
4252	pgmUnlock(pVM);
4253	return u16;
4254	}
4255
4256
4257	/**
4258	* Clear references to guest physical memory.
4259	*
4260	* @param pPool The pool.
4261	* @param pPage The page.
4262	* @param pPhysPage Pointer to the aPages entry in the ram range.
4263	* @param iPte Shadow PTE index
4264	*/
4265	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
4266	{
4267	PVM pVM = pPool->CTX_SUFF(pVM);
4268	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4269	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4270
4271	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
4272	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
4273	{
4274	pgmLock(pVM);
4275
4276	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
4277	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4278	do
4279	{
4280	Assert(iPhysExt < pPool->cMaxPhysExts);
4281
4282	/*
4283	* Look for the shadow page and check if it's all freed.
4284	*/
4285	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4286	{
4287	if ( paPhysExts[iPhysExt].aidx[i] == pPage->idx
4288	&& paPhysExts[iPhysExt].apte[i] == iPte)
4289	{
4290	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
4291	paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4292
4293	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4294	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
4295	{
4296	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
4297	pgmUnlock(pVM);
4298	return;
4299	}
4300
4301	/* we can free the node. */
4302	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
4303	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
4304	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
4305	{
4306	/* lonely node */
4307	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4308	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
4309	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
4310	}
4311	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
4312	{
4313	/* head */
4314	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
4315	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
4316	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4317	}
4318	else
4319	{
4320	/* in list */
4321	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
4322	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
4323	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4324	}
4325	iPhysExt = iPhysExtNext;
4326	pgmUnlock(pVM);
4327	return;
4328	}
4329	}
4330
4331	/* next */
4332	iPhysExtPrev = iPhysExt;
4333	iPhysExt = paPhysExts[iPhysExt].iNext;
4334	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4335
4336	pgmUnlock(pVM);
4337	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4338	}
4339	else /* nothing to do */
4340	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
4341	}
4342
4343	/**
4344	* Clear references to guest physical memory.
4345	*
4346	* This is the same as pgmPoolTracDerefGCPhysHint except that the guest
4347	* physical address is assumed to be correct, so the linear search can be
4348	* skipped and we can assert at an earlier point.
4349	*
4350	* @param pPool The pool.
4351	* @param pPage The page.
4352	* @param HCPhys The host physical address corresponding to the guest page.
4353	* @param GCPhys The guest physical address corresponding to HCPhys.
4354	* @param iPte Shadow PTE index
4355	*/
4356	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
4357	{
4358	/*
4359	* Lookup the page and check if it checks out before derefing it.
4360	*/
4361	PVM pVM = pPool->CTX_SUFF(pVM);
4362	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
4363	if (pPhysPage)
4364	{
4365	Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
4366	#ifdef LOG_ENABLED
4367	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
4368	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
4369	#endif
4370	if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
4371	{
4372	Assert(pPage->cPresent);
4373	Assert(pPool->cPresent);
4374	pPage->cPresent--;
4375	pPool->cPresent--;
4376	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4377	return;
4378	}
4379
4380	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp\n",
4381	HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage)));
4382	}
4383	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
4384	}
4385
4386
4387	/**
4388	* Clear references to guest physical memory.
4389	*
4390	* @param pPool The pool.
4391	* @param pPage The page.
4392	* @param HCPhys The host physical address corresponding to the guest page.
4393	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
4394	* @param iPte Shadow pte index
4395	*/
4396	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
4397	{
4398	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
4399
4400	/*
4401	* Try the hint first.
4402	*/
4403	RTHCPHYS HCPhysHinted;
4404	PVM pVM = pPool->CTX_SUFF(pVM);
4405	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
4406	if (pPhysPage)
4407	{
4408	HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
4409	Assert(HCPhysHinted);
4410	if (HCPhysHinted == HCPhys)
4411	{
4412	Assert(pPage->cPresent);
4413	Assert(pPool->cPresent);
4414	pPage->cPresent--;
4415	pPool->cPresent--;
4416	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4417	return;
4418	}
4419	}
4420	else
4421	HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
4422
4423	/*
4424	* Damn, the hint didn't work. We'll have to do an expensive linear search.
4425	*/
4426	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
4427	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
4428	while (pRam)
4429	{
4430	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4431	while (iPage-- > 0)
4432	{
4433	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
4434	{
4435	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
4436	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
4437	Assert(pPage->cPresent);
4438	Assert(pPool->cPresent);
4439	pPage->cPresent--;
4440	pPool->cPresent--;
4441	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
4442	return;
4443	}
4444	}
4445	pRam = pRam->CTX_SUFF(pNext);
4446	}
4447
4448	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
4449	}
4450
4451
4452	/**
4453	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
4454	*
4455	* @param pPool The pool.
4456	* @param pPage The page.
4457	* @param pShwPT The shadow page table (mapping of the page).
4458	* @param pGstPT The guest page table.
4459	*/
4460	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
4461	{
4462	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4463	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4464	{
4465	Assert(!(pShwPT->a[i].u & RT_BIT_32(10)));
4466	if (pShwPT->a[i].n.u1Present)
4467	{
4468	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
4469	i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
4470	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
4471	if (!pPage->cPresent)
4472	break;
4473	}
4474	}
4475	}
4476
4477
4478	/**
4479	* Clear references to guest physical memory in a PAE / 32-bit page table.
4480	*
4481	* @param pPool The pool.
4482	* @param pPage The page.
4483	* @param pShwPT The shadow page table (mapping of the page).
4484	* @param pGstPT The guest page table (just a half one).
4485	*/
4486	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
4487	{
4488	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4489	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4490	{
4491	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4492	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4493	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4494	{
4495	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
4496	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
4497	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4498	if (!pPage->cPresent)
4499	break;
4500	}
4501	}
4502	}
4503
4504
4505	/**
4506	* Clear references to guest physical memory in a PAE / PAE page table.
4507	*
4508	* @param pPool The pool.
4509	* @param pPage The page.
4510	* @param pShwPT The shadow page table (mapping of the page).
4511	* @param pGstPT The guest page table.
4512	*/
4513	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
4514	{
4515	RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
4516	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4517	{
4518	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4519	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4520	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4521	{
4522	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
4523	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
4524	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4525	if (!pPage->cPresent)
4526	break;
4527	}
4528	}
4529	}
4530
4531
4532	/**
4533	* Clear references to guest physical memory in a 32-bit / 4MB page table.
4534	*
4535	* @param pPool The pool.
4536	* @param pPage The page.
4537	* @param pShwPT The shadow page table (mapping of the page).
4538	*/
4539	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
4540	{
4541	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4542	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4543	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4544	{
4545	Assert(!(pShwPT->a[i].u & RT_BIT_32(10)));
4546	if (pShwPT->a[i].n.u1Present)
4547	{
4548	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
4549	i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
4550	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4551	if (!pPage->cPresent)
4552	break;
4553	}
4554	}
4555	}
4556
4557
4558	/**
4559	* Clear references to guest physical memory in a PAE / 2/4MB page table.
4560	*
4561	* @param pPool The pool.
4562	* @param pPage The page.
4563	* @param pShwPT The shadow page table (mapping of the page).
4564	*/
4565	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
4566	{
4567	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4568	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4569	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4570	{
4571	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4572	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4573	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4574	{
4575	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
4576	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
4577	pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
4578	if (!pPage->cPresent)
4579	break;
4580	}
4581	}
4582	}
4583
4584
4585	/**
4586	* Clear references to shadowed pages in an EPT page table.
4587	*
4588	* @param pPool The pool.
4589	* @param pPage The page.
4590	* @param pShwPT The shadow page directory pointer table (mapping of the
4591	* page).
4592	*/
4593	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4594	{
4595	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4596	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4597	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4598	{
4599	Assert((pShwPT->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
4600	if (pShwPT->a[i].n.u1Present)
4601	{
4602	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4603	i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
4604	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4605	if (!pPage->cPresent)
4606	break;
4607	}
4608	}
4609	}
4610
4611
4612	/**
4613	* Clear references to shadowed pages in a 32 bits page directory.
4614	*
4615	* @param pPool The pool.
4616	* @param pPage The page.
4617	* @param pShwPD The shadow page directory (mapping of the page).
4618	*/
4619	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
4620	{
4621	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4622	{
4623	if ( pShwPD->a[i].n.u1Present
4624	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
4625	)
4626	{
4627	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
4628	if (pSubPage)
4629	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4630	else
4631	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
4632	}
4633	}
4634	}
4635
4636
4637	/**
4638	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
4639	*
4640	* @param pPool The pool.
4641	* @param pPage The page.
4642	* @param pShwPD The shadow page directory (mapping of the page).
4643	*/
4644	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
4645	{
4646	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4647	{
4648	if ( pShwPD->a[i].n.u1Present
4649	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING))
4650	{
4651	#ifdef PGM_WITH_LARGE_PAGES
4652	if (pShwPD->a[i].b.u1Size)
4653	{
4654	Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
4655	i, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4656	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK,
4657	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4658	i);
4659	}
4660	else
4661	#endif
4662	{
4663	Assert((pShwPD->a[i].u & (X86_PDE_PAE_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000000))) == 0);
4664	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
4665	if (pSubPage)
4666	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4667	else
4668	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
4669	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4670	}
4671	}
4672	}
4673	}
4674
4675
4676	/**
4677	* Clear references to shadowed pages in a PAE page directory pointer table.
4678	*
4679	* @param pPool The pool.
4680	* @param pPage The page.
4681	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4682	*/
4683	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4684	{
4685	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4686	{
4687	Assert((pShwPDPT->a[i].u & (X86_PDPE_PAE_MBZ_MASK \| UINT64_C(0x7ff0000000000200))) == 0);
4688	if ( pShwPDPT->a[i].n.u1Present
4689	&& !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
4690	)
4691	{
4692	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
4693	if (pSubPage)
4694	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4695	else
4696	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
4697	}
4698	}
4699	}
4700
4701
4702	/**
4703	* Clear references to shadowed pages in a 64-bit page directory pointer table.
4704	*
4705	* @param pPool The pool.
4706	* @param pPage The page.
4707	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4708	*/
4709	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4710	{
4711	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4712	{
4713	Assert((pShwPDPT->a[i].u & (X86_PDPE_LM_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4714	if (pShwPDPT->a[i].n.u1Present)
4715	{
4716	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
4717	if (pSubPage)
4718	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4719	else
4720	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
4721	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4722	}
4723	}
4724	}
4725
4726
4727	/**
4728	* Clear references to shadowed pages in a 64-bit level 4 page table.
4729	*
4730	* @param pPool The pool.
4731	* @param pPage The page.
4732	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4733	*/
4734	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4735	{
4736	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4737	{
4738	Assert((pShwPML4->a[i].u & (X86_PML4E_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4739	if (pShwPML4->a[i].n.u1Present)
4740	{
4741	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
4742	if (pSubPage)
4743	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4744	else
4745	AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
4746	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4747	}
4748	}
4749	}
4750
4751
4752	/**
4753	* Clear references to shadowed pages in an EPT page directory.
4754	*
4755	* @param pPool The pool.
4756	* @param pPage The page.
4757	* @param pShwPD The shadow page directory (mapping of the page).
4758	*/
4759	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4760	{
4761	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4762	{
4763	Assert((pShwPD->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
4764	if (pShwPD->a[i].n.u1Present)
4765	{
4766	#ifdef PGM_WITH_LARGE_PAGES
4767	if (pShwPD->a[i].b.u1Size)
4768	{
4769	Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
4770	i, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4771	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK,
4772	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4773	i);
4774	}
4775	else
4776	#endif
4777	{
4778	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
4779	if (pSubPage)
4780	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4781	else
4782	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4783	}
4784	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4785	}
4786	}
4787	}
4788
4789
4790	/**
4791	* Clear references to shadowed pages in an EPT page directory pointer table.
4792	*
4793	* @param pPool The pool.
4794	* @param pPage The page.
4795	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4796	*/
4797	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4798	{
4799	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4800	{
4801	Assert((pShwPDPT->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
4802	if (pShwPDPT->a[i].n.u1Present)
4803	{
4804	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
4805	if (pSubPage)
4806	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4807	else
4808	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
4809	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4810	}
4811	}
4812	}
4813
4814
4815	/**
4816	* Clears all references made by this page.
4817	*
4818	* This includes other shadow pages and GC physical addresses.
4819	*
4820	* @param pPool The pool.
4821	* @param pPage The page.
4822	*/
4823	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4824	{
4825	/*
4826	* Map the shadow page and take action according to the page kind.
4827	*/
4828	PVM pVM = pPool->CTX_SUFF(pVM);
4829	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4830	switch (pPage->enmKind)
4831	{
4832	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4833	{
4834	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4835	void *pvGst;
4836	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4837	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4838	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4839	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4840	break;
4841	}
4842
4843	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4844	{
4845	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4846	void *pvGst;
4847	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4848	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
4849	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4850	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4851	break;
4852	}
4853
4854	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4855	{
4856	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4857	void *pvGst;
4858	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4859	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
4860	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4861	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4862	break;
4863	}
4864
4865	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4866	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4867	{
4868	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4869	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4870	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4871	break;
4872	}
4873
4874	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
4875	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4876	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4877	{
4878	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4879	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
4880	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4881	break;
4882	}
4883
4884	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4885	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4886	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4887	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4888	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4889	case PGMPOOLKIND_PAE_PD_PHYS:
4890	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4891	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4892	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
4893	break;
4894
4895	case PGMPOOLKIND_32BIT_PD_PHYS:
4896	case PGMPOOLKIND_32BIT_PD:
4897	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
4898	break;
4899
4900	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4901	case PGMPOOLKIND_PAE_PDPT:
4902	case PGMPOOLKIND_PAE_PDPT_PHYS:
4903	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
4904	break;
4905
4906	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4907	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4908	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
4909	break;
4910
4911	case PGMPOOLKIND_64BIT_PML4:
4912	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
4913	break;
4914
4915	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4916	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
4917	break;
4918
4919	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4920	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
4921	break;
4922
4923	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4924	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
4925	break;
4926
4927	default:
4928	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
4929	}
4930
4931	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
4932	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4933	ASMMemZeroPage(pvShw);
4934	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4935	pPage->fZeroed = true;
4936	Assert(!pPage->cPresent);
4937	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
4938	}
4939
4940
4941	/**
4942	* Flushes a pool page.
4943	*
4944	* This moves the page to the free list after removing all user references to it.
4945	*
4946	* @returns VBox status code.
4947	* @retval VINF_SUCCESS on success.
4948	* @param pPool The pool.
4949	* @param pPage The shadow page.
4950	* @param fFlush Flush the TLBS when required (should only be false in very specific use cases!!)
4951	*/
4952	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
4953	{
4954	PVM pVM = pPool->CTX_SUFF(pVM);
4955	bool fFlushRequired = false;
4956
4957	int rc = VINF_SUCCESS;
4958	STAM_PROFILE_START(&pPool->StatFlushPage, f);
4959	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4960	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4961
4962	/*
4963	* Reject any attempts at flushing any of the special root pages (shall
4964	* not happen).
4965	*/
4966	AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
4967	("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
4968	pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
4969	VINF_SUCCESS);
4970
4971	pgmLock(pVM);
4972
4973	/*
4974	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4975	*/
4976	if (pgmPoolIsPageLocked(pPage))
4977	{
4978	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4979	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4980	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4981	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
4982	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4983	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4984	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4985	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4986	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
4987	\|\| pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
4988	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
4989	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4990	pgmUnlock(pVM);
4991	return VINF_SUCCESS;
4992	}
4993
4994	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
4995	/* Start a subset so we won't run out of mapping space. */
4996	PVMCPU pVCpu = VMMGetCpu(pVM);
4997	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
4998	#endif
4999
5000	/*
5001	* Mark the page as being in need of an ASMMemZeroPage().
5002	*/
5003	pPage->fZeroed = false;
5004
5005	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5006	if (pPage->fDirty)
5007	pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
5008	#endif
5009
5010	/* If there are any users of this table, then we must issue a tlb flush on all VCPUs. */
5011	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
5012	fFlushRequired = true;
5013
5014	/*
5015	* Clear the page.
5016	*/
5017	pgmPoolTrackClearPageUsers(pPool, pPage);
5018	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
5019	pgmPoolTrackDeref(pPool, pPage);
5020	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
5021
5022	/*
5023	* Flush it from the cache.
5024	*/
5025	pgmPoolCacheFlushPage(pPool, pPage);
5026
5027	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
5028	/* Heavy stuff done. */
5029	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
5030	#endif
5031
5032	/*
5033	* Deregistering the monitoring.
5034	*/
5035	if (pPage->fMonitored)
5036	rc = pgmPoolMonitorFlush(pPool, pPage);
5037
5038	/*
5039	* Free the page.
5040	*/
5041	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
5042	pPage->iNext = pPool->iFreeHead;
5043	pPool->iFreeHead = pPage->idx;
5044	pPage->enmKind = PGMPOOLKIND_FREE;
5045	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5046	pPage->GCPhys = NIL_RTGCPHYS;
5047	pPage->fReusedFlushPending = false;
5048
5049	pPool->cUsedPages--;
5050
5051	/* Flush the TLBs of all VCPUs if required. */
5052	if ( fFlushRequired
5053	&& fFlush)
5054	{
5055	PGM_INVL_ALL_VCPU_TLBS(pVM);
5056	}
5057
5058	pgmUnlock(pVM);
5059	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
5060	return rc;
5061	}
5062
5063
5064	/**
5065	* Frees a usage of a pool page.
5066	*
5067	* The caller is responsible to updating the user table so that it no longer
5068	* references the shadow page.
5069	*
5070	* @param pPool The pool.
5071	* @param pPage The shadow page.
5072	* @param iUser The shadow page pool index of the user table.
5073	* NIL_PGMPOOL_IDX for root pages.
5074	* @param iUserTable The index into the user table (shadowed). Ignored if
5075	* root page.
5076	*/
5077	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
5078	{
5079	PVM pVM = pPool->CTX_SUFF(pVM);
5080
5081	STAM_PROFILE_START(&pPool->StatFree, a);
5082	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
5083	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
5084	AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
5085
5086	pgmLock(pVM);
5087	if (iUser != NIL_PGMPOOL_IDX)
5088	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
5089	if (!pPage->fCached)
5090	pgmPoolFlushPage(pPool, pPage);
5091	pgmUnlock(pVM);
5092	STAM_PROFILE_STOP(&pPool->StatFree, a);
5093	}
5094
5095
5096	/**
5097	* Makes one or more free page free.
5098	*
5099	* @returns VBox status code.
5100	* @retval VINF_SUCCESS on success.
5101	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
5102	*
5103	* @param pPool The pool.
5104	* @param enmKind Page table kind
5105	* @param iUser The user of the page.
5106	*/
5107	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
5108	{
5109	PVM pVM = pPool->CTX_SUFF(pVM);
5110	LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
5111	NOREF(enmKind);
5112
5113	/*
5114	* If the pool isn't full grown yet, expand it.
5115	*/
5116	if ( pPool->cCurPages < pPool->cMaxPages
5117	#if defined(IN_RC)
5118	/* Hack alert: we can't deal with jumps to ring 3 when called from MapCR3 and allocating pages for PAE PDs. */
5119	&& enmKind != PGMPOOLKIND_PAE_PD_FOR_PAE_PD
5120	&& (enmKind < PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD \|\| enmKind > PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD)
5121	#endif
5122	)
5123	{
5124	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
5125	#ifdef IN_RING3
5126	int rc = PGMR3PoolGrow(pVM);
5127	#else
5128	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
5129	#endif
5130	if (RT_FAILURE(rc))
5131	return rc;
5132	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
5133	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
5134	return VINF_SUCCESS;
5135	}
5136
5137	/*
5138	* Free one cached page.
5139	*/
5140	return pgmPoolCacheFreeOne(pPool, iUser);
5141	}
5142
5143
5144	/**
5145	* Allocates a page from the pool.
5146	*
5147	* This page may actually be a cached page and not in need of any processing
5148	* on the callers part.
5149	*
5150	* @returns VBox status code.
5151	* @retval VINF_SUCCESS if a NEW page was allocated.
5152	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
5153	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
5154	*
5155	* @param pVM The cross context VM structure.
5156	* @param GCPhys The GC physical address of the page we're gonna shadow.
5157	* For 4MB and 2MB PD entries, it's the first address the
5158	* shadow PT is covering.
5159	* @param enmKind The kind of mapping.
5160	* @param enmAccess Access type for the mapping (only relevant for big pages)
5161	* @param fA20Enabled Whether the A20 gate is enabled or not.
5162	* @param iUser The shadow page pool index of the user table. Root
5163	* pages should pass NIL_PGMPOOL_IDX.
5164	* @param iUserTable The index into the user table (shadowed). Ignored for
5165	* root pages (iUser == NIL_PGMPOOL_IDX).
5166	* @param fLockPage Lock the page
5167	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
5168	*/
5169	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
5170	uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
5171	{
5172	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5173	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
5174	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
5175	*ppPage = NULL;
5176	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
5177	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
5178	* Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
5179
5180	pgmLock(pVM);
5181
5182	if (pPool->fCacheEnabled)
5183	{
5184	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
5185	if (RT_SUCCESS(rc2))
5186	{
5187	if (fLockPage)
5188	pgmPoolLockPage(pPool, *ppPage);
5189	pgmUnlock(pVM);
5190	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5191	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
5192	return rc2;
5193	}
5194	}
5195
5196	/*
5197	* Allocate a new one.
5198	*/
5199	int rc = VINF_SUCCESS;
5200	uint16_t iNew = pPool->iFreeHead;
5201	if (iNew == NIL_PGMPOOL_IDX)
5202	{
5203	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
5204	if (RT_FAILURE(rc))
5205	{
5206	pgmUnlock(pVM);
5207	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
5208	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5209	return rc;
5210	}
5211	iNew = pPool->iFreeHead;
5212	AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_PGM_POOL_IPE);
5213	}
5214
5215	/* unlink the free head */
5216	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
5217	pPool->iFreeHead = pPage->iNext;
5218	pPage->iNext = NIL_PGMPOOL_IDX;
5219
5220	/*
5221	* Initialize it.
5222	*/
5223	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
5224	pPage->enmKind = enmKind;
5225	pPage->enmAccess = enmAccess;
5226	pPage->GCPhys = GCPhys;
5227	pPage->fA20Enabled = fA20Enabled;
5228	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
5229	pPage->fMonitored = false;
5230	pPage->fCached = false;
5231	pPage->fDirty = false;
5232	pPage->fReusedFlushPending = false;
5233	pPage->cModifications = 0;
5234	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5235	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5236	pPage->cPresent = 0;
5237	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5238	pPage->idxDirtyEntry = 0;
5239	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5240	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5241	pPage->cLastAccessHandler = 0;
5242	pPage->cLocked = 0;
5243	# ifdef VBOX_STRICT
5244	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5245	# endif
5246
5247	/*
5248	* Insert into the tracking and cache. If this fails, free the page.
5249	*/
5250	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
5251	if (RT_FAILURE(rc3))
5252	{
5253	pPool->cUsedPages--;
5254	pPage->enmKind = PGMPOOLKIND_FREE;
5255	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5256	pPage->GCPhys = NIL_RTGCPHYS;
5257	pPage->iNext = pPool->iFreeHead;
5258	pPool->iFreeHead = pPage->idx;
5259	pgmUnlock(pVM);
5260	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5261	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
5262	return rc3;
5263	}
5264
5265	/*
5266	* Commit the allocation, clear the page and return.
5267	*/
5268	#ifdef VBOX_WITH_STATISTICS
5269	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
5270	pPool->cUsedPagesHigh = pPool->cUsedPages;
5271	#endif
5272
5273	if (!pPage->fZeroed)
5274	{
5275	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5276	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
5277	ASMMemZeroPage(pv);
5278	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5279	}
5280
5281	*ppPage = pPage;
5282	if (fLockPage)
5283	pgmPoolLockPage(pPool, pPage);
5284	pgmUnlock(pVM);
5285	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
5286	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
5287	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5288	return rc;
5289	}
5290
5291
5292	/**
5293	* Frees a usage of a pool page.
5294	*
5295	* @param pVM The cross context VM structure.
5296	* @param HCPhys The HC physical address of the shadow page.
5297	* @param iUser The shadow page pool index of the user table.
5298	* NIL_PGMPOOL_IDX if root page.
5299	* @param iUserTable The index into the user table (shadowed). Ignored if
5300	* root page.
5301	*/
5302	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
5303	{
5304	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
5305	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5306	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
5307	}
5308
5309
5310	/**
5311	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
5312	*
5313	* @returns Pointer to the shadow page structure.
5314	* @param pPool The pool.
5315	* @param HCPhys The HC physical address of the shadow page.
5316	*/
5317	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
5318	{
5319	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5320
5321	/*
5322	* Look up the page.
5323	*/
5324	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5325
5326	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
5327	return pPage;
5328	}
5329
5330
5331	/**
5332	* Internal worker for finding a page for debugging purposes, no assertions.
5333	*
5334	* @returns Pointer to the shadow page structure. NULL on if not found.
5335	* @param pPool The pool.
5336	* @param HCPhys The HC physical address of the shadow page.
5337	*/
5338	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
5339	{
5340	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5341	return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5342	}
5343
5344	#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
5345
5346	/**
5347	* Flush the specified page if present
5348	*
5349	* @param pVM The cross context VM structure.
5350	* @param GCPhys Guest physical address of the page to flush
5351	*/
5352	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
5353	{
5354	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5355
5356	VM_ASSERT_EMT(pVM);
5357
5358	/*
5359	* Look up the GCPhys in the hash.
5360	*/
5361	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
5362	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
5363	if (i == NIL_PGMPOOL_IDX)
5364	return;
5365
5366	do
5367	{
5368	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5369	if (pPage->GCPhys - GCPhys < PAGE_SIZE)
5370	{
5371	switch (pPage->enmKind)
5372	{
5373	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5374	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5375	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5376	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5377	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5378	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5379	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5380	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5381	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5382	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5383	case PGMPOOLKIND_64BIT_PML4:
5384	case PGMPOOLKIND_32BIT_PD:
5385	case PGMPOOLKIND_PAE_PDPT:
5386	{
5387	Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
5388	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5389	if (pPage->fDirty)
5390	STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
5391	else
5392	#endif
5393	STAM_COUNTER_INC(&pPool->StatForceFlushPage);
5394	Assert(!pgmPoolIsPageLocked(pPage));
5395	pgmPoolMonitorChainFlush(pPool, pPage);
5396	return;
5397	}
5398
5399	/* ignore, no monitoring. */
5400	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5401	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5402	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5403	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5404	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5405	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5406	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5407	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5408	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5409	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5410	case PGMPOOLKIND_ROOT_NESTED:
5411	case PGMPOOLKIND_PAE_PD_PHYS:
5412	case PGMPOOLKIND_PAE_PDPT_PHYS:
5413	case PGMPOOLKIND_32BIT_PD_PHYS:
5414	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5415	break;
5416
5417	default:
5418	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
5419	}
5420	}
5421
5422	/* next */
5423	i = pPage->iNext;
5424	} while (i != NIL_PGMPOOL_IDX);
5425	return;
5426	}
5427
5428	#endif /* IN_RING3 */
5429	#ifdef IN_RING3
5430
5431	/**
5432	* Reset CPU on hot plugging.
5433	*
5434	* @param pVM The cross context VM structure.
5435	* @param pVCpu The cross context virtual CPU structure.
5436	*/
5437	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
5438	{
5439	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5440
5441	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5442	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5443	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5444	}
5445
5446
5447	/**
5448	* Flushes the entire cache.
5449	*
5450	* It will assert a global CR3 flush (FF) and assumes the caller is aware of
5451	* this and execute this CR3 flush.
5452	*
5453	* @param pVM The cross context VM structure.
5454	*/
5455	void pgmR3PoolReset(PVM pVM)
5456	{
5457	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5458
5459	PGM_LOCK_ASSERT_OWNER(pVM);
5460	STAM_PROFILE_START(&pPool->StatR3Reset, a);
5461	LogFlow(("pgmR3PoolReset:\n"));
5462
5463	/*
5464	* If there are no pages in the pool, there is nothing to do.
5465	*/
5466	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
5467	{
5468	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5469	return;
5470	}
5471
5472	/*
5473	* Exit the shadow mode since we're going to clear everything,
5474	* including the root page.
5475	*/
5476	for (VMCPUID i = 0; i < pVM->cCpus; i++)
5477	pgmR3ExitShadowModeBeforePoolFlush(&pVM->aCpus[i]);
5478
5479	/*
5480	* Nuke the free list and reinsert all pages into it.
5481	*/
5482	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
5483	{
5484	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5485
5486	Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
5487	if (pPage->fMonitored)
5488	pgmPoolMonitorFlush(pPool, pPage);
5489	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5490	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5491	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
5492	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
5493	pPage->GCPhys = NIL_RTGCPHYS;
5494	pPage->enmKind = PGMPOOLKIND_FREE;
5495	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5496	Assert(pPage->idx == i);
5497	pPage->iNext = i + 1;
5498	pPage->fA20Enabled = true;
5499	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
5500	pPage->fSeenNonGlobal = false;
5501	pPage->fMonitored = false;
5502	pPage->fDirty = false;
5503	pPage->fCached = false;
5504	pPage->fReusedFlushPending = false;
5505	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
5506	pPage->cPresent = 0;
5507	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5508	pPage->cModifications = 0;
5509	pPage->iAgeNext = NIL_PGMPOOL_IDX;
5510	pPage->iAgePrev = NIL_PGMPOOL_IDX;
5511	pPage->idxDirtyEntry = 0;
5512	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5513	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5514	pPage->cLastAccessHandler = 0;
5515	pPage->cLocked = 0;
5516	#ifdef VBOX_STRICT
5517	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5518	#endif
5519	}
5520	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
5521	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
5522	pPool->cUsedPages = 0;
5523
5524	/*
5525	* Zap and reinitialize the user records.
5526	*/
5527	pPool->cPresent = 0;
5528	pPool->iUserFreeHead = 0;
5529	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
5530	const unsigned cMaxUsers = pPool->cMaxUsers;
5531	for (unsigned i = 0; i < cMaxUsers; i++)
5532	{
5533	paUsers[i].iNext = i + 1;
5534	paUsers[i].iUser = NIL_PGMPOOL_IDX;
5535	paUsers[i].iUserTable = 0xfffffffe;
5536	}
5537	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
5538
5539	/*
5540	* Clear all the GCPhys links and rebuild the phys ext free list.
5541	*/
5542	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
5543	pRam;
5544	pRam = pRam->CTX_SUFF(pNext))
5545	{
5546	unsigned iPage = pRam->cb >> PAGE_SHIFT;
5547	while (iPage-- > 0)
5548	PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
5549	}
5550
5551	pPool->iPhysExtFreeHead = 0;
5552	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
5553	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
5554	for (unsigned i = 0; i < cMaxPhysExts; i++)
5555	{
5556	paPhysExts[i].iNext = i + 1;
5557	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
5558	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5559	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
5560	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5561	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
5562	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5563	}
5564	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
5565
5566	/*
5567	* Just zap the modified list.
5568	*/
5569	pPool->cModifiedPages = 0;
5570	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
5571
5572	/*
5573	* Clear the GCPhys hash and the age list.
5574	*/
5575	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
5576	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
5577	pPool->iAgeHead = NIL_PGMPOOL_IDX;
5578	pPool->iAgeTail = NIL_PGMPOOL_IDX;
5579
5580	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5581	/* Clear all dirty pages. */
5582	pPool->idxFreeDirtyPage = 0;
5583	pPool->cDirtyPages = 0;
5584	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
5585	pPool->aDirtyPages[i].uIdx = NIL_PGMPOOL_IDX;
5586	#endif
5587
5588	/*
5589	* Reinsert active pages into the hash and ensure monitoring chains are correct.
5590	*/
5591	for (VMCPUID i = 0; i < pVM->cCpus; i++)
5592	{
5593	/*
5594	* Re-enter the shadowing mode and assert Sync CR3 FF.
5595	*/
5596	PVMCPU pVCpu = &pVM->aCpus[i];
5597	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5598	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5599	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5600	}
5601
5602	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5603	}
5604
5605	#endif /* IN_RING3 */
5606
5607	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
5608	/**
5609	* Stringifies a PGMPOOLKIND value.
5610	*/
5611	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
5612	{
5613	switch ((PGMPOOLKIND)enmKind)
5614	{
5615	case PGMPOOLKIND_INVALID:
5616	return "PGMPOOLKIND_INVALID";
5617	case PGMPOOLKIND_FREE:
5618	return "PGMPOOLKIND_FREE";
5619	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5620	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
5621	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5622	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
5623	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5624	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
5625	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5626	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
5627	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5628	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
5629	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5630	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
5631	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5632	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
5633	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5634	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
5635	case PGMPOOLKIND_32BIT_PD:
5636	return "PGMPOOLKIND_32BIT_PD";
5637	case PGMPOOLKIND_32BIT_PD_PHYS:
5638	return "PGMPOOLKIND_32BIT_PD_PHYS";
5639	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5640	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
5641	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5642	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
5643	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5644	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
5645	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5646	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
5647	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5648	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
5649	case PGMPOOLKIND_PAE_PD_PHYS:
5650	return "PGMPOOLKIND_PAE_PD_PHYS";
5651	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5652	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
5653	case PGMPOOLKIND_PAE_PDPT:
5654	return "PGMPOOLKIND_PAE_PDPT";
5655	case PGMPOOLKIND_PAE_PDPT_PHYS:
5656	return "PGMPOOLKIND_PAE_PDPT_PHYS";
5657	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5658	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
5659	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5660	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
5661	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5662	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
5663	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5664	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
5665	case PGMPOOLKIND_64BIT_PML4:
5666	return "PGMPOOLKIND_64BIT_PML4";
5667	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5668	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
5669	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5670	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
5671	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5672	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
5673	case PGMPOOLKIND_ROOT_NESTED:
5674	return "PGMPOOLKIND_ROOT_NESTED";
5675	}
5676	return "Unknown kind!";
5677	}
5678	#endif /* LOG_ENABLED \|\| VBOX_STRICT */
5679

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 66974

Download in other formats: