PGMAllPool.cpp@ 103832

Last change on this file since 103832 was 103261, checked in by vboxsync, 10 months ago
VMM/PGMAllPool.cpp: Fix side effect in macro double expansion, bugref:3409
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 225.2 KB

Line
1	/* $Id: PGMAllPool.cpp 103261 2024-02-07 17:39:29Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM_POOL
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/mm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/cpum.h>
38	#include "PGMInternal.h"
39	#include <VBox/vmm/vmcc.h>
40	#include "PGMInline.h"
41	#include <VBox/vmm/hm_vmx.h>
42
43	#include <VBox/log.h>
44	#include <VBox/err.h>
45	#include <iprt/asm.h>
46	#include <iprt/asm-mem.h>
47	#include <iprt/string.h>
48
49
50	/*********************************************************************************************************************************
51	* Internal Functions *
52	*********************************************************************************************************************************/
53	RT_C_DECLS_BEGIN
54	#if 0 /* unused */
55	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
56	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
57	#endif /* unused */
58	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
59	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
60	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
61	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
62	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
63	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
64	#endif
65	#if 0 /defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)/
66	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
67	#endif
68
69	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage);
70	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt);
71	void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt);
72	void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt);
73
74	RT_C_DECLS_END
75
76
77	#if 0 /* unused */
78	/**
79	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
80	*
81	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
82	* @param enmKind The page kind.
83	*/
84	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
85	{
86	switch (enmKind)
87	{
88	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
89	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
90	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
91	return true;
92	default:
93	return false;
94	}
95	}
96	#endif /* unused */
97
98
99	/**
100	* Flushes a chain of pages sharing the same access monitor.
101	*
102	* @param pPool The pool.
103	* @param pPage A page in the chain.
104	*/
105	void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
106	{
107	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
108
109	/*
110	* Find the list head.
111	*/
112	uint16_t idx = pPage->idx;
113	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
114	{
115	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
116	{
117	idx = pPage->iMonitoredPrev;
118	Assert(idx != pPage->idx);
119	pPage = &pPool->aPages[idx];
120	}
121	}
122
123	/*
124	* Iterate the list flushing each shadow page.
125	*/
126	for (;;)
127	{
128	idx = pPage->iMonitoredNext;
129	Assert(idx != pPage->idx);
130	if (pPage->idx >= PGMPOOL_IDX_FIRST)
131	{
132	int rc2 = pgmPoolFlushPage(pPool, pPage);
133	AssertRC(rc2);
134	}
135	/* next */
136	if (idx == NIL_PGMPOOL_IDX)
137	break;
138	pPage = &pPool->aPages[idx];
139	}
140	}
141
142
143	/**
144	* Wrapper for getting the current context pointer to the entry being modified.
145	*
146	* @returns VBox status code suitable for scheduling.
147	* @param pVM The cross context VM structure.
148	* @param pvDst Destination address
149	* @param pvSrc Pointer to the mapping of @a GCPhysSrc or NULL depending
150	* on the context (e.g. \#PF in R0 & RC).
151	* @param GCPhysSrc The source guest physical address.
152	* @param cb Size of data to read
153	*/
154	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVMCC pVM, void pvDst, void const pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
155	{
156	#if defined(IN_RING3)
157	NOREF(pVM); NOREF(GCPhysSrc);
158	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
159	return VINF_SUCCESS;
160	#else
161	/** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
162	NOREF(pvSrc);
163	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
164	#endif
165	}
166
167
168	/**
169	* Process shadow entries before they are changed by the guest.
170	*
171	* For PT entries we will clear them. For PD entries, we'll simply check
172	* for mapping conflicts and set the SyncCR3 FF if found.
173	*
174	* @param pVCpu The cross context virtual CPU structure.
175	* @param pPool The pool.
176	* @param pPage The head page.
177	* @param GCPhysFault The guest physical fault address.
178	* @param pvAddress Pointer to the mapping of @a GCPhysFault or NULL
179	* depending on the context (e.g. \#PF in R0 & RC).
180	* @param cbWrite Write size; might be zero if the caller knows we're not crossing entry boundaries
181	*/
182	static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
183	void const *pvAddress, unsigned cbWrite)
184	{
185	AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
186	const unsigned off = GCPhysFault & GUEST_PAGE_OFFSET_MASK;
187	PVMCC pVM = pPool->CTX_SUFF(pVM);
188	NOREF(pVCpu);
189
190	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
191	(RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
192
193	if (PGMPOOL_PAGE_IS_NESTED(pPage))
194	Log7Func(("%RGv phys=%RGp cbWrite=%d\n", (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
195
196	for (;;)
197	{
198	union
199	{
200	void *pv;
201	PX86PT pPT;
202	PPGMSHWPTPAE pPTPae;
203	PX86PD pPD;
204	PX86PDPAE pPDPae;
205	PX86PDPT pPDPT;
206	PX86PML4 pPML4;
207	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
208	PEPTPDPT pEptPdpt;
209	PEPTPD pEptPd;
210	PEPTPT pEptPt;
211	#endif
212	} uShw;
213
214	LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
215	pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
216
217	uShw.pv = NULL;
218	switch (pPage->enmKind)
219	{
220	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
221	{
222	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
223	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
224	const unsigned iShw = off / sizeof(X86PTE);
225	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
226	X86PGUINT const uPde = uShw.pPT->a[iShw].u;
227	if (uPde & X86_PTE_P)
228	{
229	X86PTE GstPte;
230	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
231	AssertRC(rc);
232	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
233	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK, iShw);
234	ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
235	}
236	break;
237	}
238
239	/* page/2 sized */
240	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
241	{
242	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
243	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
244	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
245	{
246	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
247	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
248	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
249	{
250	X86PTE GstPte;
251	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
252	AssertRC(rc);
253
254	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
255	pgmPoolTracDerefGCPhysHint(pPool, pPage,
256	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
257	GstPte.u & X86_PTE_PG_MASK,
258	iShw);
259	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
260	}
261	}
262	break;
263	}
264
265	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
266	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
267	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
268	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
269	{
270	unsigned iGst = off / sizeof(X86PDE);
271	unsigned iShwPdpt = iGst / 256;
272	unsigned iShw = (iGst % 256) * 2;
273	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
274
275	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
276	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
277	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
278	{
279	for (unsigned i = 0; i < 2; i++)
280	{
281	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw + i].u;
282	if (uPde & X86_PDE_P)
283	{
284	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw + i, uPde));
285	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw + i);
286	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw + i].u, 0);
287	}
288
289	/* paranoia / a bit assumptive. */
290	if ( (off & 3)
291	&& (off & 3) + cbWrite > 4)
292	{
293	const unsigned iShw2 = iShw + 2 + i;
294	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
295	{
296	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
297	if (uPde2 & X86_PDE_P)
298	{
299	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
300	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
301	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
302	}
303	}
304	}
305	}
306	}
307	break;
308	}
309
310	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
311	{
312	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
313	const unsigned iShw = off / sizeof(X86PTEPAE);
314	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
315	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
316	{
317	X86PTEPAE GstPte;
318	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
319	AssertRC(rc);
320
321	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
322	pgmPoolTracDerefGCPhysHint(pPool, pPage,
323	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
324	GstPte.u & X86_PTE_PAE_PG_MASK,
325	iShw);
326	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
327	}
328
329	/* paranoia / a bit assumptive. */
330	if ( (off & 7)
331	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
332	{
333	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
334	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
335
336	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
337	{
338	X86PTEPAE GstPte;
339	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
340	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
341	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
342	AssertRC(rc);
343	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
344	pgmPoolTracDerefGCPhysHint(pPool, pPage,
345	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
346	GstPte.u & X86_PTE_PAE_PG_MASK,
347	iShw2);
348	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
349	}
350	}
351	break;
352	}
353
354	case PGMPOOLKIND_32BIT_PD:
355	{
356	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
357	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
358
359	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
360	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
361	X86PGUINT const uPde = uShw.pPD->a[iShw].u;
362	if (uPde & X86_PDE_P)
363	{
364	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
365	pgmPoolFree(pVM, uPde & X86_PDE_PG_MASK, pPage->idx, iShw);
366	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
367	}
368
369	/* paranoia / a bit assumptive. */
370	if ( (off & 3)
371	&& (off & 3) + cbWrite > sizeof(X86PTE))
372	{
373	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
374	if ( iShw2 != iShw
375	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
376	{
377	X86PGUINT const uPde2 = uShw.pPD->a[iShw2].u;
378	if (uPde2 & X86_PDE_P)
379	{
380	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
381	pgmPoolFree(pVM, uPde2 & X86_PDE_PG_MASK, pPage->idx, iShw2);
382	ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
383	}
384	}
385	}
386	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
387	if ( uShw.pPD->a[iShw].n.u1Present
388	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
389	{
390	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
391	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
392	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
393	}
394	#endif
395	break;
396	}
397
398	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
399	{
400	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
401	const unsigned iShw = off / sizeof(X86PDEPAE);
402	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
403
404	/*
405	* Causes trouble when the guest uses a PDE to refer to the whole page table level
406	* structure. (Invalidate here; faults later on when it tries to change the page
407	* table entries -> recheck; probably only applies to the RC case.)
408	*/
409	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
410	if (uPde & X86_PDE_P)
411	{
412	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
413	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
414	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
415	}
416
417	/* paranoia / a bit assumptive. */
418	if ( (off & 7)
419	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
420	{
421	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
422	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
423
424	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
425	if (uPde2 & X86_PDE_P)
426	{
427	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
428	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
429	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
430	}
431	}
432	break;
433	}
434
435	case PGMPOOLKIND_PAE_PDPT:
436	{
437	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
438	/*
439	* Hopefully this doesn't happen very often:
440	* - touching unused parts of the page
441	* - messing with the bits of pd pointers without changing the physical address
442	*/
443	/* PDPT roots are not page aligned; 32 byte only! */
444	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
445
446	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
447	const unsigned iShw = offPdpt / sizeof(X86PDPE);
448	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
449	{
450	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
451	if (uPdpe & X86_PDPE_P)
452	{
453	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
454	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
455	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
456	}
457
458	/* paranoia / a bit assumptive. */
459	if ( (offPdpt & 7)
460	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
461	{
462	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
463	if ( iShw2 != iShw
464	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
465	{
466	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
467	if (uPdpe2 & X86_PDPE_P)
468	{
469	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
470	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
471	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
472	}
473	}
474	}
475	}
476	break;
477	}
478
479	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
480	{
481	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
482	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
483	const unsigned iShw = off / sizeof(X86PDEPAE);
484	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
485	if (uPde & X86_PDE_P)
486	{
487	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
488	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
489	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
490	}
491
492	/* paranoia / a bit assumptive. */
493	if ( (off & 7)
494	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
495	{
496	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
497	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
498	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
499	if (uPde2 & X86_PDE_P)
500	{
501	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
502	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
503	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
504	}
505	}
506	break;
507	}
508
509	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
510	{
511	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
512	/*
513	* Hopefully this doesn't happen very often:
514	* - messing with the bits of pd pointers without changing the physical address
515	*/
516	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
517	const unsigned iShw = off / sizeof(X86PDPE);
518	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
519	if (uPdpe & X86_PDPE_P)
520	{
521	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uPdpe));
522	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
523	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
524	}
525	/* paranoia / a bit assumptive. */
526	if ( (off & 7)
527	&& (off & 7) + cbWrite > sizeof(X86PDPE))
528	{
529	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
530	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
531	if (uPdpe2 & X86_PDPE_P)
532	{
533	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpe2));
534	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
535	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
536	}
537	}
538	break;
539	}
540
541	case PGMPOOLKIND_64BIT_PML4:
542	{
543	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
544	/*
545	* Hopefully this doesn't happen very often:
546	* - messing with the bits of pd pointers without changing the physical address
547	*/
548	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
549	const unsigned iShw = off / sizeof(X86PDPE);
550	X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
551	if (uPml4e & X86_PML4E_P)
552	{
553	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uPml4e));
554	pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
555	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
556	}
557	/* paranoia / a bit assumptive. */
558	if ( (off & 7)
559	&& (off & 7) + cbWrite > sizeof(X86PDPE))
560	{
561	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
562	X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
563	if (uPml4e2 & X86_PML4E_P)
564	{
565	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
566	pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
567	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
568	}
569	}
570	break;
571	}
572
573	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
574	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
575	{
576	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
577	const unsigned iShw = off / sizeof(EPTPML4E);
578	X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
579	if (uPml4e & EPT_PRESENT_MASK)
580	{
581	Log7Func(("PML4 iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPml4e, pPage->GCPhys));
582	pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
583	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
584	}
585
586	/* paranoia / a bit assumptive. */
587	if ( (off & 7)
588	&& (off & 7) + cbWrite > sizeof(X86PML4E))
589	{
590	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
591	X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
592	if (uPml4e2 & EPT_PRESENT_MASK)
593	{
594	Log7Func(("PML4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
595	pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
596	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
597	}
598	}
599	break;
600	}
601
602	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
603	{
604	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
605	const unsigned iShw = off / sizeof(EPTPDPTE);
606	X86PGPAEUINT const uPdpte = uShw.pEptPdpt->a[iShw].u;
607	if (uPdpte & EPT_PRESENT_MASK)
608	{
609	Log7Func(("EPT PDPT iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPdpte, pPage->GCPhys));
610	pgmPoolFree(pVM, uPdpte & EPT_PDPTE_PG_MASK, pPage->idx, iShw);
611	ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw].u, 0);
612	}
613
614	/* paranoia / a bit assumptive. */
615	if ( (off & 7)
616	&& (off & 7) + cbWrite > sizeof(EPTPDPTE))
617	{
618	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDPTE);
619	X86PGPAEUINT const uPdpte2 = uShw.pEptPdpt->a[iShw2].u;
620	if (uPdpte2 & EPT_PRESENT_MASK)
621	{
622	Log7Func(("EPT PDPT iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpte2));
623	pgmPoolFree(pVM, uPdpte2 & EPT_PDPTE_PG_MASK, pPage->idx, iShw2);
624	ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw2].u, 0);
625	}
626	}
627	break;
628	}
629
630	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
631	{
632	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
633	const unsigned iShw = off / sizeof(EPTPDE);
634	X86PGPAEUINT const uPde = uShw.pEptPd->a[iShw].u;
635	if (uPde & EPT_PRESENT_MASK)
636	{
637	Assert(!(uPde & EPT_E_LEAF));
638	Log7Func(("EPT PD iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPde, pPage->GCPhys));
639	pgmPoolFree(pVM, uPde & EPT_PDE_PG_MASK, pPage->idx, iShw);
640	ASMAtomicWriteU64(&uShw.pEptPd->a[iShw].u, 0);
641	}
642
643	/* paranoia / a bit assumptive. */
644	if ( (off & 7)
645	&& (off & 7) + cbWrite > sizeof(EPTPDE))
646	{
647	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDE);
648	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPd->a));
649	X86PGPAEUINT const uPde2 = uShw.pEptPd->a[iShw2].u;
650	if (uPde2 & EPT_PRESENT_MASK)
651	{
652	Assert(!(uPde2 & EPT_E_LEAF));
653	Log7Func(("EPT PD (2): iShw2=%#x: %RX64 (%RGp) -> freeing it!\n", iShw2, uPde2, pPage->GCPhys));
654	pgmPoolFree(pVM, uPde2 & EPT_PDE_PG_MASK, pPage->idx, iShw2);
655	ASMAtomicWriteU64(&uShw.pEptPd->a[iShw2].u, 0);
656	}
657	}
658	break;
659	}
660
661	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
662	{
663	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
664	const unsigned iShw = off / sizeof(EPTPTE);
665	X86PGPAEUINT const uPte = uShw.pEptPt->a[iShw].u;
666	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
667	if (uPte & EPT_PRESENT_MASK)
668	{
669	EPTPTE GstPte;
670	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
671	AssertRC(rc);
672
673	Log7Func(("EPT PT: iShw=%#x %RX64 (%RGp)\n", iShw, uPte, pPage->GCPhys));
674	pgmPoolTracDerefGCPhysHint(pPool, pPage,
675	uShw.pEptPt->a[iShw].u & EPT_PTE_PG_MASK,
676	GstPte.u & EPT_PTE_PG_MASK,
677	iShw);
678	ASMAtomicWriteU64(&uShw.pEptPt->a[iShw].u, 0);
679	}
680
681	/* paranoia / a bit assumptive. */
682	if ( (off & 7)
683	&& (off & 7) + cbWrite > sizeof(EPTPTE))
684	{
685	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPTE);
686	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPt->a));
687	X86PGPAEUINT const uPte2 = uShw.pEptPt->a[iShw2].u;
688	if (uPte2 & EPT_PRESENT_MASK)
689	{
690	EPTPTE GstPte;
691	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
692	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
693	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
694	AssertRC(rc);
695	Log7Func(("EPT PT (2): iShw=%#x %RX64 (%RGp)\n", iShw2, uPte2, pPage->GCPhys));
696	pgmPoolTracDerefGCPhysHint(pPool, pPage,
697	uShw.pEptPt->a[iShw2].u & EPT_PTE_PG_MASK,
698	GstPte.u & EPT_PTE_PG_MASK,
699	iShw2);
700	ASMAtomicWriteU64(&uShw.pEptPt->a[iShw2].u, 0);
701	}
702	}
703	break;
704	}
705	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
706
707	default:
708	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
709	}
710	PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
711
712	/* next */
713	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
714	return;
715	pPage = &pPool->aPages[pPage->iMonitoredNext];
716	}
717	}
718
719	#ifndef IN_RING3
720
721	/**
722	* Checks if a access could be a fork operation in progress.
723	*
724	* Meaning, that the guest is setting up the parent process for Copy-On-Write.
725	*
726	* @returns true if it's likely that we're forking, otherwise false.
727	* @param pPool The pool.
728	* @param pDis The disassembled instruction.
729	* @param offFault The access offset.
730	*/
731	DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISSTATE pDis, unsigned offFault)
732	{
733	/*
734	* i386 linux is using btr to clear X86_PTE_RW.
735	* The functions involved are (2.6.16 source inspection):
736	* clear_bit
737	* ptep_set_wrprotect
738	* copy_one_pte
739	* copy_pte_range
740	* copy_pmd_range
741	* copy_pud_range
742	* copy_page_range
743	* dup_mmap
744	* dup_mm
745	* copy_mm
746	* copy_process
747	* do_fork
748	*/
749	if ( pDis->pCurInstr->uOpcode == OP_BTR
750	&& !(offFault & 4)
751	/** @todo Validate that the bit index is X86_PTE_RW. */
752	)
753	{
754	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
755	return true;
756	}
757	return false;
758	}
759
760
761	/**
762	* Determine whether the page is likely to have been reused.
763	*
764	* @returns true if we consider the page as being reused for a different purpose.
765	* @returns false if we consider it to still be a paging page.
766	* @param pVM The cross context VM structure.
767	* @param pVCpu The cross context virtual CPU structure.
768	* @param pCtx Pointer to the register context for the CPU.
769	* @param pDis The disassembly info for the faulting instruction.
770	* @param pvFault The fault address.
771	* @param pPage The pool page being accessed.
772	*
773	* @remark The REP prefix check is left to the caller because of STOSD/W.
774	*/
775	DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISSTATE pDis, RTGCPTR pvFault,
776	PPGMPOOLPAGE pPage)
777	{
778	/* Locked (CR3, PDPTR*4) should not be reusable. Considering them as
779	such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
780	if (pPage->cLocked)
781	{
782	Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
783	return false;
784	}
785
786	/** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
787	if ( HMHasPendingIrq(pVM)
788	&& pCtx->rsp - pvFault < 32)
789	{
790	/* Fault caused by stack writes while trying to inject an interrupt event. */
791	Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pCtx->rsp));
792	return true;
793	}
794
795	LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pCtx->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse, pDis->Param1.x86.Base.idxGenReg));
796
797	/* Non-supervisor mode write means it's used for something else. */
798	if (CPUMGetGuestCPL(pVCpu) == 3)
799	return true;
800
801	switch (pDis->pCurInstr->uOpcode)
802	{
803	/* call implies the actual push of the return address faulted */
804	case OP_CALL:
805	Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
806	return true;
807	case OP_PUSH:
808	Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
809	return true;
810	case OP_PUSHF:
811	Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
812	return true;
813	case OP_PUSHA:
814	Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
815	return true;
816	case OP_FXSAVE:
817	Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
818	return true;
819	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
820	Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
821	return true;
822	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
823	Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
824	return true;
825	case OP_MOVSWD:
826	case OP_STOSWD:
827	if ( pDis->x86.fPrefix == (DISPREFIX_REP\|DISPREFIX_REX)
828	&& pCtx->rcx >= 0x40
829	)
830	{
831	Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
832
833	Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
834	return true;
835	}
836	break;
837
838	default:
839	/*
840	* Anything having ESP on the left side means stack writes.
841	*/
842	if ( ( (pDis->Param1.fUse & DISUSE_REG_GEN32)
843	\|\| (pDis->Param1.fUse & DISUSE_REG_GEN64))
844	&& (pDis->Param1.x86.Base.idxGenReg == DISGREG_ESP))
845	{
846	Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
847	return true;
848	}
849	break;
850	}
851
852	/*
853	* Page table updates are very very unlikely to be crossing page boundraries,
854	* and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
855	*/
856	uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
857	if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
858	{
859	Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
860	return true;
861	}
862
863	/*
864	* Nobody does an unaligned 8 byte write to a page table, right.
865	*/
866	if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
867	{
868	Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
869	return true;
870	}
871
872	return false;
873	}
874
875
876	/**
877	* Flushes the page being accessed.
878	*
879	* @returns VBox status code suitable for scheduling.
880	* @param pVM The cross context VM structure.
881	* @param pVCpu The cross context virtual CPU structure.
882	* @param pPool The pool.
883	* @param pPage The pool page (head).
884	* @param pDis The disassembly of the write instruction.
885	* @param pCtx Pointer to the register context for the CPU.
886	* @param GCPhysFault The fault address as guest physical address.
887	* @todo VBOXSTRICTRC
888	*/
889	static int pgmRZPoolAccessPfHandlerFlush(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISSTATE pDis,
890	PCPUMCTX pCtx, RTGCPHYS GCPhysFault)
891	{
892	NOREF(pVM); NOREF(GCPhysFault);
893
894	/*
895	* First, do the flushing.
896	*/
897	pgmPoolMonitorChainFlush(pPool, pPage);
898
899	/*
900	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
901	* Must do this in raw mode (!); XP boot will fail otherwise.
902	*/
903	int rc = VINF_SUCCESS;
904	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pCtx->rip);
905	if (rc2 == VINF_SUCCESS)
906	{ /* do nothing */ }
907	else if (rc2 == VINF_EM_RESCHEDULE)
908	{
909	rc = VBOXSTRICTRC_VAL(rc2);
910	# ifndef IN_RING3
911	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
912	# endif
913	}
914	else if (rc2 == VERR_EM_INTERPRETER)
915	{
916	rc = VINF_EM_RAW_EMULATE_INSTR;
917	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
918	}
919	else if (RT_FAILURE_NP(rc2))
920	rc = VBOXSTRICTRC_VAL(rc2);
921	else
922	AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
923
924	LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
925	return rc;
926	}
927
928
929	/**
930	* Handles the STOSD write accesses.
931	*
932	* @returns VBox status code suitable for scheduling.
933	* @param pVM The cross context VM structure.
934	* @param pPool The pool.
935	* @param pPage The pool page (head).
936	* @param pDis The disassembly of the write instruction.
937	* @param pCtx Pointer to the register context for the CPU.
938	* @param GCPhysFault The fault address as guest physical address.
939	* @param pvFault The fault address.
940	*/
941	DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISSTATE pDis,
942	PCPUMCTX pCtx, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
943	{
944	unsigned uIncrement = pDis->Param1.x86.cb;
945	NOREF(pVM);
946
947	Assert(pDis->uCpuMode == DISCPUMODE_32BIT \|\| pDis->uCpuMode == DISCPUMODE_64BIT);
948	Assert(pCtx->rcx <= 0x20);
949
950	# ifdef VBOX_STRICT
951	if (pDis->x86.uOpMode == DISCPUMODE_32BIT)
952	Assert(uIncrement == 4);
953	else
954	Assert(uIncrement == 8);
955	# endif
956
957	Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
958
959	/*
960	* Increment the modification counter and insert it into the list
961	* of modified pages the first time.
962	*/
963	if (!pPage->cModifications++)
964	pgmPoolMonitorModifiedInsert(pPool, pPage);
965
966	/*
967	* Execute REP STOSD.
968	*
969	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
970	* write situation, meaning that it's safe to write here.
971	*/
972	PVMCPUCC pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
973	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
974	while (pCtx->rcx)
975	{
976	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
977	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pCtx->rax, uIncrement);
978	pu32 += uIncrement;
979	GCPhysFault += uIncrement;
980	pCtx->rdi += uIncrement;
981	pCtx->rcx--;
982	}
983	pCtx->rip += pDis->cbInstr;
984
985	LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
986	return VINF_SUCCESS;
987	}
988
989
990	/**
991	* Handles the simple write accesses.
992	*
993	* @returns VBox status code suitable for scheduling.
994	* @param pVM The cross context VM structure.
995	* @param pVCpu The cross context virtual CPU structure.
996	* @param pPool The pool.
997	* @param pPage The pool page (head).
998	* @param pDis The disassembly of the write instruction.
999	* @param pCtx Pointer to the register context for the CPU.
1000	* @param GCPhysFault The fault address as guest physical address.
1001	* @param pfReused Reused state (in/out)
1002	*/
1003	DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISSTATE pDis,
1004	PCPUMCTX pCtx, RTGCPHYS GCPhysFault, bool *pfReused)
1005	{
1006	Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
1007	NOREF(pVM);
1008	NOREF(pfReused); /* initialized by caller */
1009
1010	/*
1011	* Increment the modification counter and insert it into the list
1012	* of modified pages the first time.
1013	*/
1014	if (!pPage->cModifications++)
1015	pgmPoolMonitorModifiedInsert(pPool, pPage);
1016
1017	/*
1018	* Clear all the pages.
1019	*/
1020	uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
1021	if (cbWrite <= 8)
1022	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
1023	else if (cbWrite <= 16)
1024	{
1025	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
1026	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
1027	}
1028	else
1029	{
1030	Assert(cbWrite <= 32);
1031	for (uint32_t off = 0; off < cbWrite; off += 8)
1032	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
1033	}
1034
1035	/*
1036	* Interpret the instruction.
1037	*/
1038	VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pCtx->rip);
1039	if (RT_SUCCESS(rc))
1040	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
1041	else if (rc == VERR_EM_INTERPRETER)
1042	{
1043	LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
1044	pCtx->cs.Sel, (RTGCPTR)pCtx->rip, pDis->pCurInstr->uOpcode));
1045	rc = VINF_EM_RAW_EMULATE_INSTR;
1046	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
1047	}
1048
1049	# if 0 /* experimental code */
1050	if (rc == VINF_SUCCESS)
1051	{
1052	switch (pPage->enmKind)
1053	{
1054	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1055	{
1056	X86PTEPAE GstPte;
1057	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
1058	AssertRC(rc);
1059
1060	/* Check the new value written by the guest. If present and with a bogus physical address, then
1061	* it's fairly safe to assume the guest is reusing the PT.
1062	*/
1063	if (GstPte.n.u1Present)
1064	{
1065	RTHCPHYS HCPhys = -1;
1066	int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
1067	if (rc != VINF_SUCCESS)
1068	{
1069	*pfReused = true;
1070	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1071	}
1072	}
1073	break;
1074	}
1075	}
1076	}
1077	# endif
1078
1079	LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
1080	return VBOXSTRICTRC_VAL(rc);
1081	}
1082
1083
1084	/**
1085	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
1086	* \#PF access handler callback for page table pages.}
1087	*
1088	* @remarks The @a uUser argument is the index of the PGMPOOLPAGE.
1089	*/
1090	DECLCALLBACK(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
1091	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
1092	{
1093	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
1094	PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool);
1095	AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE);
1096	PPGMPOOLPAGE const pPage = &pPool->aPages[uUser];
1097	unsigned cMaxModifications;
1098	bool fForcedFlush = false;
1099	RT_NOREF_PV(uErrorCode);
1100
1101	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1102	AssertMsg(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT,
1103	("pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1104	# endif
1105	LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1106
1107	PGM_LOCK_VOID(pVM);
1108	if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1109	{
1110	/* Pool page changed while we were waiting for the lock; ignore. */
1111	Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1112	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1113	PGM_UNLOCK(pVM);
1114	return VINF_SUCCESS;
1115	}
1116	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1117	if (pPage->fDirty)
1118	{
1119	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1120	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
1121	# endif
1122	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
1123	PGM_UNLOCK(pVM);
1124	return VINF_SUCCESS; /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
1125	}
1126	# endif
1127
1128	# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
1129	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1130	{
1131	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
1132	void *pvGst;
1133	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1134	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1135	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1136	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1137	}
1138	# endif
1139
1140	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1141	if (PGMPOOL_PAGE_IS_NESTED(pPage))
1142	{
1143	Assert(!CPUMIsGuestInVmxNonRootMode(CPUMQueryGuestCtxPtr(pVCpu)));
1144	Log7Func(("Flushing pvFault=%RGv GCPhysFault=%RGp\n", pvFault, GCPhysFault));
1145	pgmPoolMonitorChainFlush(pPool, pPage);
1146	PGM_UNLOCK(pVM);
1147	return VINF_SUCCESS;
1148	}
1149	# endif
1150
1151	/*
1152	* Disassemble the faulting instruction.
1153	*/
1154	PDISSTATE pDis = &pVCpu->pgm.s.Dis;
1155	int rc = EMInterpretDisasCurrent(pVCpu, pDis, NULL);
1156	if (RT_UNLIKELY(rc != VINF_SUCCESS))
1157	{
1158	AssertMsg(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
1159	PGM_UNLOCK(pVM);
1160	return rc;
1161	}
1162
1163	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1164
1165	/*
1166	* We should ALWAYS have the list head as user parameter. This
1167	* is because we use that page to record the changes.
1168	*/
1169	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1170
1171	# ifdef IN_RING0
1172	/* Maximum nr of modifications depends on the page type. */
1173	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1174	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1175	cMaxModifications = 4;
1176	else
1177	cMaxModifications = 24;
1178	# else
1179	cMaxModifications = 48;
1180	# endif
1181
1182	/*
1183	* Incremental page table updates should weigh more than random ones.
1184	* (Only applies when started from offset 0)
1185	*/
1186	pVCpu->pgm.s.cPoolAccessHandler++;
1187	if ( pPage->GCPtrLastAccessHandlerRip >= pCtx->rip - 0x40 /* observed loops in Windows 7 x64 */
1188	&& pPage->GCPtrLastAccessHandlerRip < pCtx->rip + 0x40
1189	&& pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.x86.cb)
1190	&& pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
1191	{
1192	Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1193	Assert(pPage->cModifications < 32000);
1194	pPage->cModifications = pPage->cModifications * 2;
1195	pPage->GCPtrLastAccessHandlerFault = pvFault;
1196	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1197	if (pPage->cModifications >= cMaxModifications)
1198	{
1199	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
1200	fForcedFlush = true;
1201	}
1202	}
1203
1204	if (pPage->cModifications >= cMaxModifications)
1205	Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1206
1207	/*
1208	* Check if it's worth dealing with.
1209	*/
1210	bool fReused = false;
1211	bool fNotReusedNotForking = false;
1212	if ( ( pPage->cModifications < cMaxModifications /** @todo \#define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1213	\|\| pgmPoolIsPageLocked(pPage)
1214	)
1215	&& !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pCtx, pDis, pvFault, pPage))
1216	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1217	{
1218	/*
1219	* Simple instructions, no REP prefix.
1220	*/
1221	if (!(pDis->x86.fPrefix & (DISPREFIX_REP \| DISPREFIX_REPNE)))
1222	{
1223	rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pCtx, GCPhysFault, &fReused);
1224	if (fReused)
1225	goto flushPage;
1226
1227	/* A mov instruction to change the first page table entry will be remembered so we can detect
1228	* full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1229	*/
1230	if ( rc == VINF_SUCCESS
1231	&& !pPage->cLocked /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
1232	&& pDis->pCurInstr->uOpcode == OP_MOV
1233	&& (pvFault & PAGE_OFFSET_MASK) == 0)
1234	{
1235	pPage->GCPtrLastAccessHandlerFault = pvFault;
1236	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1237	pPage->GCPtrLastAccessHandlerRip = pCtx->rip;
1238	/* Make sure we don't kick out a page too quickly. */
1239	if (pPage->cModifications > 8)
1240	pPage->cModifications = 2;
1241	}
1242	else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
1243	{
1244	/* ignore the 2nd write to this page table entry. */
1245	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1246	}
1247	else
1248	{
1249	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
1250	pPage->GCPtrLastAccessHandlerRip = 0;
1251	}
1252
1253	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1254	PGM_UNLOCK(pVM);
1255	return rc;
1256	}
1257
1258	/*
1259	* Windows is frequently doing small memset() operations (netio test 4k+).
1260	* We have to deal with these or we'll kill the cache and performance.
1261	*/
1262	if ( pDis->pCurInstr->uOpcode == OP_STOSWD
1263	&& !pCtx->eflags.Bits.u1DF
1264	&& pDis->x86.uOpMode == pDis->uCpuMode
1265	&& pDis->x86.uAddrMode == pDis->uCpuMode)
1266	{
1267	bool fValidStosd = false;
1268
1269	if ( pDis->uCpuMode == DISCPUMODE_32BIT
1270	&& pDis->x86.fPrefix == DISPREFIX_REP
1271	&& pCtx->ecx <= 0x20
1272	&& pCtx->ecx * 4 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK)
1273	&& !((uintptr_t)pvFault & 3)
1274	&& (pCtx->eax == 0 \|\| pCtx->eax == 0x80) /* the two values observed. */
1275	)
1276	{
1277	fValidStosd = true;
1278	pCtx->rcx &= 0xffffffff; /* paranoia */
1279	}
1280	else
1281	if ( pDis->uCpuMode == DISCPUMODE_64BIT
1282	&& pDis->x86.fPrefix == (DISPREFIX_REP \| DISPREFIX_REX)
1283	&& pCtx->rcx <= 0x20
1284	&& pCtx->rcx * 8 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK)
1285	&& !((uintptr_t)pvFault & 7)
1286	&& (pCtx->rax == 0 \|\| pCtx->rax == 0x80) /* the two values observed. */
1287	)
1288	{
1289	fValidStosd = true;
1290	}
1291
1292	if (fValidStosd)
1293	{
1294	rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pCtx, GCPhysFault, pvFault);
1295	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
1296	PGM_UNLOCK(pVM);
1297	return rc;
1298	}
1299	}
1300
1301	/* REP prefix, don't bother. */
1302	STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
1303	Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1304	pCtx->eax, pCtx->ecx, pCtx->edi, pCtx->esi, (RTGCPTR)pCtx->rip, pDis->pCurInstr->uOpcode, pDis->x86.fPrefix));
1305	fNotReusedNotForking = true;
1306	}
1307
1308	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
1309	/* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1310	* leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1311	*/
1312	if ( pPage->cModifications >= cMaxModifications
1313	&& !fForcedFlush
1314	&& (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT \|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1315	&& ( fNotReusedNotForking
1316	\|\| ( !pgmRZPoolMonitorIsReused(pVM, pVCpu, pCtx, pDis, pvFault, pPage)
1317	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1318	)
1319	)
1320	{
1321	Assert(!pgmPoolIsPageLocked(pPage));
1322	Assert(pPage->fDirty == false);
1323
1324	/* Flush any monitored duplicates as we will disable write protection. */
1325	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1326	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1327	{
1328	PPGMPOOLPAGE pPageHead = pPage;
1329
1330	/* Find the monitor head. */
1331	while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1332	pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1333
1334	while (pPageHead)
1335	{
1336	unsigned idxNext = pPageHead->iMonitoredNext;
1337
1338	if (pPageHead != pPage)
1339	{
1340	STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1341	Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1342	int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1343	AssertRC(rc2);
1344	}
1345
1346	if (idxNext == NIL_PGMPOOL_IDX)
1347	break;
1348
1349	pPageHead = &pPool->aPages[idxNext];
1350	}
1351	}
1352
1353	/* The flushing above might fail for locked pages, so double check. */
1354	if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX
1355	&& pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1356	{
1357	pgmPoolAddDirtyPage(pVM, pPool, pPage);
1358
1359	/* Temporarily allow write access to the page table again. */
1360	rc = PGMHandlerPhysicalPageTempOff(pVM,
1361	pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK,
1362	pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
1363	if (rc == VINF_SUCCESS)
1364	{
1365	rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
1366	AssertMsg(rc == VINF_SUCCESS
1367	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1368	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1369	\|\| rc == VERR_PAGE_NOT_PRESENT,
1370	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1371	# ifdef VBOX_STRICT
1372	pPage->GCPtrDirtyFault = pvFault;
1373	# endif
1374
1375	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
1376	PGM_UNLOCK(pVM);
1377	return rc;
1378	}
1379	}
1380	}
1381	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT && IN_RING0 */
1382
1383	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
1384	flushPage:
1385	/*
1386	* Not worth it, so flush it.
1387	*
1388	* If we considered it to be reused, don't go back to ring-3
1389	* to emulate failed instructions since we usually cannot
1390	* interpret then. This may be a bit risky, in which case
1391	* the reuse detection must be fixed.
1392	*/
1393	rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pCtx, GCPhysFault);
1394	if ( rc == VINF_EM_RAW_EMULATE_INSTR
1395	&& fReused)
1396	{
1397	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
1398	/* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
1399	if (PGMShwGetPage(pVCpu, pCtx->rip, NULL, NULL) == VINF_SUCCESS)
1400	rc = VINF_SUCCESS; /* safe to restart the instruction. */
1401	}
1402	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
1403	PGM_UNLOCK(pVM);
1404	return rc;
1405	}
1406
1407	#endif /* !IN_RING3 */
1408
1409	/**
1410	* @callback_method_impl{FNPGMPHYSHANDLER,
1411	* Access handler for shadowed page table pages.}
1412	*
1413	* @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
1414	* @note The @a uUser argument is the index of the PGMPOOLPAGE.
1415	*/
1416	DECLCALLBACK(VBOXSTRICTRC)
1417	pgmPoolAccessHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
1418	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
1419	{
1420	PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool);
1421	STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
1422	AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE);
1423	PPGMPOOLPAGE const pPage = &pPool->aPages[uUser];
1424	LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
1425	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
1426
1427	NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
1428
1429	PGM_LOCK_VOID(pVM);
1430
1431	#ifdef VBOX_WITH_STATISTICS
1432	/*
1433	* Collect stats on the access.
1434	*/
1435	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
1436	if (cbBuf <= 16 && cbBuf > 0)
1437	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
1438	else if (cbBuf >= 17 && cbBuf < 32)
1439	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
1440	else if (cbBuf >= 32 && cbBuf < 64)
1441	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
1442	else if (cbBuf >= 64)
1443	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
1444
1445	uint8_t cbAlign;
1446	switch (pPage->enmKind)
1447	{
1448	default:
1449	cbAlign = 7;
1450	break;
1451	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1452	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1453	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1454	case PGMPOOLKIND_32BIT_PD:
1455	case PGMPOOLKIND_32BIT_PD_PHYS:
1456	cbAlign = 3;
1457	break;
1458	}
1459	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
1460	if ((uint8_t)GCPhys & cbAlign)
1461	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
1462	#endif
1463
1464	/*
1465	* Make sure the pool page wasn't modified by a different CPU.
1466	*/
1467	if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
1468	{
1469	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1470
1471	/* The max modification count before flushing depends on the context and page type. */
1472	#ifdef IN_RING3
1473	uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
1474	#else
1475	uint16_t cMaxModifications;
1476	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1477	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1478	cMaxModifications = 4;
1479	else
1480	cMaxModifications = 24;
1481	#endif
1482
1483	/*
1484	* We don't have to be very sophisticated about this since there are relativly few calls here.
1485	* However, we must try our best to detect any non-cpu accesses (disk / networking).
1486	*/
1487	if ( ( pPage->cModifications < cMaxModifications
1488	\|\| pgmPoolIsPageLocked(pPage) )
1489	&& enmOrigin != PGMACCESSORIGIN_DEVICE
1490	&& cbBuf <= 16)
1491	{
1492	/* Clear the shadow entry. */
1493	if (!pPage->cModifications++)
1494	pgmPoolMonitorModifiedInsert(pPool, pPage);
1495
1496	if (cbBuf <= 8)
1497	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
1498	else
1499	{
1500	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
1501	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
1502	}
1503	}
1504	else
1505	pgmPoolMonitorChainFlush(pPool, pPage);
1506
1507	STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1508	}
1509	else
1510	Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1511	PGM_UNLOCK(pVM);
1512	return VINF_PGM_HANDLER_DO_DEFAULT;
1513	}
1514
1515
1516	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1517
1518	# if defined(VBOX_STRICT) && !defined(IN_RING3)
1519
1520	/**
1521	* Check references to guest physical memory in a PAE / PAE page table.
1522	*
1523	* @param pPool The pool.
1524	* @param pPage The page.
1525	* @param pShwPT The shadow page table (mapping of the page).
1526	* @param pGstPT The guest page table.
1527	*/
1528	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
1529	{
1530	unsigned cErrors = 0;
1531	int LastRc = -1; /* initialized to shut up gcc */
1532	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1533	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1534	PVMCC pVM = pPool->CTX_SUFF(pVM);
1535
1536	# ifdef VBOX_STRICT
1537	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1538	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1539	# endif
1540	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1541	{
1542	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1543	{
1544	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1545	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1546	if ( rc != VINF_SUCCESS
1547	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1548	{
1549	Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1550	LastPTE = i;
1551	LastRc = rc;
1552	LastHCPhys = HCPhys;
1553	cErrors++;
1554
1555	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1556	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1557	AssertRC(rc);
1558
1559	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1560	{
1561	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1562
1563	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1564	{
1565	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1566
1567	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1568	{
1569	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1570	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1571	{
1572	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1573	}
1574	}
1575
1576	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1577	}
1578	}
1579	}
1580	}
1581	}
1582	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1583	}
1584
1585
1586	/**
1587	* Check references to guest physical memory in a PAE / 32-bit page table.
1588	*
1589	* @param pPool The pool.
1590	* @param pPage The page.
1591	* @param pShwPT The shadow page table (mapping of the page).
1592	* @param pGstPT The guest page table.
1593	*/
1594	static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
1595	{
1596	unsigned cErrors = 0;
1597	int LastRc = -1; /* initialized to shut up gcc */
1598	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1599	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1600	PVMCC pVM = pPool->CTX_SUFF(pVM);
1601
1602	# ifdef VBOX_STRICT
1603	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1604	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1605	# endif
1606	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1607	{
1608	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1609	{
1610	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1611	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1612	if ( rc != VINF_SUCCESS
1613	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1614	{
1615	Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1616	LastPTE = i;
1617	LastRc = rc;
1618	LastHCPhys = HCPhys;
1619	cErrors++;
1620
1621	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1622	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1623	AssertRC(rc);
1624
1625	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1626	{
1627	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1628
1629	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1630	{
1631	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1632
1633	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1634	{
1635	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1636	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1637	{
1638	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1639	}
1640	}
1641
1642	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1643	}
1644	}
1645	}
1646	}
1647	}
1648	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1649	}
1650
1651	# endif /* VBOX_STRICT && !IN_RING3 */
1652
1653	/**
1654	* Clear references to guest physical memory in a PAE / PAE page table.
1655	*
1656	* @returns nr of changed PTEs
1657	* @param pPool The pool.
1658	* @param pPage The page.
1659	* @param pShwPT The shadow page table (mapping of the page).
1660	* @param pGstPT The guest page table.
1661	* @param pOldGstPT The old cached guest page table.
1662	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1663	* @param pfFlush Flush reused page table (out)
1664	*/
1665	DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
1666	PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1667	{
1668	unsigned cChanged = 0;
1669
1670	# ifdef VBOX_STRICT
1671	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1672	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1673	# endif
1674	*pfFlush = false;
1675
1676	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1677	{
1678	/* Check the new value written by the guest. If present and with a bogus physical address, then
1679	* it's fairly safe to assume the guest is reusing the PT.
1680	*/
1681	if ( fAllowRemoval
1682	&& (pGstPT->a[i].u & X86_PTE_P))
1683	{
1684	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1685	{
1686	*pfFlush = true;
1687	return ++cChanged;
1688	}
1689	}
1690	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1691	{
1692	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1693	if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1694	{
1695	# ifdef VBOX_STRICT
1696	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1697	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1698	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1699	# endif
1700	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1701	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1702	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1703	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1704
1705	if ( uHostAttr == uGuestAttr
1706	&& fHostRW <= fGuestRW)
1707	continue;
1708	}
1709	cChanged++;
1710	/* Something was changed, so flush it. */
1711	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
1712	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1713	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
1714	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1715	}
1716	}
1717	return cChanged;
1718	}
1719
1720
1721	/**
1722	* Clear references to guest physical memory in a PAE / PAE page table.
1723	*
1724	* @returns nr of changed PTEs
1725	* @param pPool The pool.
1726	* @param pPage The page.
1727	* @param pShwPT The shadow page table (mapping of the page).
1728	* @param pGstPT The guest page table.
1729	* @param pOldGstPT The old cached guest page table.
1730	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1731	* @param pfFlush Flush reused page table (out)
1732	*/
1733	DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
1734	PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1735	{
1736	unsigned cChanged = 0;
1737
1738	# ifdef VBOX_STRICT
1739	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1740	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1741	# endif
1742	*pfFlush = false;
1743
1744	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1745	{
1746	/* Check the new value written by the guest. If present and with a bogus physical address, then
1747	* it's fairly safe to assume the guest is reusing the PT. */
1748	if (fAllowRemoval)
1749	{
1750	X86PGUINT const uPte = pGstPT->a[i].u;
1751	if ( (uPte & X86_PTE_P)
1752	&& !PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), uPte & X86_PTE_PG_MASK))
1753	{
1754	*pfFlush = true;
1755	return ++cChanged;
1756	}
1757	}
1758	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1759	{
1760	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1761	if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
1762	{
1763	# ifdef VBOX_STRICT
1764	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1765	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1766	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1767	# endif
1768	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1769	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1770	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1771	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1772
1773	if ( uHostAttr == uGuestAttr
1774	&& fHostRW <= fGuestRW)
1775	continue;
1776	}
1777	cChanged++;
1778	/* Something was changed, so flush it. */
1779	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
1780	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
1781	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
1782	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1783	}
1784	}
1785	return cChanged;
1786	}
1787
1788
1789	/**
1790	* Flush a dirty page
1791	*
1792	* @param pVM The cross context VM structure.
1793	* @param pPool The pool.
1794	* @param idxSlot Dirty array slot index
1795	* @param fAllowRemoval Allow a reused page table to be removed
1796	*/
1797	static void pgmPoolFlushDirtyPage(PVMCC pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
1798	{
1799	AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages));
1800
1801	Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
1802	unsigned idxPage = pPool->aidxDirtyPages[idxSlot];
1803	if (idxPage == NIL_PGMPOOL_IDX)
1804	return;
1805
1806	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1807	Assert(pPage->idx == idxPage);
1808	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1809
1810	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1811	Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1812
1813	/* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
1814	int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
1815	Assert(rc == VINF_SUCCESS);
1816	pPage->fDirty = false;
1817
1818	# ifdef VBOX_STRICT
1819	uint64_t fFlags = 0;
1820	RTHCPHYS HCPhys;
1821	rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
1822	AssertMsg( ( rc == VINF_SUCCESS
1823	&& (!(fFlags & X86_PTE_RW) \|\| HCPhys != pPage->Core.Key))
1824	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1825	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1826	\|\| rc == VERR_PAGE_NOT_PRESENT,
1827	("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
1828	# endif
1829
1830	/* Flush those PTEs that have changed. */
1831	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1832	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1833	void *pvGst;
1834	rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1835	bool fFlush;
1836	unsigned cChanges;
1837
1838	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1839	cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
1840	(PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1841	else
1842	{
1843	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
1844	cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
1845	(PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1846	}
1847
1848	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1849	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1850	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1851	/* Note: we might want to consider keeping the dirty page active in case there were many changes. */
1852
1853	/* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1854	Assert(pPage->cModifications);
1855	if (cChanges < 4)
1856	pPage->cModifications = 1; /* must use > 0 here */
1857	else
1858	pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1859
1860	STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1861	if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
1862	pPool->idxFreeDirtyPage = idxSlot;
1863
1864	pPool->cDirtyPages--;
1865	pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1866	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1867	if (fFlush)
1868	{
1869	Assert(fAllowRemoval);
1870	Log(("Flush reused page table!\n"));
1871	pgmPoolFlushPage(pPool, pPage);
1872	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1873	}
1874	else
1875	Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
1876	}
1877
1878
1879	# ifndef IN_RING3
1880	/**
1881	* Add a new dirty page
1882	*
1883	* @param pVM The cross context VM structure.
1884	* @param pPool The pool.
1885	* @param pPage The page.
1886	*/
1887	void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1888	{
1889	PGM_LOCK_ASSERT_OWNER(pVM);
1890	AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 \|\| RT_ELEMENTS(pPool->aDirtyPages) == 16);
1891	Assert(!pPage->fDirty);
1892	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
1893
1894	unsigned idxFree = pPool->idxFreeDirtyPage;
1895	Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
1896	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1897
1898	if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
1899	{
1900	STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
1901	pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
1902	}
1903	Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
1904	AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1905
1906	Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1907
1908	/*
1909	* Make a copy of the guest page table as we require valid GCPhys addresses
1910	* when removing references to physical pages.
1911	* (The HCPhys linear lookup is extremely expensive!)
1912	*/
1913	void *pvGst;
1914	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1915	memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst,
1916	pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT ? PAGE_SIZE : PAGE_SIZE / 2);
1917	# ifdef VBOX_STRICT
1918	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1919	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1920	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1921	else
1922	pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
1923	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1924	# endif
1925	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1926
1927	STAM_COUNTER_INC(&pPool->StatDirtyPage);
1928	pPage->fDirty = true;
1929	pPage->idxDirtyEntry = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
1930	pPool->aidxDirtyPages[idxFree] = pPage->idx;
1931	pPool->cDirtyPages++;
1932
1933	pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1934	if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
1935	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1936	{
1937	unsigned i;
1938	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1939	{
1940	idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1941	if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1942	{
1943	pPool->idxFreeDirtyPage = idxFree;
1944	break;
1945	}
1946	}
1947	Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
1948	}
1949
1950	Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) \|\| pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1951
1952	/*
1953	* Clear all references to this shadow table. See @bugref{7298}.
1954	*/
1955	pgmPoolTrackClearPageUsers(pPool, pPage);
1956	}
1957	# endif /* !IN_RING3 */
1958
1959
1960	/**
1961	* Check if the specified page is dirty (not write monitored)
1962	*
1963	* @return dirty or not
1964	* @param pVM The cross context VM structure.
1965	* @param GCPhys Guest physical address
1966	*/
1967	bool pgmPoolIsDirtyPageSlow(PVMCC pVM, RTGCPHYS GCPhys)
1968	{
1969	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1970	PGM_LOCK_ASSERT_OWNER(pVM);
1971	if (!pPool->cDirtyPages)
1972	return false;
1973
1974	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1975
1976	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1977	{
1978	unsigned idxPage = pPool->aidxDirtyPages[i];
1979	if (idxPage != NIL_PGMPOOL_IDX)
1980	{
1981	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1982	if (pPage->GCPhys == GCPhys)
1983	return true;
1984	}
1985	}
1986	return false;
1987	}
1988
1989
1990	/**
1991	* Reset all dirty pages by reinstating page monitoring.
1992	*
1993	* @param pVM The cross context VM structure.
1994	*/
1995	void pgmPoolResetDirtyPages(PVMCC pVM)
1996	{
1997	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1998	PGM_LOCK_ASSERT_OWNER(pVM);
1999	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2000
2001	if (!pPool->cDirtyPages)
2002	return;
2003
2004	Log(("pgmPoolResetDirtyPages\n"));
2005	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2006	pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
2007
2008	pPool->idxFreeDirtyPage = 0;
2009	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
2010	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
2011	{
2012	unsigned i;
2013	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2014	{
2015	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
2016	{
2017	pPool->idxFreeDirtyPage = i;
2018	break;
2019	}
2020	}
2021	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2022	}
2023
2024	Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX \|\| pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
2025	return;
2026	}
2027
2028
2029	/**
2030	* Invalidate the PT entry for the specified page
2031	*
2032	* @param pVM The cross context VM structure.
2033	* @param GCPtrPage Guest page to invalidate
2034	*/
2035	void pgmPoolResetDirtyPage(PVMCC pVM, RTGCPTR GCPtrPage)
2036	{
2037	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2038	PGM_LOCK_ASSERT_OWNER(pVM);
2039	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2040
2041	if (!pPool->cDirtyPages)
2042	return;
2043
2044	Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2045	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2046	{
2047	/** @todo What was intended here??? This looks incomplete... */
2048	}
2049	}
2050
2051
2052	/**
2053	* Reset all dirty pages by reinstating page monitoring.
2054	*
2055	* @param pVM The cross context VM structure.
2056	* @param GCPhysPT Physical address of the page table
2057	*/
2058	void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT)
2059	{
2060	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2061	PGM_LOCK_ASSERT_OWNER(pVM);
2062	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2063	unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
2064
2065	if (!pPool->cDirtyPages)
2066	return;
2067
2068	GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2069
2070	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2071	{
2072	unsigned idxPage = pPool->aidxDirtyPages[i];
2073	if (idxPage != NIL_PGMPOOL_IDX)
2074	{
2075	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
2076	if (pPage->GCPhys == GCPhysPT)
2077	{
2078	idxDirtyPage = i;
2079	break;
2080	}
2081	}
2082	}
2083
2084	if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
2085	{
2086	pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
2087	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
2088	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
2089	{
2090	unsigned i;
2091	for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2092	{
2093	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
2094	{
2095	pPool->idxFreeDirtyPage = i;
2096	break;
2097	}
2098	}
2099	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2100	}
2101	}
2102	}
2103
2104	#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
2105
2106	/**
2107	* Inserts a page into the GCPhys hash table.
2108	*
2109	* @param pPool The pool.
2110	* @param pPage The page.
2111	*/
2112	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2113	{
2114	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
2115	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2116	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2117	pPage->iNext = pPool->aiHash[iHash];
2118	pPool->aiHash[iHash] = pPage->idx;
2119	}
2120
2121
2122	/**
2123	* Removes a page from the GCPhys hash table.
2124	*
2125	* @param pPool The pool.
2126	* @param pPage The page.
2127	*/
2128	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2129	{
2130	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
2131	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2132	if (pPool->aiHash[iHash] == pPage->idx)
2133	pPool->aiHash[iHash] = pPage->iNext;
2134	else
2135	{
2136	uint16_t iPrev = pPool->aiHash[iHash];
2137	for (;;)
2138	{
2139	const int16_t i = pPool->aPages[iPrev].iNext;
2140	if (i == pPage->idx)
2141	{
2142	pPool->aPages[iPrev].iNext = pPage->iNext;
2143	break;
2144	}
2145	if (i == NIL_PGMPOOL_IDX)
2146	{
2147	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
2148	break;
2149	}
2150	iPrev = i;
2151	}
2152	}
2153	pPage->iNext = NIL_PGMPOOL_IDX;
2154	}
2155
2156
2157	/**
2158	* Frees up one cache page.
2159	*
2160	* @returns VBox status code.
2161	* @retval VINF_SUCCESS on success.
2162	* @param pPool The pool.
2163	* @param iUser The user index.
2164	*/
2165	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
2166	{
2167	#ifndef VBOX_VMM_TARGET_ARMV8
2168	const PVMCC pVM = pPool->CTX_SUFF(pVM);
2169	#endif
2170	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
2171	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
2172
2173	/*
2174	* Select one page from the tail of the age list.
2175	*/
2176	PPGMPOOLPAGE pPage;
2177	for (unsigned iLoop = 0; ; iLoop++)
2178	{
2179	uint16_t iToFree = pPool->iAgeTail;
2180	if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
2181	iToFree = pPool->aPages[iToFree].iAgePrev;
2182	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
2183	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
2184	{
2185	uint16_t i = pPool->aPages[iToFree].iAgePrev;
2186	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
2187	{
2188	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
2189	continue;
2190	iToFree = i;
2191	break;
2192	}
2193	}
2194	*/
2195	Assert(iToFree != iUser);
2196	AssertReleaseMsg(iToFree != NIL_PGMPOOL_IDX,
2197	("iToFree=%#x (iAgeTail=%#x) iUser=%#x iLoop=%u - pPool=%p LB %#zx\n",
2198	iToFree, pPool->iAgeTail, iUser, iLoop, pPool,
2199	RT_UOFFSETOF_DYN(PGMPOOL, aPages[pPool->cMaxPages])
2200	+ pPool->cMaxUsers * sizeof(PGMPOOLUSER)
2201	+ pPool->cMaxPhysExts * sizeof(PGMPOOLPHYSEXT) ));
2202
2203	pPage = &pPool->aPages[iToFree];
2204
2205	/*
2206	* Reject any attempts at flushing the currently active shadow CR3 mapping.
2207	* Call pgmPoolCacheUsed to move the page to the head of the age list.
2208	*/
2209	if ( !pgmPoolIsPageLocked(pPage)
2210	&& pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
2211	break;
2212	LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
2213	pgmPoolCacheUsed(pPool, pPage);
2214	AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
2215	}
2216
2217	/*
2218	* Found a usable page, flush it and return.
2219	*/
2220	int rc = pgmPoolFlushPage(pPool, pPage);
2221	/* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
2222	/** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
2223	if (rc == VINF_SUCCESS)
2224	PGM_INVL_ALL_VCPU_TLBS(pVM);
2225	return rc;
2226	}
2227
2228
2229	/**
2230	* Checks if a kind mismatch is really a page being reused
2231	* or if it's just normal remappings.
2232	*
2233	* @returns true if reused and the cached page (enmKind1) should be flushed
2234	* @returns false if not reused.
2235	* @param enmKind1 The kind of the cached page.
2236	* @param enmKind2 The kind of the requested page.
2237	*/
2238	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
2239	{
2240	switch (enmKind1)
2241	{
2242	/*
2243	* Never reuse them. There is no remapping in non-paging mode.
2244	*/
2245	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2246	case PGMPOOLKIND_32BIT_PD_PHYS:
2247	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2248	case PGMPOOLKIND_PAE_PD_PHYS:
2249	case PGMPOOLKIND_PAE_PDPT_PHYS:
2250	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2251	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2252	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2253	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2254	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2255	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
2256	return false;
2257
2258	/*
2259	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2260	*/
2261	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2262	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2263	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2264	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2265	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2266	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2267	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2268	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2269	case PGMPOOLKIND_32BIT_PD:
2270	case PGMPOOLKIND_PAE_PDPT:
2271	Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
2272	switch (enmKind2)
2273	{
2274	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2275	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2276	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2277	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2278	case PGMPOOLKIND_64BIT_PML4:
2279	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2280	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2281	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2282	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2283	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2284	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2285	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2286	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2287	return true;
2288	default:
2289	return false;
2290	}
2291
2292	/*
2293	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2294	*/
2295	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2296	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2297	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2298	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2299	case PGMPOOLKIND_64BIT_PML4:
2300	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2301	Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
2302	switch (enmKind2)
2303	{
2304	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2305	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2306	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2307	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2308	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2309	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2310	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2311	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2312	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2313	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2314	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2315	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2316	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2317	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2318	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2319	return true;
2320	default:
2321	return false;
2322	}
2323
2324	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2325	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2326	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
2327	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2328	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2329	return PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2);
2330
2331	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2332	return false;
2333	#endif
2334
2335	/*
2336	* These cannot be flushed, and it's common to reuse the PDs as PTs.
2337	*/
2338	case PGMPOOLKIND_ROOT_NESTED:
2339	return false;
2340
2341	default:
2342	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
2343	}
2344	}
2345
2346
2347	/**
2348	* Attempts to satisfy a pgmPoolAlloc request from the cache.
2349	*
2350	* @returns VBox status code.
2351	* @retval VINF_PGM_CACHED_PAGE on success.
2352	* @retval VERR_FILE_NOT_FOUND if not found.
2353	* @param pPool The pool.
2354	* @param GCPhys The GC physical address of the page we're gonna shadow.
2355	* @param enmKind The kind of mapping.
2356	* @param enmAccess Access type for the mapping (only relevant for big pages)
2357	* @param fA20Enabled Whether the CPU has the A20 gate enabled.
2358	* @param iUser The shadow page pool index of the user table. This is
2359	* NIL_PGMPOOL_IDX for root pages.
2360	* @param iUserTable The index into the user table (shadowed). Ignored if
2361	* root page
2362	* @param ppPage Where to store the pointer to the page.
2363	*/
2364	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
2365	uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
2366	{
2367	/*
2368	* Look up the GCPhys in the hash.
2369	*/
2370	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2371	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
2372	if (i != NIL_PGMPOOL_IDX)
2373	{
2374	do
2375	{
2376	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2377	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
2378	if (pPage->GCPhys == GCPhys)
2379	{
2380	if ( (PGMPOOLKIND)pPage->enmKind == enmKind
2381	&& (PGMPOOLACCESS)pPage->enmAccess == enmAccess
2382	&& pPage->fA20Enabled == fA20Enabled)
2383	{
2384	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
2385	* doesn't flush it in case there are no more free use records.
2386	*/
2387	pgmPoolCacheUsed(pPool, pPage);
2388
2389	int rc = VINF_SUCCESS;
2390	if (iUser != NIL_PGMPOOL_IDX)
2391	rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
2392	if (RT_SUCCESS(rc))
2393	{
2394	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
2395	*ppPage = pPage;
2396	if (pPage->cModifications)
2397	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
2398	STAM_COUNTER_INC(&pPool->StatCacheHits);
2399	return VINF_PGM_CACHED_PAGE;
2400	}
2401	return rc;
2402	}
2403
2404	if ((PGMPOOLKIND)pPage->enmKind != enmKind)
2405	{
2406	/*
2407	* The kind is different. In some cases we should now flush the page
2408	* as it has been reused, but in most cases this is normal remapping
2409	* of PDs as PT or big pages using the GCPhys field in a slightly
2410	* different way than the other kinds.
2411	*/
2412	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
2413	{
2414	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
2415	pgmPoolFlushPage(pPool, pPage);
2416	break;
2417	}
2418	}
2419	}
2420
2421	/* next */
2422	i = pPage->iNext;
2423	} while (i != NIL_PGMPOOL_IDX);
2424	}
2425
2426	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
2427	STAM_COUNTER_INC(&pPool->StatCacheMisses);
2428	return VERR_FILE_NOT_FOUND;
2429	}
2430
2431
2432	/**
2433	* Inserts a page into the cache.
2434	*
2435	* @param pPool The pool.
2436	* @param pPage The cached page.
2437	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
2438	*/
2439	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
2440	{
2441	/*
2442	* Insert into the GCPhys hash if the page is fit for that.
2443	*/
2444	Assert(!pPage->fCached);
2445	if (fCanBeCached)
2446	{
2447	pPage->fCached = true;
2448	pgmPoolHashInsert(pPool, pPage);
2449	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2450	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2451	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
2452	}
2453	else
2454	{
2455	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2456	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2457	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
2458	}
2459
2460	/*
2461	* Insert at the head of the age list.
2462	*/
2463	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2464	pPage->iAgeNext = pPool->iAgeHead;
2465	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
2466	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
2467	else
2468	pPool->iAgeTail = pPage->idx;
2469	pPool->iAgeHead = pPage->idx;
2470	}
2471
2472
2473	/**
2474	* Flushes a cached page.
2475	*
2476	* @param pPool The pool.
2477	* @param pPage The cached page.
2478	*/
2479	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2480	{
2481	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
2482
2483	/*
2484	* Remove the page from the hash.
2485	*/
2486	if (pPage->fCached)
2487	{
2488	pPage->fCached = false;
2489	pgmPoolHashRemove(pPool, pPage);
2490	}
2491	else
2492	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2493
2494	/*
2495	* Remove it from the age list.
2496	*/
2497	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
2498	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
2499	else
2500	pPool->iAgeTail = pPage->iAgePrev;
2501	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
2502	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
2503	else
2504	pPool->iAgeHead = pPage->iAgeNext;
2505	pPage->iAgeNext = NIL_PGMPOOL_IDX;
2506	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2507	}
2508
2509
2510	/**
2511	* Looks for pages sharing the monitor.
2512	*
2513	* @returns Pointer to the head page.
2514	* @returns NULL if not found.
2515	* @param pPool The Pool
2516	* @param pNewPage The page which is going to be monitored.
2517	*/
2518	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
2519	{
2520	/*
2521	* Look up the GCPhys in the hash.
2522	*/
2523	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2524	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2525	if (i == NIL_PGMPOOL_IDX)
2526	return NULL;
2527	do
2528	{
2529	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2530	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
2531	&& pPage != pNewPage)
2532	{
2533	switch (pPage->enmKind)
2534	{
2535	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2536	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2537	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2538	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2539	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2540	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2541	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2542	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2543	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2544	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2545	case PGMPOOLKIND_64BIT_PML4:
2546	case PGMPOOLKIND_32BIT_PD:
2547	case PGMPOOLKIND_PAE_PDPT:
2548	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2549	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2550	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2551	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2552	#endif
2553	{
2554	/* find the head */
2555	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2556	{
2557	Assert(pPage->iMonitoredPrev != pPage->idx);
2558	pPage = &pPool->aPages[pPage->iMonitoredPrev];
2559	}
2560	return pPage;
2561	}
2562
2563	/* ignore, no monitoring. */
2564	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2565	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2566	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2567	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2568	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2569	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2570	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2571	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2572	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2573	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2574	case PGMPOOLKIND_ROOT_NESTED:
2575	case PGMPOOLKIND_PAE_PD_PHYS:
2576	case PGMPOOLKIND_PAE_PDPT_PHYS:
2577	case PGMPOOLKIND_32BIT_PD_PHYS:
2578	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2579	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2580	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
2581	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2582	#endif
2583	break;
2584	default:
2585	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2586	}
2587	}
2588
2589	/* next */
2590	i = pPage->iNext;
2591	} while (i != NIL_PGMPOOL_IDX);
2592	return NULL;
2593	}
2594
2595
2596	/**
2597	* Enabled write monitoring of a guest page.
2598	*
2599	* @returns VBox status code.
2600	* @retval VINF_SUCCESS on success.
2601	* @param pPool The pool.
2602	* @param pPage The cached page.
2603	*/
2604	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2605	{
2606	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
2607
2608	/*
2609	* Filter out the relevant kinds.
2610	*/
2611	switch (pPage->enmKind)
2612	{
2613	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2614	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2615	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2616	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2617	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2618	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2619	case PGMPOOLKIND_64BIT_PML4:
2620	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2621	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2622	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2623	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2624	case PGMPOOLKIND_32BIT_PD:
2625	case PGMPOOLKIND_PAE_PDPT:
2626	break;
2627
2628	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2629	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2630	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2631	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2632	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2633	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2634	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2635	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2636	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2637	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2638	case PGMPOOLKIND_ROOT_NESTED:
2639	/* Nothing to monitor here. */
2640	return VINF_SUCCESS;
2641
2642	case PGMPOOLKIND_32BIT_PD_PHYS:
2643	case PGMPOOLKIND_PAE_PDPT_PHYS:
2644	case PGMPOOLKIND_PAE_PD_PHYS:
2645	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2646	/* Nothing to monitor here. */
2647	return VINF_SUCCESS;
2648
2649	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2650	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2651	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2652	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2653	break;
2654
2655	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
2656	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2657	/* Nothing to monitor here. */
2658	return VINF_SUCCESS;
2659	#endif
2660
2661	default:
2662	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2663	}
2664
2665	/*
2666	* Install handler.
2667	*/
2668	int rc;
2669	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2670	if (pPageHead)
2671	{
2672	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2673	Assert(pPageHead->iMonitoredPrev != pPage->idx);
2674
2675	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2676	if (pPageHead->fDirty)
2677	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
2678	#endif
2679
2680	pPage->iMonitoredPrev = pPageHead->idx;
2681	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2682	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2683	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2684	pPageHead->iMonitoredNext = pPage->idx;
2685	rc = VINF_SUCCESS;
2686	if (PGMPOOL_PAGE_IS_NESTED(pPage))
2687	Log7Func(("Adding to monitoring list GCPhysPage=%RGp\n", pPage->GCPhys));
2688	}
2689	else
2690	{
2691	if (PGMPOOL_PAGE_IS_NESTED(pPage))
2692	Log7Func(("Started monitoring GCPhysPage=%RGp HCPhys=%RHp enmKind=%s\n", pPage->GCPhys, pPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
2693
2694	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2695	PVMCC pVM = pPool->CTX_SUFF(pVM);
2696	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2697	rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
2698	pPage - &pPool->aPages[0], NIL_RTR3PTR /pszDesc/);
2699	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2700	* the heap size should suffice. */
2701	AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2702	PVMCPU pVCpu = VMMGetCpu(pVM);
2703	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
2704	}
2705	pPage->fMonitored = true;
2706	return rc;
2707	}
2708
2709
2710	/**
2711	* Disables write monitoring of a guest page.
2712	*
2713	* @returns VBox status code.
2714	* @retval VINF_SUCCESS on success.
2715	* @param pPool The pool.
2716	* @param pPage The cached page.
2717	*/
2718	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2719	{
2720	/*
2721	* Filter out the relevant kinds.
2722	*/
2723	switch (pPage->enmKind)
2724	{
2725	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2726	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2727	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2728	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2729	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2730	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2731	case PGMPOOLKIND_64BIT_PML4:
2732	case PGMPOOLKIND_32BIT_PD:
2733	case PGMPOOLKIND_PAE_PDPT:
2734	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2735	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2736	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2737	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2738	break;
2739
2740	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2741	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2742	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2743	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2744	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2745	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2746	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2747	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2748	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2749	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2750	case PGMPOOLKIND_ROOT_NESTED:
2751	case PGMPOOLKIND_PAE_PD_PHYS:
2752	case PGMPOOLKIND_PAE_PDPT_PHYS:
2753	case PGMPOOLKIND_32BIT_PD_PHYS:
2754	/* Nothing to monitor here. */
2755	Assert(!pPage->fMonitored);
2756	return VINF_SUCCESS;
2757
2758	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2759	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2760	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2761	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2762	break;
2763
2764	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
2765	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2766	/* Nothing to monitor here. */
2767	Assert(!pPage->fMonitored);
2768	return VINF_SUCCESS;
2769	#endif
2770
2771	default:
2772	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2773	}
2774	Assert(pPage->fMonitored);
2775
2776	/*
2777	* Remove the page from the monitored list or uninstall it if last.
2778	*/
2779	const PVMCC pVM = pPool->CTX_SUFF(pVM);
2780	int rc;
2781	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2782	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2783	{
2784	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2785	{
2786	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2787	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2788	rc = PGMHandlerPhysicalChangeUserArg(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK, pPage->iMonitoredNext);
2789
2790	AssertFatalRCSuccess(rc);
2791	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2792	}
2793	else
2794	{
2795	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2796	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2797	{
2798	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2799	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2800	}
2801	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2802	rc = VINF_SUCCESS;
2803	}
2804	}
2805	else
2806	{
2807	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
2808	AssertFatalRC(rc);
2809	PVMCPU pVCpu = VMMGetCpu(pVM);
2810	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2811	("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2812	}
2813	pPage->fMonitored = false;
2814
2815	/*
2816	* Remove it from the list of modified pages (if in it).
2817	*/
2818	pgmPoolMonitorModifiedRemove(pPool, pPage);
2819
2820	if (PGMPOOL_PAGE_IS_NESTED(pPage))
2821	Log7Func(("Stopped monitoring %RGp\n", pPage->GCPhys));
2822
2823	return rc;
2824	}
2825
2826
2827	/**
2828	* Inserts the page into the list of modified pages.
2829	*
2830	* @param pPool The pool.
2831	* @param pPage The page.
2832	*/
2833	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2834	{
2835	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2836	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2837	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2838	&& pPool->iModifiedHead != pPage->idx,
2839	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2840	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2841	pPool->iModifiedHead, pPool->cModifiedPages));
2842
2843	pPage->iModifiedNext = pPool->iModifiedHead;
2844	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2845	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2846	pPool->iModifiedHead = pPage->idx;
2847	pPool->cModifiedPages++;
2848	#ifdef VBOX_WITH_STATISTICS
2849	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2850	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2851	#endif
2852	}
2853
2854
2855	/**
2856	* Removes the page from the list of modified pages and resets the
2857	* modification counter.
2858	*
2859	* @param pPool The pool.
2860	* @param pPage The page which is believed to be in the list of modified pages.
2861	*/
2862	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2863	{
2864	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2865	if (pPool->iModifiedHead == pPage->idx)
2866	{
2867	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2868	pPool->iModifiedHead = pPage->iModifiedNext;
2869	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2870	{
2871	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2872	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2873	}
2874	pPool->cModifiedPages--;
2875	}
2876	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2877	{
2878	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2879	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2880	{
2881	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2882	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2883	}
2884	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2885	pPool->cModifiedPages--;
2886	}
2887	else
2888	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2889	pPage->cModifications = 0;
2890	}
2891
2892
2893	/**
2894	* Zaps the list of modified pages, resetting their modification counters in the process.
2895	*
2896	* @param pVM The cross context VM structure.
2897	*/
2898	static void pgmPoolMonitorModifiedClearAll(PVMCC pVM)
2899	{
2900	PGM_LOCK_VOID(pVM);
2901	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2902	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2903
2904	unsigned cPages = 0; NOREF(cPages);
2905
2906	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2907	pgmPoolResetDirtyPages(pVM);
2908	#endif
2909
2910	uint16_t idx = pPool->iModifiedHead;
2911	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2912	while (idx != NIL_PGMPOOL_IDX)
2913	{
2914	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2915	idx = pPage->iModifiedNext;
2916	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2917	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2918	pPage->cModifications = 0;
2919	Assert(++cPages);
2920	}
2921	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2922	pPool->cModifiedPages = 0;
2923	PGM_UNLOCK(pVM);
2924	}
2925
2926
2927	/**
2928	* Handle SyncCR3 pool tasks
2929	*
2930	* @returns VBox status code.
2931	* @retval VINF_SUCCESS if successfully added.
2932	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2933	* @param pVCpu The cross context virtual CPU structure.
2934	* @remark Should only be used when monitoring is available, thus placed in
2935	* the PGMPOOL_WITH_MONITORING \#ifdef.
2936	*/
2937	int pgmPoolSyncCR3(PVMCPUCC pVCpu)
2938	{
2939	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2940	LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
2941
2942	/*
2943	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2944	* Occasionally we will have to clear all the shadow page tables because we wanted
2945	* to monitor a page which was mapped by too many shadowed page tables. This operation
2946	* sometimes referred to as a 'lightweight flush'.
2947	*/
2948	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2949	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2950	pgmR3PoolClearAll(pVM, false /fFlushRemTlb/);
2951	# else /* !IN_RING3 */
2952	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2953	{
2954	Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2955	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2956
2957	/* Make sure all other VCPUs return to ring 3. */
2958	if (pVM->cCpus > 1)
2959	{
2960	VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
2961	PGM_INVL_ALL_VCPU_TLBS(pVM);
2962	}
2963	return VINF_PGM_SYNC_CR3;
2964	}
2965	# endif /* !IN_RING3 */
2966	else
2967	{
2968	pgmPoolMonitorModifiedClearAll(pVM);
2969
2970	/* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
2971	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2972	{
2973	Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
2974	return pgmPoolSyncCR3(pVCpu);
2975	}
2976	}
2977	return VINF_SUCCESS;
2978	}
2979
2980
2981	/**
2982	* Frees up at least one user entry.
2983	*
2984	* @returns VBox status code.
2985	* @retval VINF_SUCCESS if successfully added.
2986	*
2987	* @param pPool The pool.
2988	* @param iUser The user index.
2989	*/
2990	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2991	{
2992	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2993	/*
2994	* Just free cached pages in a braindead fashion.
2995	*/
2996	/** @todo walk the age list backwards and free the first with usage. */
2997	int rc = VINF_SUCCESS;
2998	do
2999	{
3000	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
3001	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
3002	rc = rc2;
3003	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
3004	return rc;
3005	}
3006
3007
3008	/**
3009	* Inserts a page into the cache.
3010	*
3011	* This will create user node for the page, insert it into the GCPhys
3012	* hash, and insert it into the age list.
3013	*
3014	* @returns VBox status code.
3015	* @retval VINF_SUCCESS if successfully added.
3016	*
3017	* @param pPool The pool.
3018	* @param pPage The cached page.
3019	* @param GCPhys The GC physical address of the page we're gonna shadow.
3020	* @param iUser The user index.
3021	* @param iUserTable The user table index.
3022	*/
3023	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
3024	{
3025	int rc = VINF_SUCCESS;
3026	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3027
3028	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
3029
3030	if (iUser != NIL_PGMPOOL_IDX)
3031	{
3032	#ifdef VBOX_STRICT
3033	/*
3034	* Check that the entry doesn't already exists.
3035	*/
3036	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
3037	{
3038	uint16_t i = pPage->iUserHead;
3039	do
3040	{
3041	Assert(i < pPool->cMaxUsers);
3042	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
3043	i = paUsers[i].iNext;
3044	} while (i != NIL_PGMPOOL_USER_INDEX);
3045	}
3046	#endif
3047
3048	/*
3049	* Find free a user node.
3050	*/
3051	uint16_t i = pPool->iUserFreeHead;
3052	if (i == NIL_PGMPOOL_USER_INDEX)
3053	{
3054	rc = pgmPoolTrackFreeOneUser(pPool, iUser);
3055	if (RT_FAILURE(rc))
3056	return rc;
3057	i = pPool->iUserFreeHead;
3058	}
3059
3060	/*
3061	* Unlink the user node from the free list,
3062	* initialize and insert it into the user list.
3063	*/
3064	pPool->iUserFreeHead = paUsers[i].iNext;
3065	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
3066	paUsers[i].iUser = iUser;
3067	paUsers[i].iUserTable = iUserTable;
3068	pPage->iUserHead = i;
3069	}
3070	else
3071	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3072
3073
3074	/*
3075	* Insert into cache and enable monitoring of the guest page if enabled.
3076	*
3077	* Until we implement caching of all levels, including the CR3 one, we'll
3078	* have to make sure we don't try monitor & cache any recursive reuse of
3079	* a monitored CR3 page. Because all windows versions are doing this we'll
3080	* have to be able to do combined access monitoring, CR3 + PT and
3081	* PD + PT (guest PAE).
3082	*
3083	* Update:
3084	* We're now cooperating with the CR3 monitor if an uncachable page is found.
3085	*/
3086	const bool fCanBeMonitored = true;
3087	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
3088	if (fCanBeMonitored)
3089	{
3090	rc = pgmPoolMonitorInsert(pPool, pPage);
3091	AssertRC(rc);
3092	}
3093	return rc;
3094	}
3095
3096
3097	/**
3098	* Adds a user reference to a page.
3099	*
3100	* This will move the page to the head of the
3101	*
3102	* @returns VBox status code.
3103	* @retval VINF_SUCCESS if successfully added.
3104	*
3105	* @param pPool The pool.
3106	* @param pPage The cached page.
3107	* @param iUser The user index.
3108	* @param iUserTable The user table.
3109	*/
3110	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3111	{
3112	Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
3113	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3114	Assert(iUser != NIL_PGMPOOL_IDX);
3115
3116	# ifdef VBOX_STRICT
3117	/*
3118	* Check that the entry doesn't already exists. We only allow multiple
3119	* users of top-level paging structures (SHW_POOL_ROOT_IDX).
3120	*/
3121	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
3122	{
3123	uint16_t i = pPage->iUserHead;
3124	do
3125	{
3126	Assert(i < pPool->cMaxUsers);
3127	/** @todo this assertion looks odd... Shouldn't it be && here? */
3128	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
3129	i = paUsers[i].iNext;
3130	} while (i != NIL_PGMPOOL_USER_INDEX);
3131	}
3132	# endif
3133
3134	/*
3135	* Allocate a user node.
3136	*/
3137	uint16_t i = pPool->iUserFreeHead;
3138	if (i == NIL_PGMPOOL_USER_INDEX)
3139	{
3140	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
3141	if (RT_FAILURE(rc))
3142	return rc;
3143	i = pPool->iUserFreeHead;
3144	}
3145	pPool->iUserFreeHead = paUsers[i].iNext;
3146
3147	/*
3148	* Initialize the user node and insert it.
3149	*/
3150	paUsers[i].iNext = pPage->iUserHead;
3151	paUsers[i].iUser = iUser;
3152	paUsers[i].iUserTable = iUserTable;
3153	pPage->iUserHead = i;
3154
3155	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3156	if (pPage->fDirty)
3157	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
3158	# endif
3159
3160	/*
3161	* Tell the cache to update its replacement stats for this page.
3162	*/
3163	pgmPoolCacheUsed(pPool, pPage);
3164	return VINF_SUCCESS;
3165	}
3166
3167
3168	/**
3169	* Frees a user record associated with a page.
3170	*
3171	* This does not clear the entry in the user table, it simply replaces the
3172	* user record to the chain of free records.
3173	*
3174	* @param pPool The pool.
3175	* @param pPage The shadow page.
3176	* @param iUser The shadow page pool index of the user table.
3177	* @param iUserTable The index into the user table (shadowed).
3178	*
3179	* @remarks Don't call this for root pages.
3180	*/
3181	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3182	{
3183	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
3184	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3185	Assert(iUser != NIL_PGMPOOL_IDX);
3186
3187	/*
3188	* Unlink and free the specified user entry.
3189	*/
3190
3191	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
3192	uint16_t i = pPage->iUserHead;
3193	if ( i != NIL_PGMPOOL_USER_INDEX
3194	&& paUsers[i].iUser == iUser
3195	&& paUsers[i].iUserTable == iUserTable)
3196	{
3197	pPage->iUserHead = paUsers[i].iNext;
3198
3199	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3200	paUsers[i].iNext = pPool->iUserFreeHead;
3201	pPool->iUserFreeHead = i;
3202	return;
3203	}
3204
3205	/* General: Linear search. */
3206	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
3207	while (i != NIL_PGMPOOL_USER_INDEX)
3208	{
3209	if ( paUsers[i].iUser == iUser
3210	&& paUsers[i].iUserTable == iUserTable)
3211	{
3212	if (iPrev != NIL_PGMPOOL_USER_INDEX)
3213	paUsers[iPrev].iNext = paUsers[i].iNext;
3214	else
3215	pPage->iUserHead = paUsers[i].iNext;
3216
3217	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3218	paUsers[i].iNext = pPool->iUserFreeHead;
3219	pPool->iUserFreeHead = i;
3220	return;
3221	}
3222	iPrev = i;
3223	i = paUsers[i].iNext;
3224	}
3225
3226	/* Fatal: didn't find it */
3227	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
3228	iUser, iUserTable, pPage->GCPhys));
3229	}
3230
3231
3232	#if 0 /* unused */
3233	/**
3234	* Gets the entry size of a shadow table.
3235	*
3236	* @param enmKind The kind of page.
3237	*
3238	* @returns The size of the entry in bytes. That is, 4 or 8.
3239	* @returns If the kind is not for a table, an assertion is raised and 0 is
3240	* returned.
3241	*/
3242	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
3243	{
3244	switch (enmKind)
3245	{
3246	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3247	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3248	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3249	case PGMPOOLKIND_32BIT_PD:
3250	case PGMPOOLKIND_32BIT_PD_PHYS:
3251	return 4;
3252
3253	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3254	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3255	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3256	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3257	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3258	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3259	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3260	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3261	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3262	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3263	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3264	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3265	case PGMPOOLKIND_64BIT_PML4:
3266	case PGMPOOLKIND_PAE_PDPT:
3267	case PGMPOOLKIND_ROOT_NESTED:
3268	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3269	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3270	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3271	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3272	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3273	case PGMPOOLKIND_PAE_PD_PHYS:
3274	case PGMPOOLKIND_PAE_PDPT_PHYS:
3275	return 8;
3276
3277	default:
3278	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3279	}
3280	}
3281	#endif /* unused */
3282
3283	#if 0 /* unused */
3284	/**
3285	* Gets the entry size of a guest table.
3286	*
3287	* @param enmKind The kind of page.
3288	*
3289	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
3290	* @returns If the kind is not for a table, an assertion is raised and 0 is
3291	* returned.
3292	*/
3293	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
3294	{
3295	switch (enmKind)
3296	{
3297	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3298	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3299	case PGMPOOLKIND_32BIT_PD:
3300	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3301	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3302	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3303	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3304	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3305	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3306	return 4;
3307
3308	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3309	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3310	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3311	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3312	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3313	case PGMPOOLKIND_64BIT_PML4:
3314	case PGMPOOLKIND_PAE_PDPT:
3315	return 8;
3316
3317	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3318	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3319	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3320	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3321	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3322	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3323	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3324	case PGMPOOLKIND_ROOT_NESTED:
3325	case PGMPOOLKIND_PAE_PD_PHYS:
3326	case PGMPOOLKIND_PAE_PDPT_PHYS:
3327	case PGMPOOLKIND_32BIT_PD_PHYS:
3328	/** @todo can we return 0? (nobody is calling this...) */
3329	AssertFailed();
3330	return 0;
3331
3332	default:
3333	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3334	}
3335	}
3336	#endif /* unused */
3337
3338
3339	/**
3340	* Checks one shadow page table entry for a mapping of a physical page.
3341	*
3342	* @returns true / false indicating removal of all relevant PTEs
3343	*
3344	* @param pVM The cross context VM structure.
3345	* @param pPhysPage The guest page in question.
3346	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3347	* @param iShw The shadow page table.
3348	* @param iPte Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
3349	*/
3350	static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
3351	{
3352	LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
3353	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3354	bool fRet = false;
3355
3356	/*
3357	* Assert sanity.
3358	*/
3359	Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
3360	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
3361	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
3362
3363	/*
3364	* Then, clear the actual mappings to the page in the shadow PT.
3365	*/
3366	switch (pPage->enmKind)
3367	{
3368	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3369	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3370	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3371	{
3372	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3373	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3374	uint32_t u32AndMask = 0;
3375	uint32_t u32OrMask = 0;
3376
3377	if (!fFlushPTEs)
3378	{
3379	/* Note! Disregarding the PGMPHYSHANDLER_F_NOT_IN_HM bit here. Should be harmless. */
3380	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3381	{
3382	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3383	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3384	u32OrMask = X86_PTE_RW;
3385	u32AndMask = UINT32_MAX;
3386	fRet = true;
3387	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3388	break;
3389
3390	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3391	u32OrMask = 0;
3392	u32AndMask = ~X86_PTE_RW;
3393	fRet = true;
3394	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3395	break;
3396	default:
3397	/* We will end up here when called with an "ALL" access handler. */
3398	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3399	break;
3400	}
3401	}
3402	else
3403	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3404
3405	/* Update the counter if we're removing references. */
3406	if (!u32AndMask)
3407	{
3408	Assert(pPage->cPresent);
3409	Assert(pPool->cPresent);
3410	pPage->cPresent--;
3411	pPool->cPresent--;
3412	}
3413
3414	if ((pPT->a[iPte].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3415	{
3416	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
3417	X86PTE Pte;
3418	Pte.u = (pPT->a[iPte].u & u32AndMask) \| u32OrMask;
3419	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3420	Pte.u &= ~(X86PGUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3421
3422	ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
3423	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3424	return fRet;
3425	}
3426	#ifdef LOG_ENABLED
3427	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3428	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3429	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3430	{
3431	Log(("i=%d cFound=%d\n", i, ++cFound));
3432	}
3433	#endif
3434	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
3435	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3436	break;
3437	}
3438
3439	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3440	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3441	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3442	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3443	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3444	case PGMPOOLKIND_EPT_PT_FOR_PHYS: /* physical mask the same as PAE; RW bit as well; be careful! */
3445	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3446	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
3447	# ifdef PGM_WITH_LARGE_PAGES
3448	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
3449	# endif
3450	#endif
3451	{
3452	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3453	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3454	uint64_t u64OrMask = 0;
3455	uint64_t u64AndMask = 0;
3456
3457	if (!fFlushPTEs)
3458	{
3459	/* Note! Disregarding the PGMPHYSHANDLER_F_NOT_IN_HM bit here. Should be harmless. */
3460	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3461	{
3462	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3463	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3464	u64OrMask = X86_PTE_RW;
3465	u64AndMask = UINT64_MAX;
3466	fRet = true;
3467	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3468	break;
3469
3470	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3471	u64OrMask = 0;
3472	u64AndMask = ~(uint64_t)X86_PTE_RW;
3473	fRet = true;
3474	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3475	break;
3476
3477	default:
3478	/* We will end up here when called with an "ALL" access handler. */
3479	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3480	break;
3481	}
3482	}
3483	else
3484	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3485
3486	/* Update the counter if we're removing references. */
3487	if (!u64AndMask)
3488	{
3489	Assert(pPage->cPresent);
3490	Assert(pPool->cPresent);
3491	pPage->cPresent--;
3492	pPool->cPresent--;
3493	}
3494
3495	if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3496	{
3497	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3498	X86PTEPAE Pte;
3499	Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) \| u64OrMask;
3500	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3501	Pte.u &= ~(X86PGPAEUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3502
3503	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
3504	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3505	return fRet;
3506	}
3507	#ifdef LOG_ENABLED
3508	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3509	Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX), u64));
3510	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3511	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3512	Log(("i=%d cFound=%d\n", i, ++cFound));
3513	#endif
3514	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3515	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3516	break;
3517	}
3518
3519	#ifdef PGM_WITH_LARGE_PAGES
3520	/* Large page case only. */
3521	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3522	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
3523	{
3524	Assert(pVM->pgm.s.fNestedPaging);
3525
3526	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3527	PEPTPD pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3528
3529	Assert( pPage->enmKind != PGMPOOLKIND_EPT_PD_FOR_EPT_PD
3530	\|\| (pPD->a[iPte].u & EPT_E_LEAF));
3531
3532	if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3533	{
3534	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3535	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3536	pPD->a[iPte].u = 0;
3537	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3538
3539	/* Update the counter as we're removing references. */
3540	Assert(pPage->cPresent);
3541	Assert(pPool->cPresent);
3542	pPage->cPresent--;
3543	pPool->cPresent--;
3544
3545	return fRet;
3546	}
3547	# ifdef LOG_ENABLED
3548	LogRel(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3549	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3550	if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3551	{
3552	cFound++;
3553	LogRel(("i=%d cFound=%d\n", i, cFound));
3554	}
3555	# endif
3556	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d enmKind=%d\n", pPage->iFirstPresent, pPage->cPresent, pPage->enmKind));
3557	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3558	break;
3559	}
3560
3561	/* AMD-V nested paging / /* @todo merge with EPT as we only check the parts that are identical. */
3562	case PGMPOOLKIND_PAE_PD_PHYS:
3563	{
3564	Assert(pVM->pgm.s.fNestedPaging);
3565
3566	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3567	PX86PDPAE pPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3568
3569	if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3570	{
3571	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3572	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3573	pPD->a[iPte].u = 0;
3574	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3575
3576	/* Update the counter as we're removing references. */
3577	Assert(pPage->cPresent);
3578	Assert(pPool->cPresent);
3579	pPage->cPresent--;
3580	pPool->cPresent--;
3581	return fRet;
3582	}
3583	# ifdef LOG_ENABLED
3584	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3585	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3586	if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3587	Log(("i=%d cFound=%d\n", i, ++cFound));
3588	# endif
3589	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3590	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3591	break;
3592	}
3593	#endif /* PGM_WITH_LARGE_PAGES */
3594
3595	default:
3596	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3597	}
3598
3599	/* not reached. */
3600	#ifndef _MSC_VER
3601	return fRet;
3602	#endif
3603	}
3604
3605
3606	/**
3607	* Scans one shadow page table for mappings of a physical page.
3608	*
3609	* @param pVM The cross context VM structure.
3610	* @param pPhysPage The guest page in question.
3611	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3612	* @param iShw The shadow page table.
3613	*/
3614	static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
3615	{
3616	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3617
3618	/* We should only come here with when there's only one reference to this physical page. */
3619	Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
3620
3621	Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
3622	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3623	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
3624	if (!fKeptPTEs)
3625	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3626	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3627	}
3628
3629
3630	/**
3631	* Flushes a list of shadow page tables mapping the same physical page.
3632	*
3633	* @param pVM The cross context VM structure.
3634	* @param pPhysPage The guest page in question.
3635	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3636	* @param iPhysExt The physical cross reference extent list to flush.
3637	*/
3638	static void pgmPoolTrackFlushGCPhysPTs(PVMCC pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
3639	{
3640	PGM_LOCK_ASSERT_OWNER(pVM);
3641	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3642	bool fKeepList = false;
3643
3644	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3645	Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
3646
3647	const uint16_t iPhysExtStart = iPhysExt;
3648	PPGMPOOLPHYSEXT pPhysExt;
3649	do
3650	{
3651	Assert(iPhysExt < pPool->cMaxPhysExts);
3652	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3653	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3654	{
3655	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3656	{
3657	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
3658	if (!fKeptPTEs)
3659	{
3660	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3661	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3662	}
3663	else
3664	fKeepList = true;
3665	}
3666	}
3667	/* next */
3668	iPhysExt = pPhysExt->iNext;
3669	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3670
3671	if (!fKeepList)
3672	{
3673	/* insert the list into the free list and clear the ram range entry. */
3674	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3675	pPool->iPhysExtFreeHead = iPhysExtStart;
3676	/* Invalidate the tracking data. */
3677	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3678	}
3679
3680	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3681	}
3682
3683
3684	/**
3685	* Flushes all shadow page table mappings of the given guest page.
3686	*
3687	* This is typically called when the host page backing the guest one has been
3688	* replaced or when the page protection was changed due to a guest access
3689	* caught by the monitoring.
3690	*
3691	* @returns VBox status code.
3692	* @retval VINF_SUCCESS if all references has been successfully cleared.
3693	* @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3694	* pool cleaning. FF and sync flags are set.
3695	*
3696	* @param pVM The cross context VM structure.
3697	* @param GCPhysPage GC physical address of the page in question
3698	* @param pPhysPage The guest page in question.
3699	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3700	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3701	* flushed, it is NOT touched if this isn't necessary.
3702	* The caller MUST initialized this to @a false.
3703	*/
3704	int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
3705	{
3706	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3707	PGM_LOCK_VOID(pVM);
3708	int rc = VINF_SUCCESS;
3709
3710	#ifdef PGM_WITH_LARGE_PAGES
3711	/* Is this page part of a large page? */
3712	if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
3713	{
3714	RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
3715	GCPhysPage &= X86_PDE_PAE_PG_MASK;
3716
3717	/* Fetch the large page base. */
3718	PPGMPAGE pLargePage;
3719	if (GCPhysBase != GCPhysPage)
3720	{
3721	pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
3722	AssertFatal(pLargePage);
3723	}
3724	else
3725	pLargePage = pPhysPage;
3726
3727	Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
3728
3729	if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
3730	{
3731	/* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
3732	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3733	pVM->pgm.s.cLargePagesDisabled++;
3734
3735	/* Update the base as that only that one has a reference and there's only one PDE to clear. */
3736	rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
3737
3738	*pfFlushTLBs = true;
3739	PGM_UNLOCK(pVM);
3740	return rc;
3741	}
3742	}
3743	#else
3744	NOREF(GCPhysPage);
3745	#endif /* PGM_WITH_LARGE_PAGES */
3746
3747	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3748	if (u16)
3749	{
3750	/*
3751	* The zero page is currently screwing up the tracking and we'll
3752	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3753	* is defined, zero pages won't normally be mapped. Some kind of solution
3754	* will be needed for this problem of course, but it will have to wait...
3755	*/
3756	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC /* end up guruing after pgmR0PhysAllocateLargePage otherwise. */
3757	if ( PGM_PAGE_IS_ZERO(pPhysPage)
3758	\|\| PGM_PAGE_IS_BALLOONED(pPhysPage))
3759	# else
3760	if (PGM_PAGE_IS_BALLOONED(pPhysPage))
3761	# endif
3762	rc = VINF_PGM_GCPHYS_ALIASED;
3763	else
3764	{
3765	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3766	{
3767	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3768	pgmPoolTrackFlushGCPhysPT(pVM,
3769	pPhysPage,
3770	fFlushPTEs,
3771	PGMPOOL_TD_GET_IDX(u16));
3772	}
3773	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3774	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
3775	else
3776	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3777	*pfFlushTLBs = true;
3778	}
3779	}
3780
3781	if (rc == VINF_PGM_GCPHYS_ALIASED)
3782	{
3783	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3784	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3785	rc = VINF_PGM_SYNC_CR3;
3786	}
3787	PGM_UNLOCK(pVM);
3788	return rc;
3789	}
3790
3791
3792	/**
3793	* Scans all shadow page tables for mappings of a physical page.
3794	*
3795	* This may be slow, but it's most likely more efficient than cleaning
3796	* out the entire page pool / cache.
3797	*
3798	* @returns VBox status code.
3799	* @retval VINF_SUCCESS if all references has been successfully cleared.
3800	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3801	* a page pool cleaning.
3802	*
3803	* @param pVM The cross context VM structure.
3804	* @param pPhysPage The guest page in question.
3805	*/
3806	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage)
3807	{
3808	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3809	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3810	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3811	pPool->cUsedPages, pPool->cPresent, pPhysPage));
3812
3813	/*
3814	* There is a limit to what makes sense.
3815	*/
3816	if ( pPool->cPresent > 1024
3817	&& pVM->cCpus == 1)
3818	{
3819	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3820	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3821	return VINF_PGM_GCPHYS_ALIASED;
3822	}
3823
3824	/*
3825	* Iterate all the pages until we've encountered all that in use.
3826	* This is simple but not quite optimal solution.
3827	*/
3828	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage);
3829	unsigned cLeft = pPool->cUsedPages;
3830	unsigned iPage = pPool->cCurPages;
3831	while (--iPage >= PGMPOOL_IDX_FIRST)
3832	{
3833	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3834	if ( pPage->GCPhys != NIL_RTGCPHYS
3835	&& pPage->cPresent)
3836	{
3837	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* see if it hits */
3838	switch (pPage->enmKind)
3839	{
3840	/*
3841	* We only care about shadow page tables.
3842	*/
3843	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3844	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3845	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3846	{
3847	const uint32_t u32 = (uint32_t)u64;
3848	unsigned cPresent = pPage->cPresent;
3849	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3850	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3851	{
3852	const X86PGUINT uPte = pPT->a[i].u;
3853	if (uPte & X86_PTE_P)
3854	{
3855	if ((uPte & X86_PTE_PG_MASK) == u32)
3856	{
3857	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3858	ASMAtomicWriteU32(&pPT->a[i].u, 0);
3859
3860	/* Update the counter as we're removing references. */
3861	Assert(pPage->cPresent);
3862	Assert(pPool->cPresent);
3863	pPage->cPresent--;
3864	pPool->cPresent--;
3865	}
3866	if (!--cPresent)
3867	break;
3868	}
3869	}
3870	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3871	break;
3872	}
3873
3874	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3875	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3876	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3877	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3878	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3879	{
3880	unsigned cPresent = pPage->cPresent;
3881	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3882	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3883	if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
3884	{
3885	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_PAE_PG_MASK) == u64)
3886	{
3887	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3888	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[i], 0); /// @todo why not atomic?
3889
3890	/* Update the counter as we're removing references. */
3891	Assert(pPage->cPresent);
3892	Assert(pPool->cPresent);
3893	pPage->cPresent--;
3894	pPool->cPresent--;
3895	}
3896	if (!--cPresent)
3897	break;
3898	}
3899	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3900	break;
3901	}
3902
3903	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3904	{
3905	unsigned cPresent = pPage->cPresent;
3906	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3907	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3908	{
3909	X86PGPAEUINT const uPte = pPT->a[i].u;
3910	if (uPte & EPT_E_READ)
3911	{
3912	if ((uPte & EPT_PTE_PG_MASK) == u64)
3913	{
3914	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3915	ASMAtomicWriteU64(&pPT->a[i].u, 0);
3916
3917	/* Update the counter as we're removing references. */
3918	Assert(pPage->cPresent);
3919	Assert(pPool->cPresent);
3920	pPage->cPresent--;
3921	pPool->cPresent--;
3922	}
3923	if (!--cPresent)
3924	break;
3925	}
3926	}
3927	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3928	break;
3929	}
3930	}
3931
3932	if (!--cLeft)
3933	break;
3934	}
3935	}
3936
3937	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3938	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3939
3940	/*
3941	* There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
3942	*/
3943	if (pPool->cPresent > 1024)
3944	{
3945	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3946	return VINF_PGM_GCPHYS_ALIASED;
3947	}
3948
3949	return VINF_SUCCESS;
3950	}
3951
3952
3953	/**
3954	* Clears the user entry in a user table.
3955	*
3956	* This is used to remove all references to a page when flushing it.
3957	*/
3958	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3959	{
3960	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3961	Assert(pUser->iUser < pPool->cCurPages);
3962	uint32_t iUserTable = pUser->iUserTable;
3963
3964	/*
3965	* Map the user page. Ignore references made by fictitious pages.
3966	*/
3967	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3968	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3969	union
3970	{
3971	uint64_t *pau64;
3972	uint32_t *pau32;
3973	} u;
3974	if (pUserPage->idx < PGMPOOL_IDX_FIRST)
3975	{
3976	Assert(!pUserPage->pvPageR3);
3977	return;
3978	}
3979	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3980
3981
3982	/* Safety precaution in case we change the paging for other modes too in the future. */
3983	Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
3984
3985	#ifdef VBOX_STRICT
3986	/*
3987	* Some sanity checks.
3988	*/
3989	switch (pUserPage->enmKind)
3990	{
3991	case PGMPOOLKIND_32BIT_PD:
3992	case PGMPOOLKIND_32BIT_PD_PHYS:
3993	Assert(iUserTable < X86_PG_ENTRIES);
3994	break;
3995	case PGMPOOLKIND_PAE_PDPT:
3996	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3997	case PGMPOOLKIND_PAE_PDPT_PHYS:
3998	Assert(iUserTable < 4);
3999	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
4000	break;
4001	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4002	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4003	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4004	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4005	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4006	case PGMPOOLKIND_PAE_PD_PHYS:
4007	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4008	break;
4009	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4010	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4011	break;
4012	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4013	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4014	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
4015	break;
4016	case PGMPOOLKIND_64BIT_PML4:
4017	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
4018	/* GCPhys >> PAGE_SHIFT is the index here */
4019	break;
4020	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4021	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4022	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4023	break;
4024
4025	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4026	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4027	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4028	break;
4029
4030	case PGMPOOLKIND_ROOT_NESTED:
4031	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4032	break;
4033
4034	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4035	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
4036	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
4037	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
4038	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
4039	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
4040	Assert(iUserTable < EPT_PG_ENTRIES);
4041	break;
4042	# endif
4043
4044	default:
4045	AssertMsgFailed(("enmKind=%d GCPhys=%RGp\n", pUserPage->enmKind, pPage->GCPhys));
4046	break;
4047	}
4048	#endif /* VBOX_STRICT */
4049
4050	/*
4051	* Clear the entry in the user page.
4052	*/
4053	switch (pUserPage->enmKind)
4054	{
4055	/* 32-bit entries */
4056	case PGMPOOLKIND_32BIT_PD:
4057	case PGMPOOLKIND_32BIT_PD_PHYS:
4058	ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
4059	break;
4060
4061	/* 64-bit entries */
4062	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4063	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4064	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4065	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4066	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4067	case PGMPOOLKIND_PAE_PD_PHYS:
4068	case PGMPOOLKIND_PAE_PDPT_PHYS:
4069	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4070	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4071	case PGMPOOLKIND_64BIT_PML4:
4072	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4073	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4074	case PGMPOOLKIND_PAE_PDPT:
4075	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4076	case PGMPOOLKIND_ROOT_NESTED:
4077	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4078	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4079	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4080	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
4081	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
4082	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
4083	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
4084	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
4085	#endif
4086	ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
4087	break;
4088
4089	default:
4090	AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
4091	}
4092	PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
4093	}
4094
4095
4096	/**
4097	* Clears all users of a page.
4098	*/
4099	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4100	{
4101	/*
4102	* Free all the user records.
4103	*/
4104	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
4105
4106	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4107	uint16_t i = pPage->iUserHead;
4108	while (i != NIL_PGMPOOL_USER_INDEX)
4109	{
4110	/* Clear enter in user table. */
4111	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
4112
4113	/* Free it. */
4114	const uint16_t iNext = paUsers[i].iNext;
4115	paUsers[i].iUser = NIL_PGMPOOL_IDX;
4116	paUsers[i].iNext = pPool->iUserFreeHead;
4117	pPool->iUserFreeHead = i;
4118
4119	/* Next. */
4120	i = iNext;
4121	}
4122	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4123	}
4124
4125
4126	/**
4127	* Allocates a new physical cross reference extent.
4128	*
4129	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
4130	* @param pVM The cross context VM structure.
4131	* @param piPhysExt Where to store the phys ext index.
4132	*/
4133	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt)
4134	{
4135	PGM_LOCK_ASSERT_OWNER(pVM);
4136	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4137	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
4138	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4139	{
4140	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
4141	return NULL;
4142	}
4143	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4144	pPool->iPhysExtFreeHead = pPhysExt->iNext;
4145	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4146	*piPhysExt = iPhysExt;
4147	return pPhysExt;
4148	}
4149
4150
4151	/**
4152	* Frees a physical cross reference extent.
4153	*
4154	* @param pVM The cross context VM structure.
4155	* @param iPhysExt The extent to free.
4156	*/
4157	void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt)
4158	{
4159	PGM_LOCK_ASSERT_OWNER(pVM);
4160	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4161	Assert(iPhysExt < pPool->cMaxPhysExts);
4162	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4163	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4164	{
4165	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4166	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4167	}
4168	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4169	pPool->iPhysExtFreeHead = iPhysExt;
4170	}
4171
4172
4173	/**
4174	* Frees a physical cross reference extent.
4175	*
4176	* @param pVM The cross context VM structure.
4177	* @param iPhysExt The extent to free.
4178	*/
4179	void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt)
4180	{
4181	PGM_LOCK_ASSERT_OWNER(pVM);
4182	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4183
4184	const uint16_t iPhysExtStart = iPhysExt;
4185	PPGMPOOLPHYSEXT pPhysExt;
4186	do
4187	{
4188	Assert(iPhysExt < pPool->cMaxPhysExts);
4189	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4190	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4191	{
4192	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4193	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4194	}
4195
4196	/* next */
4197	iPhysExt = pPhysExt->iNext;
4198	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4199
4200	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4201	pPool->iPhysExtFreeHead = iPhysExtStart;
4202	}
4203
4204
4205	/**
4206	* Insert a reference into a list of physical cross reference extents.
4207	*
4208	* @returns The new tracking data for PGMPAGE.
4209	*
4210	* @param pVM The cross context VM structure.
4211	* @param iPhysExt The physical extent index of the list head.
4212	* @param iShwPT The shadow page table index.
4213	* @param iPte Page table entry
4214	*
4215	*/
4216	static uint16_t pgmPoolTrackPhysExtInsert(PVMCC pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
4217	{
4218	PGM_LOCK_ASSERT_OWNER(pVM);
4219	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4220	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4221
4222	/*
4223	* Special common cases.
4224	*/
4225	if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
4226	{
4227	paPhysExts[iPhysExt].aidx[1] = iShwPT;
4228	paPhysExts[iPhysExt].apte[1] = iPte;
4229	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
4230	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
4231	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4232	}
4233	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
4234	{
4235	paPhysExts[iPhysExt].aidx[2] = iShwPT;
4236	paPhysExts[iPhysExt].apte[2] = iPte;
4237	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
4238	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
4239	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4240	}
4241	AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
4242
4243	/*
4244	* General treatment.
4245	*/
4246	const uint16_t iPhysExtStart = iPhysExt;
4247	unsigned cMax = 15;
4248	for (;;)
4249	{
4250	Assert(iPhysExt < pPool->cMaxPhysExts);
4251	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4252	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
4253	{
4254	paPhysExts[iPhysExt].aidx[i] = iShwPT;
4255	paPhysExts[iPhysExt].apte[i] = iPte;
4256	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
4257	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
4258	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
4259	}
4260	if (!--cMax)
4261	{
4262	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackOverflows);
4263	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4264	LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
4265	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4266	}
4267
4268	/* advance */
4269	iPhysExt = paPhysExts[iPhysExt].iNext;
4270	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4271	break;
4272	}
4273
4274	/*
4275	* Add another extent to the list.
4276	*/
4277	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4278	if (!pNew)
4279	{
4280	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackNoExtentsLeft);
4281	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4282	LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
4283	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4284	}
4285	pNew->iNext = iPhysExtStart;
4286	pNew->aidx[0] = iShwPT;
4287	pNew->apte[0] = iPte;
4288	LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
4289	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4290	}
4291
4292
4293	/**
4294	* Add a reference to guest physical page where extents are in use.
4295	*
4296	* @returns The new tracking data for PGMPAGE.
4297	*
4298	* @param pVM The cross context VM structure.
4299	* @param pPhysPage Pointer to the aPages entry in the ram range.
4300	* @param u16 The ram range flags (top 16-bits).
4301	* @param iShwPT The shadow page table index.
4302	* @param iPte Page table entry
4303	*/
4304	uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
4305	{
4306	PGM_LOCK_VOID(pVM);
4307	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
4308	{
4309	/*
4310	* Convert to extent list.
4311	*/
4312	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
4313	uint16_t iPhysExt;
4314	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4315	if (pPhysExt)
4316	{
4317	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
4318	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliased);
4319	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
4320	pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
4321	pPhysExt->aidx[1] = iShwPT;
4322	pPhysExt->apte[1] = iPte;
4323	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4324	}
4325	else
4326	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4327	}
4328	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
4329	{
4330	/*
4331	* Insert into the extent list.
4332	*/
4333	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
4334	}
4335	else
4336	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedLots);
4337	PGM_UNLOCK(pVM);
4338	return u16;
4339	}
4340
4341
4342	/**
4343	* Clear references to guest physical memory.
4344	*
4345	* @param pPool The pool.
4346	* @param pPage The page.
4347	* @param pPhysPage Pointer to the aPages entry in the ram range.
4348	* @param iPte Shadow PTE index
4349	*/
4350	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
4351	{
4352	PVMCC pVM = pPool->CTX_SUFF(pVM);
4353	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4354	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4355
4356	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
4357	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
4358	{
4359	PGM_LOCK_VOID(pVM);
4360
4361	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
4362	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4363	do
4364	{
4365	Assert(iPhysExt < pPool->cMaxPhysExts);
4366
4367	/*
4368	* Look for the shadow page and check if it's all freed.
4369	*/
4370	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4371	{
4372	if ( paPhysExts[iPhysExt].aidx[i] == pPage->idx
4373	&& paPhysExts[iPhysExt].apte[i] == iPte)
4374	{
4375	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
4376	paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4377
4378	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4379	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
4380	{
4381	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
4382	PGM_UNLOCK(pVM);
4383	return;
4384	}
4385
4386	/* we can free the node. */
4387	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
4388	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
4389	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
4390	{
4391	/* lonely node */
4392	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4393	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
4394	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
4395	}
4396	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
4397	{
4398	/* head */
4399	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
4400	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
4401	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4402	}
4403	else
4404	{
4405	/* in list */
4406	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
4407	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
4408	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4409	}
4410	iPhysExt = iPhysExtNext;
4411	PGM_UNLOCK(pVM);
4412	return;
4413	}
4414	}
4415
4416	/* next */
4417	iPhysExtPrev = iPhysExt;
4418	iPhysExt = paPhysExts[iPhysExt].iNext;
4419	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4420
4421	PGM_UNLOCK(pVM);
4422	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4423	}
4424	else /* nothing to do */
4425	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
4426	}
4427
4428	/**
4429	* Clear references to guest physical memory.
4430	*
4431	* This is the same as pgmPoolTracDerefGCPhysHint except that the guest
4432	* physical address is assumed to be correct, so the linear search can be
4433	* skipped and we can assert at an earlier point.
4434	*
4435	* @param pPool The pool.
4436	* @param pPage The page.
4437	* @param HCPhys The host physical address corresponding to the guest page.
4438	* @param GCPhys The guest physical address corresponding to HCPhys.
4439	* @param iPte Shadow PTE index
4440	*/
4441	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
4442	{
4443	/*
4444	* Lookup the page and check if it checks out before derefing it.
4445	*/
4446	PVMCC pVM = pPool->CTX_SUFF(pVM);
4447	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
4448	if (pPhysPage)
4449	{
4450	Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
4451	#ifdef LOG_ENABLED
4452	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
4453	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
4454	#endif
4455	if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
4456	{
4457	Assert(pPage->cPresent);
4458	Assert(pPool->cPresent);
4459	pPage->cPresent--;
4460	pPool->cPresent--;
4461	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4462	return;
4463	}
4464
4465	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp iPte=%u fIsNested=%RTbool\n",
4466	HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage), iPte, PGMPOOL_PAGE_IS_NESTED(pPage)));
4467	}
4468	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
4469	}
4470
4471
4472	/**
4473	* Clear references to guest physical memory.
4474	*
4475	* @param pPool The pool.
4476	* @param pPage The page.
4477	* @param HCPhys The host physical address corresponding to the guest page.
4478	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
4479	* @param iPte Shadow pte index
4480	*/
4481	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
4482	{
4483	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
4484
4485	/*
4486	* Try the hint first.
4487	*/
4488	RTHCPHYS HCPhysHinted;
4489	PVMCC pVM = pPool->CTX_SUFF(pVM);
4490	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
4491	if (pPhysPage)
4492	{
4493	HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
4494	Assert(HCPhysHinted);
4495	if (HCPhysHinted == HCPhys)
4496	{
4497	Assert(pPage->cPresent);
4498	Assert(pPool->cPresent);
4499	pPage->cPresent--;
4500	pPool->cPresent--;
4501	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4502	return;
4503	}
4504	}
4505	else
4506	HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
4507
4508	/*
4509	* Damn, the hint didn't work. We'll have to do an expensive linear search.
4510	*/
4511	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
4512	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
4513	while (pRam)
4514	{
4515	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4516	while (iPage-- > 0)
4517	{
4518	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
4519	{
4520	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
4521	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
4522	Assert(pPage->cPresent);
4523	Assert(pPool->cPresent);
4524	pPage->cPresent--;
4525	pPool->cPresent--;
4526	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
4527	return;
4528	}
4529	}
4530	pRam = pRam->CTX_SUFF(pNext);
4531	}
4532
4533	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
4534	}
4535
4536
4537	/**
4538	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
4539	*
4540	* @param pPool The pool.
4541	* @param pPage The page.
4542	* @param pShwPT The shadow page table (mapping of the page).
4543	* @param pGstPT The guest page table.
4544	*/
4545	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
4546	{
4547	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4548	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4549	{
4550	const X86PGUINT uPte = pShwPT->a[i].u;
4551	Assert(!(uPte & RT_BIT_32(10)));
4552	if (uPte & X86_PTE_P)
4553	{
4554	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
4555	i, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
4556	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
4557	if (!pPage->cPresent)
4558	break;
4559	}
4560	}
4561	}
4562
4563
4564	/**
4565	* Clear references to guest physical memory in a PAE / 32-bit page table.
4566	*
4567	* @param pPool The pool.
4568	* @param pPage The page.
4569	* @param pShwPT The shadow page table (mapping of the page).
4570	* @param pGstPT The guest page table (just a half one).
4571	*/
4572	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
4573	{
4574	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4575	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4576	{
4577	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4578	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4579	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4580	{
4581	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
4582	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
4583	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4584	if (!pPage->cPresent)
4585	break;
4586	}
4587	}
4588	}
4589
4590
4591	/**
4592	* Clear references to guest physical memory in a PAE / PAE page table.
4593	*
4594	* @param pPool The pool.
4595	* @param pPage The page.
4596	* @param pShwPT The shadow page table (mapping of the page).
4597	* @param pGstPT The guest page table.
4598	*/
4599	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
4600	{
4601	RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
4602	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4603	{
4604	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4605	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4606	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4607	{
4608	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
4609	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
4610	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4611	if (!pPage->cPresent)
4612	break;
4613	}
4614	}
4615	}
4616
4617
4618	/**
4619	* Clear references to guest physical memory in a 32-bit / 4MB page table.
4620	*
4621	* @param pPool The pool.
4622	* @param pPage The page.
4623	* @param pShwPT The shadow page table (mapping of the page).
4624	*/
4625	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
4626	{
4627	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4628	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4629	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4630	{
4631	const X86PGUINT uPte = pShwPT->a[i].u;
4632	Assert(!(uPte & RT_BIT_32(10)));
4633	if (uPte & X86_PTE_P)
4634	{
4635	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
4636	i, uPte & X86_PTE_PG_MASK, GCPhys));
4637	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4638	if (!pPage->cPresent)
4639	break;
4640	}
4641	}
4642	}
4643
4644
4645	/**
4646	* Clear references to guest physical memory in a PAE / 2/4MB page table.
4647	*
4648	* @param pPool The pool.
4649	* @param pPage The page.
4650	* @param pShwPT The shadow page table (mapping of the page).
4651	*/
4652	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
4653	{
4654	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4655	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4656	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4657	{
4658	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4659	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4660	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4661	{
4662	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
4663	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
4664	pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
4665	if (!pPage->cPresent)
4666	break;
4667	}
4668	}
4669	}
4670
4671
4672	/**
4673	* Clear references to shadowed pages in an EPT page table.
4674	*
4675	* @param pPool The pool.
4676	* @param pPage The page.
4677	* @param pShwPT The shadow page directory pointer table (mapping of the
4678	* page).
4679	*/
4680	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4681	{
4682	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4683	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4684	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4685	{
4686	X86PGPAEUINT const uPte = pShwPT->a[i].u;
4687	Assert((uPte & UINT64_C(0xfff0000000000f80)) == 0);
4688	if (uPte & EPT_E_READ)
4689	{
4690	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4691	i, uPte & EPT_PTE_PG_MASK, pPage->GCPhys));
4692	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4693	if (!pPage->cPresent)
4694	break;
4695	}
4696	}
4697	}
4698
4699	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4700
4701	/**
4702	* Clears references to shadowed pages in a SLAT EPT page table.
4703	*
4704	* @param pPool The pool.
4705	* @param pPage The page.
4706	* @param pShwPT The shadow page table (mapping of the page).
4707	* @param pGstPT The guest page table.
4708	*/
4709	DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT, PCEPTPT pGstPT)
4710	{
4711	Assert(PGMPOOL_PAGE_IS_NESTED(pPage));
4712	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4713	{
4714	X86PGPAEUINT const uShwPte = pShwPT->a[i].u;
4715	Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */
4716	if (uShwPte & EPT_PRESENT_MASK)
4717	{
4718	Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, pGstPT->a[i].u));
4719	pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, pGstPT->a[i].u & EPT_PTE_PG_MASK, i);
4720	if (!pPage->cPresent)
4721	break;
4722	}
4723	}
4724	}
4725
4726
4727	/**
4728	* Clear references to guest physical memory in a SLAT 2MB EPT page table.
4729	*
4730	* @param pPool The pool.
4731	* @param pPage The page.
4732	* @param pShwPT The shadow page table (mapping of the page).
4733	*/
4734	DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT2MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4735	{
4736	Assert(pPage->fA20Enabled);
4737	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4738	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4739	{
4740	X86PGPAEUINT const uShwPte = pShwPT->a[i].u;
4741	Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */
4742	if (uShwPte & EPT_PRESENT_MASK)
4743	{
4744	Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, GCPhys));
4745	pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, GCPhys, i);
4746	if (!pPage->cPresent)
4747	break;
4748	}
4749	}
4750	}
4751
4752
4753	/**
4754	* Clear references to shadowed pages in a SLAT EPT page directory.
4755	*
4756	* @param pPool The pool.
4757	* @param pPage The page.
4758	* @param pShwPD The shadow page directory (mapping of the page).
4759	* @param pGstPD The guest page directory.
4760	*/
4761	DECLINLINE(void) pgmPoolTrackDerefNestedPDEpt(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD, PCEPTPD pGstPD)
4762	{
4763	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4764	{
4765	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4766	#ifdef PGM_WITH_LARGE_PAGES
4767	AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde));
4768	#else
4769	AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde));
4770	#endif
4771	if (uPde & EPT_PRESENT_MASK)
4772	{
4773	#ifdef PGM_WITH_LARGE_PAGES
4774	if (uPde & EPT_E_LEAF)
4775	{
4776	Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n", i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
4777	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK, pGstPD->a[i].u & EPT_PDE2M_PG_MASK, i);
4778	}
4779	else
4780	#endif
4781	{
4782	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK);
4783	if (pSubPage)
4784	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4785	else
4786	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4787	}
4788	}
4789	}
4790	}
4791
4792	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
4793
4794
4795	/**
4796	* Clear references to shadowed pages in a 32 bits page directory.
4797	*
4798	* @param pPool The pool.
4799	* @param pPage The page.
4800	* @param pShwPD The shadow page directory (mapping of the page).
4801	*/
4802	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
4803	{
4804	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4805	{
4806	X86PGUINT const uPde = pShwPD->a[i].u;
4807	if (uPde & X86_PDE_P)
4808	{
4809	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
4810	if (pSubPage)
4811	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4812	else
4813	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
4814	}
4815	}
4816	}
4817
4818
4819	/**
4820	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
4821	*
4822	* @param pPool The pool.
4823	* @param pPage The page.
4824	* @param pShwPD The shadow page directory (mapping of the page).
4825	*/
4826	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
4827	{
4828	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4829	{
4830	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4831	if (uPde & X86_PDE_P)
4832	{
4833	#ifdef PGM_WITH_LARGE_PAGES
4834	if (uPde & X86_PDE_PS)
4835	{
4836	Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
4837	i, uPde & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4838	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & X86_PDE2M_PAE_PG_MASK,
4839	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4840	i);
4841	}
4842	else
4843	#endif
4844	{
4845	Assert((uPde & (X86_PDE_PAE_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000000))) == 0);
4846	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & X86_PDE_PAE_PG_MASK);
4847	if (pSubPage)
4848	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4849	else
4850	AssertFatalMsgFailed(("%RX64\n", uPde & X86_PDE_PAE_PG_MASK));
4851	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4852	}
4853	}
4854	}
4855	}
4856
4857
4858	/**
4859	* Clear references to shadowed pages in a PAE page directory pointer table.
4860	*
4861	* @param pPool The pool.
4862	* @param pPage The page.
4863	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4864	*/
4865	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4866	{
4867	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4868	{
4869	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4870	Assert((uPdpe & (X86_PDPE_PAE_MBZ_MASK \| UINT64_C(0x7ff0000000000200))) == 0);
4871	if (uPdpe & X86_PDPE_P)
4872	{
4873	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4874	if (pSubPage)
4875	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4876	else
4877	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4878	}
4879	}
4880	}
4881
4882
4883	/**
4884	* Clear references to shadowed pages in a 64-bit page directory pointer table.
4885	*
4886	* @param pPool The pool.
4887	* @param pPage The page.
4888	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4889	*/
4890	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4891	{
4892	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4893	{
4894	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4895	Assert((uPdpe & (X86_PDPE_LM_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4896	if (uPdpe & X86_PDPE_P)
4897	{
4898	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4899	if (pSubPage)
4900	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4901	else
4902	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4903	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4904	}
4905	}
4906	}
4907
4908
4909	/**
4910	* Clear references to shadowed pages in a 64-bit level 4 page table.
4911	*
4912	* @param pPool The pool.
4913	* @param pPage The page.
4914	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4915	*/
4916	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4917	{
4918	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4919	{
4920	X86PGPAEUINT const uPml4e = pShwPML4->a[i].u;
4921	Assert((uPml4e & (X86_PML4E_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4922	if (uPml4e & X86_PML4E_P)
4923	{
4924	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & X86_PDPE_PG_MASK);
4925	if (pSubPage)
4926	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4927	else
4928	AssertFatalMsgFailed(("%RX64\n", uPml4e & X86_PML4E_PG_MASK));
4929	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4930	}
4931	}
4932	}
4933
4934
4935	/**
4936	* Clear references to shadowed pages in an EPT page directory.
4937	*
4938	* @param pPool The pool.
4939	* @param pPage The page.
4940	* @param pShwPD The shadow page directory (mapping of the page).
4941	*/
4942	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4943	{
4944	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4945	{
4946	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4947	#ifdef PGM_WITH_LARGE_PAGES
4948	AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde));
4949	#else
4950	AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde));
4951	#endif
4952	if (uPde & EPT_E_READ)
4953	{
4954	#ifdef PGM_WITH_LARGE_PAGES
4955	if (uPde & EPT_E_LEAF)
4956	{
4957	Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
4958	i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
4959	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK,
4960	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4961	i);
4962	}
4963	else
4964	#endif
4965	{
4966	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK);
4967	if (pSubPage)
4968	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4969	else
4970	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4971	}
4972	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4973	}
4974	}
4975	}
4976
4977
4978	/**
4979	* Clear references to shadowed pages in an EPT page directory pointer table.
4980	*
4981	* @param pPool The pool.
4982	* @param pPage The page.
4983	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4984	*/
4985	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4986	{
4987	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4988	{
4989	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4990	Assert((uPdpe & UINT64_C(0xfff0000000000f80)) == 0);
4991	if (uPdpe & EPT_E_READ)
4992	{
4993	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & EPT_PDPTE_PG_MASK);
4994	if (pSubPage)
4995	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4996	else
4997	AssertFatalMsgFailed(("%RX64\n", uPdpe & EPT_PDPTE_PG_MASK));
4998	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4999	}
5000	}
5001	}
5002
5003
5004	/**
5005	* Clears all references made by this page.
5006	*
5007	* This includes other shadow pages and GC physical addresses.
5008	*
5009	* @param pPool The pool.
5010	* @param pPage The page.
5011	*/
5012	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
5013	{
5014	/*
5015	* Map the shadow page and take action according to the page kind.
5016	*/
5017	PVMCC pVM = pPool->CTX_SUFF(pVM);
5018	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
5019	switch (pPage->enmKind)
5020	{
5021	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5022	{
5023	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
5024	void *pvGst;
5025	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
5026	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
5027	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
5028	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
5029	break;
5030	}
5031
5032	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5033	{
5034	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
5035	void *pvGst;
5036	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
5037	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
5038	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
5039	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
5040	break;
5041	}
5042
5043	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5044	{
5045	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
5046	void *pvGst;
5047	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
5048	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
5049	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
5050	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
5051	break;
5052	}
5053
5054	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
5055	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5056	{
5057	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
5058	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
5059	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
5060	break;
5061	}
5062
5063	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
5064	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5065	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5066	{
5067	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
5068	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
5069	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
5070	break;
5071	}
5072
5073	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5074	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5075	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5076	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5077	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5078	case PGMPOOLKIND_PAE_PD_PHYS:
5079	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5080	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5081	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
5082	break;
5083
5084	case PGMPOOLKIND_32BIT_PD_PHYS:
5085	case PGMPOOLKIND_32BIT_PD:
5086	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
5087	break;
5088
5089	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5090	case PGMPOOLKIND_PAE_PDPT:
5091	case PGMPOOLKIND_PAE_PDPT_PHYS:
5092	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
5093	break;
5094
5095	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5096	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5097	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
5098	break;
5099
5100	case PGMPOOLKIND_64BIT_PML4:
5101	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
5102	break;
5103
5104	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5105	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
5106	break;
5107
5108	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5109	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
5110	break;
5111
5112	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5113	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
5114	break;
5115
5116	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5117	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
5118	{
5119	void *pvGst;
5120	int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
5121	pgmPoolTrackDerefNestedPTEPT(pPool, pPage, (PEPTPT)pvShw, (PCEPTPT)pvGst);
5122	break;
5123	}
5124
5125	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
5126	pgmPoolTrackDerefNestedPTEPT2MB(pPool, pPage, (PEPTPT)pvShw);
5127	break;
5128
5129	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
5130	{
5131	void *pvGst;
5132	int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
5133	pgmPoolTrackDerefNestedPDEpt(pPool, pPage, (PEPTPD)pvShw, (PCEPTPD)pvGst);
5134	break;
5135	}
5136
5137	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
5138	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
5139	break;
5140	#endif
5141
5142	default:
5143	AssertFatalMsgFailed(("enmKind=%d GCPhys=%RGp\n", pPage->enmKind, pPage->GCPhys));
5144	}
5145
5146	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
5147	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5148	RT_BZERO(pvShw, PAGE_SIZE);
5149	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5150	pPage->fZeroed = true;
5151	Assert(!pPage->cPresent);
5152	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
5153	}
5154
5155
5156	/**
5157	* Flushes a pool page.
5158	*
5159	* This moves the page to the free list after removing all user references to it.
5160	*
5161	* @returns VBox status code.
5162	* @retval VINF_SUCCESS on success.
5163	* @param pPool The pool.
5164	* @param pPage The shadow page.
5165	* @param fFlush Flush the TLBS when required (should only be false in very specific use cases!!)
5166	*/
5167	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
5168	{
5169	PVMCC pVM = pPool->CTX_SUFF(pVM);
5170	bool fFlushRequired = false;
5171
5172	int rc = VINF_SUCCESS;
5173	STAM_PROFILE_START(&pPool->StatFlushPage, f);
5174	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
5175	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
5176
5177	if (PGMPOOL_PAGE_IS_NESTED(pPage))
5178	Log7Func(("pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
5179	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
5180
5181	/*
5182	* Reject any attempts at flushing any of the special root pages (shall
5183	* not happen).
5184	*/
5185	AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
5186	("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
5187	pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
5188	VINF_SUCCESS);
5189
5190	PGM_LOCK_VOID(pVM);
5191
5192	/*
5193	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
5194	*/
5195	if (pgmPoolIsPageLocked(pPage))
5196	{
5197	#if !defined(VBOX_VMM_TARGET_ARMV8)
5198	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
5199	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
5200	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
5201	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
5202	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
5203	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
5204	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
5205	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
5206	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
5207	\|\| pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
5208	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
5209	#endif
5210	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
5211	PGM_UNLOCK(pVM);
5212	return VINF_SUCCESS;
5213	}
5214
5215	/*
5216	* Mark the page as being in need of an ASMMemZeroPage().
5217	*/
5218	pPage->fZeroed = false;
5219
5220	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5221	if (pPage->fDirty)
5222	pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
5223	#endif
5224
5225	/* If there are any users of this table, then we must issue a tlb flush on all VCPUs. */
5226	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
5227	fFlushRequired = true;
5228
5229	/*
5230	* Clear the page.
5231	*/
5232	pgmPoolTrackClearPageUsers(pPool, pPage);
5233	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
5234	pgmPoolTrackDeref(pPool, pPage);
5235	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
5236
5237	/*
5238	* Flush it from the cache.
5239	*/
5240	pgmPoolCacheFlushPage(pPool, pPage);
5241
5242	/*
5243	* Deregistering the monitoring.
5244	*/
5245	if (pPage->fMonitored)
5246	rc = pgmPoolMonitorFlush(pPool, pPage);
5247
5248	/*
5249	* Free the page.
5250	*/
5251	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
5252	pPage->iNext = pPool->iFreeHead;
5253	pPool->iFreeHead = pPage->idx;
5254	pPage->enmKind = PGMPOOLKIND_FREE;
5255	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5256	pPage->GCPhys = NIL_RTGCPHYS;
5257	pPage->fReusedFlushPending = false;
5258
5259	pPool->cUsedPages--;
5260
5261	/* Flush the TLBs of all VCPUs if required. */
5262	if ( fFlushRequired
5263	&& fFlush)
5264	{
5265	PGM_INVL_ALL_VCPU_TLBS(pVM);
5266	}
5267
5268	PGM_UNLOCK(pVM);
5269	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
5270	return rc;
5271	}
5272
5273
5274	/**
5275	* Frees a usage of a pool page.
5276	*
5277	* The caller is responsible to updating the user table so that it no longer
5278	* references the shadow page.
5279	*
5280	* @param pPool The pool.
5281	* @param pPage The shadow page.
5282	* @param iUser The shadow page pool index of the user table.
5283	* NIL_PGMPOOL_IDX for root pages.
5284	* @param iUserTable The index into the user table (shadowed). Ignored if
5285	* root page.
5286	*/
5287	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
5288	{
5289	PVMCC pVM = pPool->CTX_SUFF(pVM);
5290
5291	STAM_PROFILE_START(&pPool->StatFree, a);
5292	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
5293	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
5294	AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
5295
5296	PGM_LOCK_VOID(pVM);
5297	if (iUser != NIL_PGMPOOL_IDX)
5298	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
5299	if (!pPage->fCached)
5300	pgmPoolFlushPage(pPool, pPage);
5301	PGM_UNLOCK(pVM);
5302	STAM_PROFILE_STOP(&pPool->StatFree, a);
5303	}
5304
5305
5306	/**
5307	* Makes one or more free page free.
5308	*
5309	* @returns VBox status code.
5310	* @retval VINF_SUCCESS on success.
5311	*
5312	* @param pPool The pool.
5313	* @param enmKind Page table kind
5314	* @param iUser The user of the page.
5315	*/
5316	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
5317	{
5318	PVMCC pVM = pPool->CTX_SUFF(pVM);
5319	LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
5320	NOREF(enmKind);
5321
5322	/*
5323	* If the pool isn't full grown yet, expand it.
5324	*/
5325	if (pPool->cCurPages < pPool->cMaxPages)
5326	{
5327	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
5328	#ifdef IN_RING3
5329	int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM));
5330	#else
5331	int rc = PGMR0PoolGrow(pVM, VMMGetCpuId(pVM));
5332	#endif
5333	if (RT_FAILURE(rc))
5334	return rc;
5335	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
5336	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
5337	return VINF_SUCCESS;
5338	}
5339
5340	/*
5341	* Free one cached page.
5342	*/
5343	return pgmPoolCacheFreeOne(pPool, iUser);
5344	}
5345
5346
5347	/**
5348	* Allocates a page from the pool.
5349	*
5350	* This page may actually be a cached page and not in need of any processing
5351	* on the callers part.
5352	*
5353	* @returns VBox status code.
5354	* @retval VINF_SUCCESS if a NEW page was allocated.
5355	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
5356	*
5357	* @param pVM The cross context VM structure.
5358	* @param GCPhys The GC physical address of the page we're gonna shadow.
5359	* For 4MB and 2MB PD entries, it's the first address the
5360	* shadow PT is covering.
5361	* @param enmKind The kind of mapping.
5362	* @param enmAccess Access type for the mapping (only relevant for big pages)
5363	* @param fA20Enabled Whether the A20 gate is enabled or not.
5364	* @param iUser The shadow page pool index of the user table. Root
5365	* pages should pass NIL_PGMPOOL_IDX.
5366	* @param iUserTable The index into the user table (shadowed). Ignored for
5367	* root pages (iUser == NIL_PGMPOOL_IDX).
5368	* @param fLockPage Lock the page
5369	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
5370	*/
5371	int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
5372	uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
5373	{
5374	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5375	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
5376	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
5377	*ppPage = NULL;
5378	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
5379	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
5380	* Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
5381
5382	#if defined(VBOX_STRICT) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
5383	PVMCPUCC pVCpu = VMMGetCpu(pVM);
5384	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT \|\| PGMPOOL_PAGE_IS_KIND_NESTED(enmKind));
5385	#endif
5386
5387	PGM_LOCK_VOID(pVM);
5388
5389	if (pPool->fCacheEnabled)
5390	{
5391	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
5392	if (RT_SUCCESS(rc2))
5393	{
5394	if (fLockPage)
5395	pgmPoolLockPage(pPool, *ppPage);
5396	PGM_UNLOCK(pVM);
5397	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5398	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
5399	return rc2;
5400	}
5401	}
5402
5403	/*
5404	* Allocate a new one.
5405	*/
5406	int rc = VINF_SUCCESS;
5407	uint16_t iNew = pPool->iFreeHead;
5408	if (iNew == NIL_PGMPOOL_IDX)
5409	{
5410	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
5411	if (RT_FAILURE(rc))
5412	{
5413	PGM_UNLOCK(pVM);
5414	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
5415	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5416	return rc;
5417	}
5418	iNew = pPool->iFreeHead;
5419	AssertReleaseMsgReturn(iNew != NIL_PGMPOOL_IDX, ("iNew=%#x\n", iNew), VERR_PGM_POOL_IPE);
5420	}
5421
5422	/* unlink the free head */
5423	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
5424	pPool->iFreeHead = pPage->iNext;
5425	pPage->iNext = NIL_PGMPOOL_IDX;
5426
5427	/*
5428	* Initialize it.
5429	*/
5430	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
5431	pPage->enmKind = enmKind;
5432	pPage->enmAccess = enmAccess;
5433	pPage->GCPhys = GCPhys;
5434	pPage->fA20Enabled = fA20Enabled;
5435	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
5436	pPage->fMonitored = false;
5437	pPage->fCached = false;
5438	pPage->fDirty = false;
5439	pPage->fReusedFlushPending = false;
5440	pPage->cModifications = 0;
5441	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5442	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5443	pPage->cPresent = 0;
5444	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5445	pPage->idxDirtyEntry = 0;
5446	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5447	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5448	pPage->cLastAccessHandler = 0;
5449	pPage->cLocked = 0;
5450	# ifdef VBOX_STRICT
5451	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5452	# endif
5453
5454	/*
5455	* Insert into the tracking and cache. If this fails, free the page.
5456	*/
5457	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
5458	if (RT_FAILURE(rc3))
5459	{
5460	pPool->cUsedPages--;
5461	pPage->enmKind = PGMPOOLKIND_FREE;
5462	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5463	pPage->GCPhys = NIL_RTGCPHYS;
5464	pPage->iNext = pPool->iFreeHead;
5465	pPool->iFreeHead = pPage->idx;
5466	PGM_UNLOCK(pVM);
5467	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5468	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
5469	return rc3;
5470	}
5471
5472	/*
5473	* Commit the allocation, clear the page and return.
5474	*/
5475	#ifdef VBOX_WITH_STATISTICS
5476	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
5477	pPool->cUsedPagesHigh = pPool->cUsedPages;
5478	#endif
5479
5480	if (!pPage->fZeroed)
5481	{
5482	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5483	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
5484	RT_BZERO(pv, PAGE_SIZE);
5485	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5486	}
5487
5488	*ppPage = pPage;
5489	if (fLockPage)
5490	pgmPoolLockPage(pPool, pPage);
5491	PGM_UNLOCK(pVM);
5492	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
5493	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
5494	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5495	return rc;
5496	}
5497
5498
5499	/**
5500	* Frees a usage of a pool page.
5501	*
5502	* @param pVM The cross context VM structure.
5503	* @param HCPhys The HC physical address of the shadow page.
5504	* @param iUser The shadow page pool index of the user table.
5505	* NIL_PGMPOOL_IDX if root page.
5506	* @param iUserTable The index into the user table (shadowed). Ignored if
5507	* root page.
5508	*/
5509	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
5510	{
5511	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
5512	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5513	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
5514	}
5515
5516
5517	/**
5518	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
5519	*
5520	* @returns Pointer to the shadow page structure.
5521	* @param pPool The pool.
5522	* @param HCPhys The HC physical address of the shadow page.
5523	*/
5524	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
5525	{
5526	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5527
5528	/*
5529	* Look up the page.
5530	*/
5531	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5532
5533	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
5534	return pPage;
5535	}
5536
5537
5538	/**
5539	* Internal worker for finding a page for debugging purposes, no assertions.
5540	*
5541	* @returns Pointer to the shadow page structure. NULL on if not found.
5542	* @param pPool The pool.
5543	* @param HCPhys The HC physical address of the shadow page.
5544	*/
5545	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
5546	{
5547	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5548	return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5549	}
5550
5551
5552	/**
5553	* Internal worker for PGM_HCPHYS_2_PTR.
5554	*
5555	* @returns VBox status code.
5556	* @param pVM The cross context VM structure.
5557	* @param HCPhys The HC physical address of the shadow page.
5558	* @param ppv Where to return the address.
5559	*/
5560	int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv)
5561	{
5562	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5563	AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE,
5564	("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0),
5565	VERR_PGM_POOL_GET_PAGE_FAILED);
5566	ppv = (uint8_t )pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK);
5567	return VINF_SUCCESS;
5568	}
5569
5570	#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
5571
5572	/**
5573	* Flush the specified page if present
5574	*
5575	* @param pVM The cross context VM structure.
5576	* @param GCPhys Guest physical address of the page to flush
5577	*/
5578	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
5579	{
5580	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5581
5582	VM_ASSERT_EMT(pVM);
5583
5584	/*
5585	* Look up the GCPhys in the hash.
5586	*/
5587	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
5588	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
5589	if (i == NIL_PGMPOOL_IDX)
5590	return;
5591
5592	do
5593	{
5594	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5595	if (pPage->GCPhys - GCPhys < PAGE_SIZE)
5596	{
5597	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* Temporary to see if it hits. Remove later. */
5598	switch (pPage->enmKind)
5599	{
5600	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5601	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5602	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5603	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5604	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5605	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5606	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5607	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5608	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5609	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5610	case PGMPOOLKIND_64BIT_PML4:
5611	case PGMPOOLKIND_32BIT_PD:
5612	case PGMPOOLKIND_PAE_PDPT:
5613	{
5614	Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
5615	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5616	if (pPage->fDirty)
5617	STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
5618	else
5619	# endif
5620	STAM_COUNTER_INC(&pPool->StatForceFlushPage);
5621	Assert(!pgmPoolIsPageLocked(pPage));
5622	pgmPoolMonitorChainFlush(pPool, pPage);
5623	return;
5624	}
5625
5626	/* ignore, no monitoring. */
5627	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5628	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5629	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5630	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5631	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5632	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5633	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5634	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5635	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5636	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5637	case PGMPOOLKIND_ROOT_NESTED:
5638	case PGMPOOLKIND_PAE_PD_PHYS:
5639	case PGMPOOLKIND_PAE_PDPT_PHYS:
5640	case PGMPOOLKIND_32BIT_PD_PHYS:
5641	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5642	break;
5643
5644	default:
5645	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
5646	}
5647	}
5648
5649	/* next */
5650	i = pPage->iNext;
5651	} while (i != NIL_PGMPOOL_IDX);
5652	return;
5653	}
5654
5655
5656	/**
5657	* Reset CPU on hot plugging.
5658	*
5659	* @param pVM The cross context VM structure.
5660	* @param pVCpu The cross context virtual CPU structure.
5661	*/
5662	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
5663	{
5664	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5665
5666	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5667	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5668	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5669	}
5670
5671
5672	/**
5673	* Flushes the entire cache.
5674	*
5675	* It will assert a global CR3 flush (FF) and assumes the caller is aware of
5676	* this and execute this CR3 flush.
5677	*
5678	* @param pVM The cross context VM structure.
5679	*/
5680	void pgmR3PoolReset(PVM pVM)
5681	{
5682	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5683
5684	PGM_LOCK_ASSERT_OWNER(pVM);
5685	STAM_PROFILE_START(&pPool->StatR3Reset, a);
5686	LogFlow(("pgmR3PoolReset:\n"));
5687
5688	/*
5689	* If there are no pages in the pool, there is nothing to do.
5690	*/
5691	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
5692	{
5693	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5694	return;
5695	}
5696
5697	/*
5698	* Exit the shadow mode since we're going to clear everything,
5699	* including the root page.
5700	*/
5701	VMCC_FOR_EACH_VMCPU(pVM)
5702	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5703	VMCC_FOR_EACH_VMCPU_END(pVM);
5704
5705
5706	/*
5707	* Nuke the free list and reinsert all pages into it.
5708	*/
5709	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
5710	{
5711	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5712
5713	if (pPage->fMonitored)
5714	pgmPoolMonitorFlush(pPool, pPage);
5715	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5716	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5717	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
5718	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
5719	pPage->GCPhys = NIL_RTGCPHYS;
5720	pPage->enmKind = PGMPOOLKIND_FREE;
5721	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5722	Assert(pPage->idx == i);
5723	pPage->iNext = i + 1;
5724	pPage->fA20Enabled = true;
5725	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
5726	pPage->fSeenNonGlobal = false;
5727	pPage->fMonitored = false;
5728	pPage->fDirty = false;
5729	pPage->fCached = false;
5730	pPage->fReusedFlushPending = false;
5731	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
5732	pPage->cPresent = 0;
5733	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5734	pPage->cModifications = 0;
5735	pPage->iAgeNext = NIL_PGMPOOL_IDX;
5736	pPage->iAgePrev = NIL_PGMPOOL_IDX;
5737	pPage->idxDirtyEntry = 0;
5738	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5739	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5740	pPage->cLastAccessHandler = 0;
5741	pPage->cLocked = 0;
5742	# ifdef VBOX_STRICT
5743	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5744	# endif
5745	}
5746	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
5747	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
5748	pPool->cUsedPages = 0;
5749
5750	/*
5751	* Zap and reinitialize the user records.
5752	*/
5753	pPool->cPresent = 0;
5754	pPool->iUserFreeHead = 0;
5755	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
5756	const unsigned cMaxUsers = pPool->cMaxUsers;
5757	for (unsigned i = 0; i < cMaxUsers; i++)
5758	{
5759	paUsers[i].iNext = i + 1;
5760	paUsers[i].iUser = NIL_PGMPOOL_IDX;
5761	paUsers[i].iUserTable = 0xfffffffe;
5762	}
5763	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
5764
5765	/*
5766	* Clear all the GCPhys links and rebuild the phys ext free list.
5767	*/
5768	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
5769	pRam;
5770	pRam = pRam->CTX_SUFF(pNext))
5771	{
5772	unsigned iPage = pRam->cb >> PAGE_SHIFT;
5773	while (iPage-- > 0)
5774	PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
5775	}
5776
5777	pPool->iPhysExtFreeHead = 0;
5778	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
5779	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
5780	for (unsigned i = 0; i < cMaxPhysExts; i++)
5781	{
5782	paPhysExts[i].iNext = i + 1;
5783	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
5784	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5785	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
5786	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5787	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
5788	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5789	}
5790	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
5791
5792	/*
5793	* Just zap the modified list.
5794	*/
5795	pPool->cModifiedPages = 0;
5796	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
5797
5798	/*
5799	* Clear the GCPhys hash and the age list.
5800	*/
5801	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
5802	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
5803	pPool->iAgeHead = NIL_PGMPOOL_IDX;
5804	pPool->iAgeTail = NIL_PGMPOOL_IDX;
5805
5806	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5807	/* Clear all dirty pages. */
5808	pPool->idxFreeDirtyPage = 0;
5809	pPool->cDirtyPages = 0;
5810	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++)
5811	pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
5812	# endif
5813
5814	/*
5815	* Reinsert active pages into the hash and ensure monitoring chains are correct.
5816	*/
5817	VMCC_FOR_EACH_VMCPU(pVM)
5818	{
5819	/*
5820	* Re-enter the shadowing mode and assert Sync CR3 FF.
5821	*/
5822	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5823	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5824	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5825	}
5826	VMCC_FOR_EACH_VMCPU_END(pVM);
5827
5828	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5829	}
5830
5831	#endif /* IN_RING3 */
5832
5833	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
5834	/**
5835	* Stringifies a PGMPOOLKIND value.
5836	*/
5837	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
5838	{
5839	switch ((PGMPOOLKIND)enmKind)
5840	{
5841	case PGMPOOLKIND_INVALID:
5842	return "PGMPOOLKIND_INVALID";
5843	case PGMPOOLKIND_FREE:
5844	return "PGMPOOLKIND_FREE";
5845	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5846	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
5847	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5848	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
5849	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5850	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
5851	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5852	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
5853	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5854	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
5855	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5856	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
5857	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5858	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
5859	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5860	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
5861	case PGMPOOLKIND_32BIT_PD:
5862	return "PGMPOOLKIND_32BIT_PD";
5863	case PGMPOOLKIND_32BIT_PD_PHYS:
5864	return "PGMPOOLKIND_32BIT_PD_PHYS";
5865	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5866	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
5867	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5868	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
5869	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5870	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
5871	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5872	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
5873	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5874	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
5875	case PGMPOOLKIND_PAE_PD_PHYS:
5876	return "PGMPOOLKIND_PAE_PD_PHYS";
5877	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5878	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
5879	case PGMPOOLKIND_PAE_PDPT:
5880	return "PGMPOOLKIND_PAE_PDPT";
5881	case PGMPOOLKIND_PAE_PDPT_PHYS:
5882	return "PGMPOOLKIND_PAE_PDPT_PHYS";
5883	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5884	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
5885	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5886	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
5887	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5888	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
5889	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5890	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
5891	case PGMPOOLKIND_64BIT_PML4:
5892	return "PGMPOOLKIND_64BIT_PML4";
5893	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5894	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
5895	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5896	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
5897	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5898	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
5899	case PGMPOOLKIND_ROOT_NESTED:
5900	return "PGMPOOLKIND_ROOT_NESTED";
5901	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
5902	return "PGMPOOLKIND_EPT_PT_FOR_EPT_PT";
5903	case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
5904	return "PGMPOOLKIND_EPT_PT_FOR_EPT_2MB";
5905	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
5906	return "PGMPOOLKIND_EPT_PD_FOR_EPT_PD";
5907	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
5908	return "PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT";
5909	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
5910	return "PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4";
5911	}
5912	return "Unknown kind!";
5913	}
5914	#endif /* LOG_ENABLED \|\| VBOX_STRICT */
5915

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 103832

Download in other formats: