PGMPool.cpp@ 33462

Last change on this file since 33462 was 33343, checked in by vboxsync, 14 years ago
Fixed missing clearing of large page PDs when clearing the page pool.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 53.6 KB

Line
1	/* $Id: PGMPool.cpp 33343 2010-10-22 11:36:56Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/** @page pg_pgm_pool PGM Shadow Page Pool
19	*
20	* Motivations:
21	* -# Relationship between shadow page tables and physical guest pages. This
22	* should allow us to skip most of the global flushes now following access
23	* handler changes. The main expense is flushing shadow pages.
24	* -# Limit the pool size if necessary (default is kind of limitless).
25	* -# Allocate shadow pages from RC. We use to only do this in SyncCR3.
26	* -# Required for 64-bit guests.
27	* -# Combining the PD cache and page pool in order to simplify caching.
28	*
29	*
30	* @section sec_pgm_pool_outline Design Outline
31	*
32	* The shadow page pool tracks pages used for shadowing paging structures (i.e.
33	* page tables, page directory, page directory pointer table and page map
34	* level-4). Each page in the pool has an unique identifier. This identifier is
35	* used to link a guest physical page to a shadow PT. The identifier is a
36	* non-zero value and has a relativly low max value - say 14 bits. This makes it
37	* possible to fit it into the upper bits of the of the aHCPhys entries in the
38	* ram range.
39	*
40	* By restricting host physical memory to the first 48 bits (which is the
41	* announced physical memory range of the K8L chip (scheduled for 2008)), we
42	* can safely use the upper 16 bits for shadow page ID and reference counting.
43	*
44	* Update: The 48 bit assumption will be lifted with the new physical memory
45	* management (PGMPAGE), so we won't have any trouble when someone stuffs 2TB
46	* into a box in some years.
47	*
48	* Now, it's possible for a page to be aliased, i.e. mapped by more than one PT
49	* or PD. This is solved by creating a list of physical cross reference extents
50	* when ever this happens. Each node in the list (extent) is can contain 3 page
51	* pool indexes. The list it self is chained using indexes into the paPhysExt
52	* array.
53	*
54	*
55	* @section sec_pgm_pool_life Life Cycle of a Shadow Page
56	*
57	* -# The SyncPT function requests a page from the pool.
58	* The request includes the kind of page it is (PT/PD, PAE/legacy), the
59	* address of the page it's shadowing, and more.
60	* -# The pool responds to the request by allocating a new page.
61	* When the cache is enabled, it will first check if it's in the cache.
62	* Should the pool be exhausted, one of two things can be done:
63	* -# Flush the whole pool and current CR3.
64	* -# Use the cache to find a page which can be flushed (~age).
65	* -# The SyncPT function will sync one or more pages and insert it into the
66	* shadow PD.
67	* -# The SyncPage function may sync more pages on a later \#PFs.
68	* -# The page is freed / flushed in SyncCR3 (perhaps) and some other cases.
69	* When caching is enabled, the page isn't flush but remains in the cache.
70	*
71	*
72	* @section sec_pgm_pool_impl Monitoring
73	*
74	* We always monitor PAGE_SIZE chunks of memory. When we've got multiple shadow
75	* pages for the same PAGE_SIZE of guest memory (PAE and mixed PD/PT) the pages
76	* sharing the monitor get linked using the iMonitoredNext/Prev. The head page
77	* is the pvUser to the access handlers.
78	*
79	*
80	* @section sec_pgm_pool_impl Implementation
81	*
82	* The pool will take pages from the MM page pool. The tracking data
83	* (attributes, bitmaps and so on) are allocated from the hypervisor heap. The
84	* pool content can be accessed both by using the page id and the physical
85	* address (HC). The former is managed by means of an array, the latter by an
86	* offset based AVL tree.
87	*
88	* Flushing of a pool page means that we iterate the content (we know what kind
89	* it is) and updates the link information in the ram range.
90	*
91	* ...
92	*/
93
94
95	/*******************************************************************************
96	* Header Files *
97	*******************************************************************************/
98	#define LOG_GROUP LOG_GROUP_PGM_POOL
99	#include <VBox/pgm.h>
100	#include <VBox/mm.h>
101	#include "PGMInternal.h"
102	#include <VBox/vm.h>
103	#include "PGMInline.h"
104
105	#include <VBox/log.h>
106	#include <VBox/err.h>
107	#include <iprt/asm.h>
108	#include <iprt/string.h>
109	#include <VBox/dbg.h>
110
111
112	/*******************************************************************************
113	* Internal Functions *
114	*******************************************************************************/
115	static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
116	#ifdef VBOX_WITH_DEBUGGER
117	static DECLCALLBACK(int) pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
118	#endif
119
120	#ifdef VBOX_WITH_DEBUGGER
121	/** Command descriptors. */
122	static const DBGCCMD g_aCmds[] =
123	{
124	/* pszCmd, cArgsMin, cArgsMax, paArgDesc, cArgDescs, pResultDesc, fFlags, pfnHandler pszSyntax, ....pszDescription */
125	{ "pgmpoolcheck", 0, 0, NULL, 0, NULL, 0, pgmR3PoolCmdCheck, "", "Check the pgm pool pages." },
126	};
127	#endif
128
129	/**
130	* Initalizes the pool
131	*
132	* @returns VBox status code.
133	* @param pVM The VM handle.
134	*/
135	int pgmR3PoolInit(PVM pVM)
136	{
137	AssertCompile(NIL_PGMPOOL_IDX == 0);
138	/* pPage->cLocked is an unsigned byte. */
139	AssertCompile(VMM_MAX_CPU_COUNT <= 255);
140
141	/*
142	* Query Pool config.
143	*/
144	PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM/Pool");
145
146	/* Default pgm pool size equals 1024 pages. */
147	uint16_t cMaxPages = 4*_1M >> PAGE_SHIFT;
148
149	#if HC_ARCH_BITS == 64
150	uint64_t cbRam = 0;
151	CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
152
153	/* We should increase the pgm pool size for guests with more than 2 GB of ram */
154	if (cbRam >= UINT64_C(2) * _1G)
155	{
156	/* In the nested paging case we require 2 + 513 * (cbRam/1GB) pages to
157	* store the entire page table descriptors.
158	*/
159	uint64_t u64MaxPages = cbRam / (_1G / UINT64_C(512));
160	if (u64MaxPages > PGMPOOL_IDX_LAST)
161	cMaxPages = PGMPOOL_IDX_LAST;
162	else
163	cMaxPages = (uint16_t)u64MaxPages;
164	}
165	#endif
166
167	/** @cfgm{/PGM/Pool/MaxPages, uint16_t, #pages, 16, 0x3fff, 1024}
168	* The max size of the shadow page pool in pages. The pool will grow dynamically
169	* up to this limit.
170	*/
171	int rc = CFGMR3QueryU16Def(pCfg, "MaxPages", &cMaxPages, cMaxPages);
172	AssertLogRelRCReturn(rc, rc);
173	AssertLogRelMsgReturn(cMaxPages <= PGMPOOL_IDX_LAST && cMaxPages >= RT_ALIGN(PGMPOOL_IDX_FIRST, 16),
174	("cMaxPages=%u (%#x)\n", cMaxPages, cMaxPages), VERR_INVALID_PARAMETER);
175	cMaxPages = RT_ALIGN(cMaxPages, 16);
176
177	/** todo:
178	* We need to be much more careful with our allocation strategy here.
179	* For nested paging we don't need pool user info nor extents at all, but we can't check for nested paging here (too early during init to get a confirmation it can be used)
180	* The default for large memory configs is a bit large for shadow paging, so I've restricted the extent maximum to 2k (2k * 16 = 32k of hyper heap)
181	*
182	* Also when large page support is enabled, we typically don't need so much,
183	* although that depends on the availability of 2 MB chunks on the host.
184	*/
185
186	/** @cfgm{/PGM/Pool/MaxUsers, uint16_t, #users, MaxUsers, 32K, MaxPages*2}
187	* The max number of shadow page user tracking records. Each shadow page has
188	* zero of other shadow pages (or CR3s) that references it, or uses it if you
189	* like. The structures describing these relationships are allocated from a
190	* fixed sized pool. This configuration variable defines the pool size.
191	*/
192	uint16_t cMaxUsers;
193	rc = CFGMR3QueryU16Def(pCfg, "MaxUsers", &cMaxUsers, cMaxPages * 2);
194	AssertLogRelRCReturn(rc, rc);
195	AssertLogRelMsgReturn(cMaxUsers >= cMaxPages && cMaxPages <= _32K,
196	("cMaxUsers=%u (%#x)\n", cMaxUsers, cMaxUsers), VERR_INVALID_PARAMETER);
197
198	/** @cfgm{/PGM/Pool/MaxPhysExts, uint16_t, #extents, 16, MaxPages * 2, MAX(MaxPages*2,0x3fff)}
199	* The max number of extents for tracking aliased guest pages.
200	*/
201	uint16_t cMaxPhysExts;
202	rc = CFGMR3QueryU16Def(pCfg, "MaxPhysExts", &cMaxPhysExts, RT_MAX(cMaxPages * 2, 2048 /* 2k max as this eat too much hyper heap */));
203	AssertLogRelRCReturn(rc, rc);
204	AssertLogRelMsgReturn(cMaxPhysExts >= 16 && cMaxPages <= PGMPOOL_IDX_LAST,
205	("cMaxPhysExts=%u (%#x)\n", cMaxPhysExts, cMaxPhysExts), VERR_INVALID_PARAMETER);
206
207	/** @cfgm{/PGM/Pool/ChacheEnabled, bool, true}
208	* Enables or disabling caching of shadow pages. Caching means that we will try
209	* reuse shadow pages instead of recreating them everything SyncCR3, SyncPT or
210	* SyncPage requests one. When reusing a shadow page, we can save time
211	* reconstructing it and it's children.
212	*/
213	bool fCacheEnabled;
214	rc = CFGMR3QueryBoolDef(pCfg, "CacheEnabled", &fCacheEnabled, true);
215	AssertLogRelRCReturn(rc, rc);
216
217	LogRel(("pgmR3PoolInit: cMaxPages=%#RX16 cMaxUsers=%#RX16 cMaxPhysExts=%#RX16 fCacheEnable=%RTbool\n",
218	cMaxPages, cMaxUsers, cMaxPhysExts, fCacheEnabled));
219
220	/*
221	* Allocate the data structures.
222	*/
223	uint32_t cb = RT_OFFSETOF(PGMPOOL, aPages[cMaxPages]);
224	cb += cMaxUsers * sizeof(PGMPOOLUSER);
225	cb += cMaxPhysExts * sizeof(PGMPOOLPHYSEXT);
226	PPGMPOOL pPool;
227	rc = MMR3HyperAllocOnceNoRel(pVM, cb, 0, MM_TAG_PGM_POOL, (void **)&pPool);
228	if (RT_FAILURE(rc))
229	return rc;
230	pVM->pgm.s.pPoolR3 = pPool;
231	pVM->pgm.s.pPoolR0 = MMHyperR3ToR0(pVM, pPool);
232	pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pPool);
233
234	/*
235	* Initialize it.
236	*/
237	pPool->pVMR3 = pVM;
238	pPool->pVMR0 = pVM->pVMR0;
239	pPool->pVMRC = pVM->pVMRC;
240	pPool->cMaxPages = cMaxPages;
241	pPool->cCurPages = PGMPOOL_IDX_FIRST;
242	pPool->iUserFreeHead = 0;
243	pPool->cMaxUsers = cMaxUsers;
244	PPGMPOOLUSER paUsers = (PPGMPOOLUSER)&pPool->aPages[pPool->cMaxPages];
245	pPool->paUsersR3 = paUsers;
246	pPool->paUsersR0 = MMHyperR3ToR0(pVM, paUsers);
247	pPool->paUsersRC = MMHyperR3ToRC(pVM, paUsers);
248	for (unsigned i = 0; i < cMaxUsers; i++)
249	{
250	paUsers[i].iNext = i + 1;
251	paUsers[i].iUser = NIL_PGMPOOL_IDX;
252	paUsers[i].iUserTable = 0xfffffffe;
253	}
254	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
255	pPool->iPhysExtFreeHead = 0;
256	pPool->cMaxPhysExts = cMaxPhysExts;
257	PPGMPOOLPHYSEXT paPhysExts = (PPGMPOOLPHYSEXT)&paUsers[cMaxUsers];
258	pPool->paPhysExtsR3 = paPhysExts;
259	pPool->paPhysExtsR0 = MMHyperR3ToR0(pVM, paPhysExts);
260	pPool->paPhysExtsRC = MMHyperR3ToRC(pVM, paPhysExts);
261	for (unsigned i = 0; i < cMaxPhysExts; i++)
262	{
263	paPhysExts[i].iNext = i + 1;
264	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
265	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
266	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
267	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
268	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
269	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
270	}
271	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
272	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
273	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
274	pPool->iAgeHead = NIL_PGMPOOL_IDX;
275	pPool->iAgeTail = NIL_PGMPOOL_IDX;
276	pPool->fCacheEnabled = fCacheEnabled;
277	pPool->pfnAccessHandlerR3 = pgmR3PoolAccessHandler;
278	pPool->pszAccessHandler = "Guest Paging Access Handler";
279	pPool->HCPhysTree = 0;
280
281	/* The NIL entry. */
282	Assert(NIL_PGMPOOL_IDX == 0);
283	pPool->aPages[NIL_PGMPOOL_IDX].enmKind = PGMPOOLKIND_INVALID;
284
285	/* The Shadow 32-bit PD. (32 bits guest paging) */
286	pPool->aPages[PGMPOOL_IDX_PD].Core.Key = NIL_RTHCPHYS;
287	pPool->aPages[PGMPOOL_IDX_PD].GCPhys = NIL_RTGCPHYS;
288	pPool->aPages[PGMPOOL_IDX_PD].pvPageR3 = 0;
289	pPool->aPages[PGMPOOL_IDX_PD].enmKind = PGMPOOLKIND_32BIT_PD;
290	pPool->aPages[PGMPOOL_IDX_PD].idx = PGMPOOL_IDX_PD;
291
292	/* The Shadow PDPT. */
293	pPool->aPages[PGMPOOL_IDX_PDPT].Core.Key = NIL_RTHCPHYS;
294	pPool->aPages[PGMPOOL_IDX_PDPT].GCPhys = NIL_RTGCPHYS;
295	pPool->aPages[PGMPOOL_IDX_PDPT].pvPageR3 = 0;
296	pPool->aPages[PGMPOOL_IDX_PDPT].enmKind = PGMPOOLKIND_PAE_PDPT;
297	pPool->aPages[PGMPOOL_IDX_PDPT].idx = PGMPOOL_IDX_PDPT;
298
299	/* The Shadow AMD64 CR3. */
300	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].Core.Key = NIL_RTHCPHYS;
301	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].GCPhys = NIL_RTGCPHYS;
302	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].pvPageR3 = 0;
303	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].enmKind = PGMPOOLKIND_64BIT_PML4;
304	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].idx = PGMPOOL_IDX_AMD64_CR3;
305
306	/* The Nested Paging CR3. */
307	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].Core.Key = NIL_RTHCPHYS;
308	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].GCPhys = NIL_RTGCPHYS;
309	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].pvPageR3 = 0;
310	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].enmKind = PGMPOOLKIND_ROOT_NESTED;
311	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].idx = PGMPOOL_IDX_NESTED_ROOT;
312
313	/*
314	* Set common stuff.
315	*/
316	for (unsigned iPage = 1; iPage < PGMPOOL_IDX_FIRST; iPage++)
317	{
318	pPool->aPages[iPage].iNext = NIL_PGMPOOL_IDX;
319	pPool->aPages[iPage].iUserHead = NIL_PGMPOOL_USER_INDEX;
320	pPool->aPages[iPage].iModifiedNext = NIL_PGMPOOL_IDX;
321	pPool->aPages[iPage].iModifiedPrev = NIL_PGMPOOL_IDX;
322	pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
323	pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
324	pPool->aPages[iPage].iAgeNext = NIL_PGMPOOL_IDX;
325	pPool->aPages[iPage].iAgePrev = NIL_PGMPOOL_IDX;
326	Assert(pPool->aPages[iPage].idx == iPage);
327	Assert(pPool->aPages[iPage].GCPhys == NIL_RTGCPHYS);
328	Assert(!pPool->aPages[iPage].fSeenNonGlobal);
329	Assert(!pPool->aPages[iPage].fMonitored);
330	Assert(!pPool->aPages[iPage].fCached);
331	Assert(!pPool->aPages[iPage].fZeroed);
332	Assert(!pPool->aPages[iPage].fReusedFlushPending);
333	}
334
335	#ifdef VBOX_WITH_STATISTICS
336	/*
337	* Register statistics.
338	*/
339	STAM_REG(pVM, &pPool->cCurPages, STAMTYPE_U16, "/PGM/Pool/cCurPages", STAMUNIT_PAGES, "Current pool size.");
340	STAM_REG(pVM, &pPool->cMaxPages, STAMTYPE_U16, "/PGM/Pool/cMaxPages", STAMUNIT_PAGES, "Max pool size.");
341	STAM_REG(pVM, &pPool->cUsedPages, STAMTYPE_U16, "/PGM/Pool/cUsedPages", STAMUNIT_PAGES, "The number of pages currently in use.");
342	STAM_REG(pVM, &pPool->cUsedPagesHigh, STAMTYPE_U16_RESET, "/PGM/Pool/cUsedPagesHigh", STAMUNIT_PAGES, "The high watermark for cUsedPages.");
343	STAM_REG(pVM, &pPool->StatAlloc, STAMTYPE_PROFILE_ADV, "/PGM/Pool/Alloc", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolAlloc.");
344	STAM_REG(pVM, &pPool->StatClearAll, STAMTYPE_PROFILE, "/PGM/Pool/ClearAll", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmR3PoolClearAll.");
345	STAM_REG(pVM, &pPool->StatR3Reset, STAMTYPE_PROFILE, "/PGM/Pool/R3Reset", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmR3PoolReset.");
346	STAM_REG(pVM, &pPool->StatFlushPage, STAMTYPE_PROFILE, "/PGM/Pool/FlushPage", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolFlushPage.");
347	STAM_REG(pVM, &pPool->StatFree, STAMTYPE_PROFILE, "/PGM/Pool/Free", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolFree.");
348	STAM_REG(pVM, &pPool->StatForceFlushPage, STAMTYPE_COUNTER, "/PGM/Pool/FlushForce", STAMUNIT_OCCURENCES, "Counting explicit flushes by PGMPoolFlushPage().");
349	STAM_REG(pVM, &pPool->StatForceFlushDirtyPage, STAMTYPE_COUNTER, "/PGM/Pool/FlushForceDirty", STAMUNIT_OCCURENCES, "Counting explicit flushes of dirty pages by PGMPoolFlushPage().");
350	STAM_REG(pVM, &pPool->StatForceFlushReused, STAMTYPE_COUNTER, "/PGM/Pool/FlushReused", STAMUNIT_OCCURENCES, "Counting flushes for reused pages.");
351	STAM_REG(pVM, &pPool->StatZeroPage, STAMTYPE_PROFILE, "/PGM/Pool/ZeroPage", STAMUNIT_TICKS_PER_CALL, "Profiling time spent zeroing pages. Overlaps with Alloc.");
352	STAM_REG(pVM, &pPool->cMaxUsers, STAMTYPE_U16, "/PGM/Pool/Track/cMaxUsers", STAMUNIT_COUNT, "Max user tracking records.");
353	STAM_REG(pVM, &pPool->cPresent, STAMTYPE_U32, "/PGM/Pool/Track/cPresent", STAMUNIT_COUNT, "Number of present page table entries.");
354	STAM_REG(pVM, &pPool->StatTrackDeref, STAMTYPE_PROFILE, "/PGM/Pool/Track/Deref", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackDeref.");
355	STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPT, STAMTYPE_PROFILE, "/PGM/Pool/Track/FlushGCPhysPT", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPT.");
356	STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTs, STAMTYPE_PROFILE, "/PGM/Pool/Track/FlushGCPhysPTs", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTs.");
357	STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTsSlow, STAMTYPE_PROFILE, "/PGM/Pool/Track/FlushGCPhysPTsSlow", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTsSlow.");
358	STAM_REG(pVM, &pPool->StatTrackFlushEntry, STAMTYPE_COUNTER, "/PGM/Pool/Track/Entry/Flush", STAMUNIT_COUNT, "Nr of flushed entries.");
359	STAM_REG(pVM, &pPool->StatTrackFlushEntryKeep, STAMTYPE_COUNTER, "/PGM/Pool/Track/Entry/Update", STAMUNIT_COUNT, "Nr of updated entries.");
360	STAM_REG(pVM, &pPool->StatTrackFreeUpOneUser, STAMTYPE_COUNTER, "/PGM/Pool/Track/FreeUpOneUser", STAMUNIT_TICKS_PER_CALL, "The number of times we were out of user tracking records.");
361	STAM_REG(pVM, &pPool->StatTrackDerefGCPhys, STAMTYPE_PROFILE, "/PGM/Pool/Track/DrefGCPhys", STAMUNIT_TICKS_PER_CALL, "Profiling deref activity related tracking GC physical pages.");
362	STAM_REG(pVM, &pPool->StatTrackLinearRamSearches, STAMTYPE_COUNTER, "/PGM/Pool/Track/LinearRamSearches", STAMUNIT_OCCURENCES, "The number of times we had to do linear ram searches.");
363	STAM_REG(pVM, &pPool->StamTrackPhysExtAllocFailures,STAMTYPE_COUNTER, "/PGM/Pool/Track/PhysExtAllocFailures", STAMUNIT_OCCURENCES, "The number of failing pgmPoolTrackPhysExtAlloc calls.");
364	STAM_REG(pVM, &pPool->StatMonitorRZ, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ", STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access handler.");
365	STAM_REG(pVM, &pPool->StatMonitorRZEmulateInstr, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/EmulateInstr", STAMUNIT_OCCURENCES, "Times we've failed interpreting the instruction.");
366	STAM_REG(pVM, &pPool->StatMonitorRZFlushPage, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/FlushPage", STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler.");
367	STAM_REG(pVM, &pPool->StatMonitorRZFlushReinit, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/FlushReinit", STAMUNIT_OCCURENCES, "Times we've detected a page table reinit.");
368	STAM_REG(pVM, &pPool->StatMonitorRZFlushModOverflow,STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/FlushOverflow", STAMUNIT_OCCURENCES, "Counting flushes for pages that are modified too often.");
369	STAM_REG(pVM, &pPool->StatMonitorRZFork, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fork", STAMUNIT_OCCURENCES, "Times we've detected fork().");
370	STAM_REG(pVM, &pPool->StatMonitorRZHandled, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/Handled", STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access we've handled (except REP STOSD).");
371	STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch1, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/IntrFailPatch1", STAMUNIT_OCCURENCES, "Times we've failed interpreting a patch code instruction.");
372	STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch2, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/IntrFailPatch2", STAMUNIT_OCCURENCES, "Times we've failed interpreting a patch code instruction during flushing.");
373	STAM_REG(pVM, &pPool->StatMonitorRZRepPrefix, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/RepPrefix", STAMUNIT_OCCURENCES, "The number of times we've seen rep prefixes we can't handle.");
374	STAM_REG(pVM, &pPool->StatMonitorRZRepStosd, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/RepStosd", STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
375	STAM_REG(pVM, &pPool->StatMonitorRZFaultPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PT", STAMUNIT_OCCURENCES, "Nr of handled PT faults.");
376	STAM_REG(pVM, &pPool->StatMonitorRZFaultPD, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PD", STAMUNIT_OCCURENCES, "Nr of handled PD faults.");
377	STAM_REG(pVM, &pPool->StatMonitorRZFaultPDPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PDPT", STAMUNIT_OCCURENCES, "Nr of handled PDPT faults.");
378	STAM_REG(pVM, &pPool->StatMonitorRZFaultPML4, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PML4", STAMUNIT_OCCURENCES, "Nr of handled PML4 faults.");
379	STAM_REG(pVM, &pPool->StatMonitorR3, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3", STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access handler.");
380	STAM_REG(pVM, &pPool->StatMonitorR3EmulateInstr, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/EmulateInstr", STAMUNIT_OCCURENCES, "Times we've failed interpreting the instruction.");
381	STAM_REG(pVM, &pPool->StatMonitorR3FlushPage, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/FlushPage", STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the R3 access handler.");
382	STAM_REG(pVM, &pPool->StatMonitorR3FlushReinit, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/FlushReinit", STAMUNIT_OCCURENCES, "Times we've detected a page table reinit.");
383	STAM_REG(pVM, &pPool->StatMonitorR3FlushModOverflow,STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/FlushOverflow", STAMUNIT_OCCURENCES, "Counting flushes for pages that are modified too often.");
384	STAM_REG(pVM, &pPool->StatMonitorR3Fork, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fork", STAMUNIT_OCCURENCES, "Times we've detected fork().");
385	STAM_REG(pVM, &pPool->StatMonitorR3Handled, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Handled", STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access we've handled (except REP STOSD).");
386	STAM_REG(pVM, &pPool->StatMonitorR3RepPrefix, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/RepPrefix", STAMUNIT_OCCURENCES, "The number of times we've seen rep prefixes we can't handle.");
387	STAM_REG(pVM, &pPool->StatMonitorR3RepStosd, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/RepStosd", STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
388	STAM_REG(pVM, &pPool->StatMonitorR3FaultPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PT", STAMUNIT_OCCURENCES, "Nr of handled PT faults.");
389	STAM_REG(pVM, &pPool->StatMonitorR3FaultPD, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PD", STAMUNIT_OCCURENCES, "Nr of handled PD faults.");
390	STAM_REG(pVM, &pPool->StatMonitorR3FaultPDPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PDPT", STAMUNIT_OCCURENCES, "Nr of handled PDPT faults.");
391	STAM_REG(pVM, &pPool->StatMonitorR3FaultPML4, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PML4", STAMUNIT_OCCURENCES, "Nr of handled PML4 faults.");
392	STAM_REG(pVM, &pPool->StatMonitorR3Async, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Async", STAMUNIT_OCCURENCES, "Times we're called in an async thread and need to flush.");
393	STAM_REG(pVM, &pPool->cModifiedPages, STAMTYPE_U16, "/PGM/Pool/Monitor/cModifiedPages", STAMUNIT_PAGES, "The current cModifiedPages value.");
394	STAM_REG(pVM, &pPool->cModifiedPagesHigh, STAMTYPE_U16_RESET, "/PGM/Pool/Monitor/cModifiedPagesHigh", STAMUNIT_PAGES, "The high watermark for cModifiedPages.");
395	STAM_REG(pVM, &pPool->StatResetDirtyPages, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/Resets", STAMUNIT_OCCURENCES, "Times we've called pgmPoolResetDirtyPages (and there were dirty page).");
396	STAM_REG(pVM, &pPool->StatDirtyPage, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/Pages", STAMUNIT_OCCURENCES, "Times we've called pgmPoolAddDirtyPage.");
397	STAM_REG(pVM, &pPool->StatDirtyPageDupFlush, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/FlushDup", STAMUNIT_OCCURENCES, "Times we've had to flush duplicates for dirty page management.");
398	STAM_REG(pVM, &pPool->StatDirtyPageOverFlowFlush, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/FlushOverflow",STAMUNIT_OCCURENCES, "Times we've had to flush because of overflow.");
399	STAM_REG(pVM, &pPool->StatCacheHits, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Hits", STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls satisfied by the cache.");
400	STAM_REG(pVM, &pPool->StatCacheMisses, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Misses", STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls not statisfied by the cache.");
401	STAM_REG(pVM, &pPool->StatCacheKindMismatches, STAMTYPE_COUNTER, "/PGM/Pool/Cache/KindMismatches", STAMUNIT_OCCURENCES, "The number of shadow page kind mismatches. (Better be low, preferably 0!)");
402	STAM_REG(pVM, &pPool->StatCacheFreeUpOne, STAMTYPE_COUNTER, "/PGM/Pool/Cache/FreeUpOne", STAMUNIT_OCCURENCES, "The number of times the cache was asked to free up a page.");
403	STAM_REG(pVM, &pPool->StatCacheCacheable, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Cacheable", STAMUNIT_OCCURENCES, "The number of cacheable allocations.");
404	STAM_REG(pVM, &pPool->StatCacheUncacheable, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Uncacheable", STAMUNIT_OCCURENCES, "The number of uncacheable allocations.");
405	#endif /* VBOX_WITH_STATISTICS */
406
407	#ifdef VBOX_WITH_DEBUGGER
408	/*
409	* Debugger commands.
410	*/
411	static bool s_fRegisteredCmds = false;
412	if (!s_fRegisteredCmds)
413	{
414	rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
415	if (RT_SUCCESS(rc))
416	s_fRegisteredCmds = true;
417	}
418	#endif
419
420	return VINF_SUCCESS;
421	}
422
423
424	/**
425	* Relocate the page pool data.
426	*
427	* @param pVM The VM handle.
428	*/
429	void pgmR3PoolRelocate(PVM pVM)
430	{
431	pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3);
432	pVM->pgm.s.pPoolR3->pVMRC = pVM->pVMRC;
433	pVM->pgm.s.pPoolR3->paUsersRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paUsersR3);
434	pVM->pgm.s.pPoolR3->paPhysExtsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paPhysExtsR3);
435	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerRC);
436	AssertReleaseRC(rc);
437	/* init order hack. */
438	if (!pVM->pgm.s.pPoolR3->pfnAccessHandlerR0)
439	{
440	rc = PDMR3LdrGetSymbolR0(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerR0);
441	AssertReleaseRC(rc);
442	}
443	}
444
445
446	/**
447	* Grows the shadow page pool.
448	*
449	* I.e. adds more pages to it, assuming that hasn't reached cMaxPages yet.
450	*
451	* @returns VBox status code.
452	* @param pVM The VM handle.
453	*/
454	VMMR3DECL(int) PGMR3PoolGrow(PVM pVM)
455	{
456	PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
457	AssertReturn(pPool->cCurPages < pPool->cMaxPages, VERR_INTERNAL_ERROR);
458
459	pgmLock(pVM);
460
461	/*
462	* How much to grow it by?
463	*/
464	uint32_t cPages = pPool->cMaxPages - pPool->cCurPages;
465	cPages = RT_MIN(PGMPOOL_CFG_MAX_GROW, cPages);
466	LogFlow(("PGMR3PoolGrow: Growing the pool by %d (%#x) pages.\n", cPages, cPages));
467
468	for (unsigned i = pPool->cCurPages; cPages-- > 0; i++)
469	{
470	PPGMPOOLPAGE pPage = &pPool->aPages[i];
471
472	/* Allocate all pages in low (below 4 GB) memory as 32 bits guests need a page table root in low memory. */
473	pPage->pvPageR3 = MMR3PageAllocLow(pVM);
474	if (!pPage->pvPageR3)
475	{
476	Log(("We're out of memory!! i=%d\n", i));
477	pgmUnlock(pVM);
478	return i ? VINF_SUCCESS : VERR_NO_PAGE_MEMORY;
479	}
480	pPage->Core.Key = MMPage2Phys(pVM, pPage->pvPageR3);
481	AssertFatal(pPage->Core.Key < _4G);
482	pPage->GCPhys = NIL_RTGCPHYS;
483	pPage->enmKind = PGMPOOLKIND_FREE;
484	pPage->idx = pPage - &pPool->aPages[0];
485	LogFlow(("PGMR3PoolGrow: insert page #%#x - %RHp\n", pPage->idx, pPage->Core.Key));
486	pPage->iNext = pPool->iFreeHead;
487	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
488	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
489	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
490	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
491	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
492	pPage->iAgeNext = NIL_PGMPOOL_IDX;
493	pPage->iAgePrev = NIL_PGMPOOL_IDX;
494	/* commit it */
495	bool fRc = RTAvloHCPhysInsert(&pPool->HCPhysTree, &pPage->Core); Assert(fRc); NOREF(fRc);
496	pPool->iFreeHead = i;
497	pPool->cCurPages = i + 1;
498	}
499
500	pgmUnlock(pVM);
501	Assert(pPool->cCurPages <= pPool->cMaxPages);
502	return VINF_SUCCESS;
503	}
504
505
506
507	/**
508	* Worker used by pgmR3PoolAccessHandler when it's invoked by an async thread.
509	*
510	* @param pPool The pool.
511	* @param pPage The page.
512	*/
513	static DECLCALLBACK(void) pgmR3PoolFlushReusedPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
514	{
515	/* for the present this should be safe enough I think... */
516	pgmLock(pPool->pVMR3);
517	if ( pPage->fReusedFlushPending
518	&& pPage->enmKind != PGMPOOLKIND_FREE)
519	pgmPoolFlushPage(pPool, pPage);
520	pgmUnlock(pPool->pVMR3);
521	}
522
523
524	/**
525	* \#PF Handler callback for PT write accesses.
526	*
527	* The handler can not raise any faults, it's mainly for monitoring write access
528	* to certain pages.
529	*
530	* @returns VINF_SUCCESS if the handler has carried out the operation.
531	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
532	* @param pVM VM Handle.
533	* @param GCPhys The physical address the guest is writing to.
534	* @param pvPhys The HC mapping of that address.
535	* @param pvBuf What the guest is reading/writing.
536	* @param cbBuf How much it's reading/writing.
537	* @param enmAccessType The access type.
538	* @param pvUser User argument.
539	*/
540	static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
541	{
542	STAM_PROFILE_START(&pVM->pgm.s.pPoolR3->StatMonitorR3, a);
543	PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
544	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
545	LogFlow(("pgmR3PoolAccessHandler: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
546	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
547
548	PVMCPU pVCpu = VMMGetCpu(pVM);
549
550	/*
551	* We don't have to be very sophisticated about this since there are relativly few calls here.
552	* However, we must try our best to detect any non-cpu accesses (disk / networking).
553	*
554	* Just to make life more interesting, we'll have to deal with the async threads too.
555	* We cannot flush a page if we're in an async thread because of REM notifications.
556	*/
557	pgmLock(pVM);
558	if (PHYS_PAGE_ADDRESS(GCPhys) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
559	{
560	/* Pool page changed while we were waiting for the lock; ignore. */
561	Log(("CPU%d: pgmR3PoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
562	pgmUnlock(pVM);
563	return VINF_PGM_HANDLER_DO_DEFAULT;
564	}
565
566	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
567
568	/* @todo this code doesn't make any sense. remove the if (!pVCpu) block */
569	if (!pVCpu) /** @todo This shouldn't happen any longer, all access handlers will be called on an EMT. All ring-3 handlers, except MMIO, already own the PGM lock. @bugref{3170} */
570	{
571	Log(("pgmR3PoolAccessHandler: async thread, requesting EMT to flush the page: %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
572	pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
573	STAM_COUNTER_INC(&pPool->StatMonitorR3Async);
574	if (!pPage->fReusedFlushPending)
575	{
576	pgmUnlock(pVM);
577	int rc = VMR3ReqCallVoidNoWait(pPool->pVMR3, VMCPUID_ANY, (PFNRT)pgmR3PoolFlushReusedPage, 2, pPool, pPage);
578	AssertRCReturn(rc, rc);
579	pgmLock(pVM);
580	pPage->fReusedFlushPending = true;
581	pPage->cModifications += 0x1000;
582	}
583
584	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
585	/** @todo r=bird: making unsafe assumption about not crossing entries here! */
586	while (cbBuf > 4)
587	{
588	cbBuf -= 4;
589	pvPhys = (uint8_t *)pvPhys + 4;
590	GCPhys += 4;
591	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
592	}
593	STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
594	}
595	else if ( ( pPage->cModifications < 96 /* it's cheaper here. */
596	\|\| pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
597	)
598	&& cbBuf <= 4)
599	{
600	/* Clear the shadow entry. */
601	if (!pPage->cModifications++)
602	pgmPoolMonitorModifiedInsert(pPool, pPage);
603	/** @todo r=bird: making unsafe assumption about not crossing entries here! */
604	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
605	STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
606	}
607	else
608	{
609	pgmPoolMonitorChainFlush(pPool, pPage); /* ASSUME that VERR_PGM_POOL_CLEARED can be ignored here and that FFs will deal with it in due time. */
610	STAM_PROFILE_STOP_EX(&pPool->StatMonitorR3, &pPool->StatMonitorR3FlushPage, a);
611	}
612	pgmUnlock(pVM);
613	return VINF_PGM_HANDLER_DO_DEFAULT;
614	}
615
616
617	/**
618	* Rendezvous callback used by pgmR3PoolClearAll that clears all shadow pages
619	* and all modification counters.
620	*
621	* This is only called on one of the EMTs while the other ones are waiting for
622	* it to complete this function.
623	*
624	* @returns VINF_SUCCESS (VBox strict status code).
625	* @param pVM The VM handle.
626	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
627	* @param fpvFlushRemTlb When not NULL, we'll flush the REM TLB as well.
628	* (This is the pvUser, so it has to be void *.)
629	*
630	*/
631	DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl)
632	{
633	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
634	STAM_PROFILE_START(&pPool->StatClearAll, c);
635
636	pgmLock(pVM);
637	Log(("pgmR3PoolClearAllRendezvous: cUsedPages=%d fpvFlushRemTbl=%RTbool\n", pPool->cUsedPages, !!fpvFlushRemTbl));
638
639	/*
640	* Iterate all the pages until we've encountered all that are in use.
641	* This is a simple but not quite optimal solution.
642	*/
643	unsigned cModifiedPages = 0; NOREF(cModifiedPages);
644	unsigned cLeft = pPool->cUsedPages;
645	unsigned iPage = pPool->cCurPages;
646	while (--iPage >= PGMPOOL_IDX_FIRST)
647	{
648	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
649	if (pPage->GCPhys != NIL_RTGCPHYS)
650	{
651	switch (pPage->enmKind)
652	{
653	/*
654	* We only care about shadow page tables that reference physical memory
655	*/
656	#ifdef PGM_WITH_LARGE_PAGES
657	case PGMPOOLKIND_EPT_PD_FOR_PHYS: /* Large pages reference 2 MB of physical memory, so we must clear them. */
658	if (pPage->cPresent)
659	{
660	PX86PDPAE pShwPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
661	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
662	{
663	Assert((pShwPD->a[i].u & (X86_PDE_PAE_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
664	if ( pShwPD->a[i].n.u1Present
665	&& pShwPD->a[i].b.u1Size)
666	{
667	Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
668	pShwPD->a[i].u = 0;
669	Assert(pPage->cPresent);
670	pPage->cPresent--;
671	}
672	}
673	if (pPage->cPresent == 0)
674	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
675	}
676	goto default_case;
677
678	case PGMPOOLKIND_PAE_PD_PHYS: /* Large pages reference 2 MB of physical memory, so we must clear them. */
679	if (pPage->cPresent)
680	{
681	PEPTPD pShwPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
682	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
683	{
684	Assert((pShwPD->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
685	if ( pShwPD->a[i].n.u1Present
686	&& pShwPD->a[i].b.u1Size)
687	{
688	Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
689	pShwPD->a[i].u = 0;
690	Assert(pPage->cPresent);
691	pPage->cPresent--;
692	}
693	}
694	if (pPage->cPresent == 0)
695	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
696	}
697	goto default_case;
698	#endif /* PGM_WITH_LARGE_PAGES */
699
700	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
701	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
702	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
703	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
704	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
705	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
706	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
707	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
708	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
709	{
710	if (pPage->cPresent)
711	{
712	void *pvShw = PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
713	STAM_PROFILE_START(&pPool->StatZeroPage, z);
714	#if 0
715	/* Useful check for leaking references; very expensive though. */
716	switch (pPage->enmKind)
717	{
718	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
719	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
720	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
721	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
722	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
723	{
724	bool fFoundFirst = false;
725	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)pvShw;
726	for (unsigned ptIndex = 0; ptIndex < RT_ELEMENTS(pPT->a); ptIndex++)
727	{
728	if (pPT->a[ptIndex].u)
729	{
730	if (!fFoundFirst)
731	{
732	AssertFatalMsg(pPage->iFirstPresent <= ptIndex, ("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
733	if (pPage->iFirstPresent != ptIndex)
734	Log(("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
735	fFoundFirst = true;
736	}
737	if (PGMSHWPTEPAE_IS_P(pPT->a[ptIndex]))
738	{
739	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pPT->a[ptIndex]), NIL_RTGCPHYS);
740	if (pPage->iFirstPresent == ptIndex)
741	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
742	}
743	}
744	}
745	AssertFatalMsg(pPage->cPresent == 0, ("cPresent = %d pPage = %RGv\n", pPage->cPresent, pPage->GCPhys));
746	break;
747	}
748	default:
749	break;
750	}
751	#endif
752	ASMMemZeroPage(pvShw);
753	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
754	pPage->cPresent = 0;
755	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
756	}
757	}
758	/* fall thru */
759
760	#ifdef PGM_WITH_LARGE_PAGES
761	default_case:
762	#endif
763	default:
764	Assert(!pPage->cModifications \|\| ++cModifiedPages);
765	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
766	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
767	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
768	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
769	pPage->cModifications = 0;
770	break;
771
772	}
773	if (!--cLeft)
774	break;
775	}
776	}
777
778	/* swipe the special pages too. */
779	for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
780	{
781	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
782	if (pPage->GCPhys != NIL_RTGCPHYS)
783	{
784	Assert(!pPage->cModifications \|\| ++cModifiedPages);
785	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
786	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
787	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
788	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
789	pPage->cModifications = 0;
790	}
791	}
792
793	#ifndef DEBUG_michael
794	AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
795	#endif
796	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
797	pPool->cModifiedPages = 0;
798
799	/*
800	* Clear all the GCPhys links and rebuild the phys ext free list.
801	*/
802	for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
803	pRam;
804	pRam = pRam->CTX_SUFF(pNext))
805	{
806	iPage = pRam->cb >> PAGE_SHIFT;
807	while (iPage-- > 0)
808	PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
809	}
810
811	pPool->iPhysExtFreeHead = 0;
812	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
813	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
814	for (unsigned i = 0; i < cMaxPhysExts; i++)
815	{
816	paPhysExts[i].iNext = i + 1;
817	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
818	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
819	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
820	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
821	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
822	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
823	}
824	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
825
826
827	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
828	/* Reset all dirty pages to reactivate the page monitoring. */
829	/* Note: we must do this after clearing all page references and shadow page tables as there might be stale references to
830	* recently removed MMIO ranges around that might otherwise end up asserting in pgmPoolTracDerefGCPhysHint
831	*/
832	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
833	{
834	PPGMPOOLPAGE pPage;
835	unsigned idxPage;
836
837	if (pPool->aDirtyPages[i].uIdx == NIL_PGMPOOL_IDX)
838	continue;
839
840	idxPage = pPool->aDirtyPages[i].uIdx;
841	AssertRelease(idxPage != NIL_PGMPOOL_IDX);
842	pPage = &pPool->aPages[idxPage];
843	Assert(pPage->idx == idxPage);
844	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
845
846	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, i));
847
848	Log(("Reactivate dirty page %RGp\n", pPage->GCPhys));
849
850	/* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
851	int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
852	Assert(rc == VINF_SUCCESS);
853	pPage->fDirty = false;
854
855	pPool->aDirtyPages[i].uIdx = NIL_PGMPOOL_IDX;
856	}
857
858	/* Clear all dirty pages. */
859	pPool->idxFreeDirtyPage = 0;
860	pPool->cDirtyPages = 0;
861	#endif
862
863	/* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
864	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
865	pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
866
867	/* Flush job finished. */
868	VM_FF_CLEAR(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
869	pPool->cPresent = 0;
870	pgmUnlock(pVM);
871
872	PGM_INVL_ALL_VCPU_TLBS(pVM);
873
874	if (fpvFlushRemTbl)
875	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
876	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
877
878	STAM_PROFILE_STOP(&pPool->StatClearAll, c);
879	return VINF_SUCCESS;
880	}
881
882
883	/**
884	* Clears the shadow page pool.
885	*
886	* @param pVM The VM handle.
887	* @param fFlushRemTlb When set, the REM TLB is scheduled for flushing as
888	* well.
889	*/
890	void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb)
891	{
892	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PoolClearAllRendezvous, &fFlushRemTlb);
893	AssertRC(rc);
894	}
895
896	/**
897	* Protect all pgm pool page table entries to monitor writes
898	*
899	* @param pVM The VM handle.
900	*
901	* Remark: assumes the caller will flush all TLBs (!!)
902	*/
903	void pgmR3PoolWriteProtectPages(PVM pVM)
904	{
905	Assert(PGMIsLockOwner(pVM));
906	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
907	unsigned cLeft = pPool->cUsedPages;
908	unsigned iPage = pPool->cCurPages;
909	while (--iPage >= PGMPOOL_IDX_FIRST)
910	{
911	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
912	if ( pPage->GCPhys != NIL_RTGCPHYS
913	&& pPage->cPresent)
914	{
915	union
916	{
917	void *pv;
918	PX86PT pPT;
919	PPGMSHWPTPAE pPTPae;
920	PEPTPT pPTEpt;
921	} uShw;
922	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
923
924	switch (pPage->enmKind)
925	{
926	/*
927	* We only care about shadow page tables.
928	*/
929	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
930	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
931	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
932	for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPT->a); iShw++)
933	{
934	if (uShw.pPT->a[iShw].n.u1Present)
935	uShw.pPT->a[iShw].n.u1Write = 0;
936	}
937	break;
938
939	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
940	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
941	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
942	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
943	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
944	for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTPae->a); iShw++)
945	{
946	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
947	PGMSHWPTEPAE_SET_RO(uShw.pPTPae->a[iShw]);
948	}
949	break;
950
951	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
952	for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTEpt->a); iShw++)
953	{
954	if (uShw.pPTEpt->a[iShw].n.u1Present)
955	uShw.pPTEpt->a[iShw].n.u1Write = 0;
956	}
957	break;
958
959	default:
960	break;
961	}
962	if (!--cLeft)
963	break;
964	}
965	}
966	}
967
968	#ifdef VBOX_WITH_DEBUGGER
969	/**
970	* The '.pgmpoolcheck' command.
971	*
972	* @returns VBox status.
973	* @param pCmd Pointer to the command descriptor (as registered).
974	* @param pCmdHlp Pointer to command helper functions.
975	* @param pVM Pointer to the current VM (if any).
976	* @param paArgs Pointer to (readonly) array of arguments.
977	* @param cArgs Number of arguments in the array.
978	*/
979	static DECLCALLBACK(int) pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
980	{
981	/*
982	* Validate input.
983	*/
984	if (!pVM)
985	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
986
987	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
988
989	for (unsigned i = 0; i < pPool->cCurPages; i++)
990	{
991	PPGMPOOLPAGE pPage = &pPool->aPages[i];
992	bool fFirstMsg = true;
993
994	/* Todo: cover other paging modes too. */
995	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
996	{
997	PPGMSHWPTPAE pShwPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
998	{
999	PX86PTPAE pGstPT;
1000	PGMPAGEMAPLOCK LockPage;
1001	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, pPage->GCPhys, (const void **)&pGstPT, &LockPage); AssertReleaseRC(rc);
1002
1003	/* Check if any PTEs are out of sync. */
1004	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1005	{
1006	if (PGMSHWPTEPAE_IS_P(pShwPT->a[j]))
1007	{
1008	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1009	rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[j].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1010	if ( rc != VINF_SUCCESS
1011	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[j]) != HCPhys)
1012	{
1013	if (fFirstMsg)
1014	{
1015	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
1016	fFirstMsg = false;
1017	}
1018	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch HCPhys: rc=%Rrc idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
1019	}
1020	else if ( PGMSHWPTEPAE_IS_RW(pShwPT->a[j])
1021	&& !pGstPT->a[j].n.u1Write)
1022	{
1023	if (fFirstMsg)
1024	{
1025	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
1026	fFirstMsg = false;
1027	}
1028	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch r/w gst/shw: idx=%d guest %RX64 shw=%RX64 vs %RHp\n", j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
1029	}
1030	}
1031	}
1032	PGMPhysReleasePageMappingLock(pVM, &LockPage);
1033	}
1034
1035	/* Make sure this page table can't be written to from any shadow mapping. */
1036	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1037	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pPage->GCPhys, &HCPhysPT);
1038	AssertMsgRC(rc, ("PGMPhysGCPhys2HCPhys failed with rc=%d for %RGp\n", rc, pPage->GCPhys));
1039	if (rc == VINF_SUCCESS)
1040	{
1041	for (unsigned j = 0; j < pPool->cCurPages; j++)
1042	{
1043	PPGMPOOLPAGE pTempPage = &pPool->aPages[j];
1044
1045	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1046	{
1047	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pTempPage);
1048
1049	for (unsigned k = 0; k < RT_ELEMENTS(pShwPT->a); k++)
1050	{
1051	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[k])
1052	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1053	&& !pPage->fDirty
1054	# endif
1055	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[k]) == HCPhysPT)
1056	{
1057	if (fFirstMsg)
1058	{
1059	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
1060	fFirstMsg = false;
1061	}
1062	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch: r/w: GCPhys=%RGp idx=%d shw %RX64 %RX64\n", pTempPage->GCPhys, k, PGMSHWPTEPAE_GET_LOG(pShwPT->a[k]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[k]));
1063	}
1064	}
1065	}
1066	}
1067	}
1068	}
1069	}
1070	return VINF_SUCCESS;
1071	}
1072	#endif /* VBOX_WITH_DEBUGGER */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMPool.cpp@ 33462

Download in other formats: