MMPagePool.cpp@ 6298

Last change on this file since 6298 was 6298, checked in by vboxsync, 17 years ago
When using VMSet.*Error() don't put %Vrc into the error message if the same code is passed to the function. Otherwise the error code is displayed twice in the popping up error box -- looks ugly.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 17.8 KB

Line
1	/* $Id: MMPagePool.cpp 6298 2008-01-09 16:21:32Z vboxsync $ */
2	/** @file
3	* MM - Memory Monitor(/Manager) - Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 innotek GmbH
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_MM_POOL
22	#include <VBox/mm.h>
23	#include <VBox/pgm.h>
24	#include <VBox/stam.h>
25	#include "MMInternal.h"
26	#include <VBox/vm.h>
27	#include <VBox/param.h>
28	#include <VBox/err.h>
29	#include <VBox/log.h>
30	#include <iprt/alloc.h>
31	#include <iprt/assert.h>
32	#define USE_INLINE_ASM_BIT_OPS
33	#ifdef USE_INLINE_ASM_BIT_OPS
34	# include <iprt/asm.h>
35	#endif
36	#include <iprt/string.h>
37
38
39
40	/*******************************************************************************
41	* Internal Functions *
42	*******************************************************************************/
43	#ifdef IN_RING3
44	static void * mmr3PagePoolAlloc(PMMPAGEPOOL pPool);
45	static void mmr3PagePoolFree(PMMPAGEPOOL pPool, void *pv);
46	#endif
47
48
49	/**
50	* Initializes the page pool
51	*
52	* @return VBox status.
53	* @param pVM VM handle.
54	* @thread The Emulation Thread.
55	*/
56	int mmr3PagePoolInit(PVM pVM)
57	{
58	AssertMsg(!pVM->mm.s.pPagePool, ("Already initialized!\n"));
59
60	/*
61	* Allocate the pool structures.
62	*/
63	AssertRelease(sizeof(pVM->mm.s.pPagePool) + sizeof(pVM->mm.s.pPagePoolLow) < PAGE_SIZE);
64	int rc = SUPPageAllocLocked(1, (void **)&pVM->mm.s.pPagePool);
65	if (VBOX_FAILURE(rc))
66	return rc;
67	memset(pVM->mm.s.pPagePool, 0, PAGE_SIZE);
68	pVM->mm.s.pPagePool->pVM = pVM;
69	STAM_REG(pVM, &pVM->mm.s.pPagePool->cPages, STAMTYPE_U32, "/MM/Page/Def/cPages", STAMUNIT_PAGES, "Number of pages in the default pool.");
70	STAM_REG(pVM, &pVM->mm.s.pPagePool->cFreePages, STAMTYPE_U32, "/MM/Page/Def/cFreePages", STAMUNIT_PAGES, "Number of free pages in the default pool.");
71	STAM_REG(pVM, &pVM->mm.s.pPagePool->cSubPools, STAMTYPE_U32, "/MM/Page/Def/cSubPools", STAMUNIT_COUNT, "Number of sub pools in the default pool.");
72	STAM_REG(pVM, &pVM->mm.s.pPagePool->cAllocCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cAllocCalls", STAMUNIT_CALLS, "Number of MMR3PageAlloc() calls for the default pool.");
73	STAM_REG(pVM, &pVM->mm.s.pPagePool->cFreeCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cFreeCalls", STAMUNIT_CALLS, "Number of MMR3PageFree()+MMR3PageFreeByPhys() calls for the default pool.");
74	STAM_REG(pVM, &pVM->mm.s.pPagePool->cToPhysCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cToPhysCalls", STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for this pool.");
75	STAM_REG(pVM, &pVM->mm.s.pPagePool->cToVirtCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cToVirtCalls", STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the default pool.");
76	STAM_REG(pVM, &pVM->mm.s.pPagePool->cErrors, STAMTYPE_COUNTER, "/MM/Page/Def/cErrors", STAMUNIT_ERRORS,"Number of errors for the default pool.");
77
78	pVM->mm.s.pPagePoolLow = pVM->mm.s.pPagePool + 1;
79	pVM->mm.s.pPagePoolLow->pVM = pVM;
80	pVM->mm.s.pPagePoolLow->fLow = true;
81	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cPages, STAMTYPE_U32, "/MM/Page/Low/cPages", STAMUNIT_PAGES, "Number of pages in the <4GB pool.");
82	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cFreePages, STAMTYPE_U32, "/MM/Page/Low/cFreePages", STAMUNIT_PAGES, "Number of free pages in the <4GB pool.");
83	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cSubPools, STAMTYPE_U32, "/MM/Page/Low/cSubPools", STAMUNIT_COUNT, "Number of sub pools in the <4GB pool.");
84	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cAllocCalls, STAMTYPE_COUNTER, "/MM/Page/Low/cAllocCalls", STAMUNIT_CALLS, "Number of MMR3PageAllocLow() calls for the <4GB pool.");
85	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cFreeCalls, STAMTYPE_COUNTER, "/MM/Page/Low/cFreeCalls", STAMUNIT_CALLS, "Number of MMR3PageFreeLow()+MMR3PageFreeByPhys() calls for the <4GB pool.");
86	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cToPhysCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToPhysCalls", STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for the <4GB pool.");
87	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cToVirtCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToVirtCalls", STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the <4GB pool.");
88	STAM_REG(pVM, &pVM->mm.s.pPagePoolLow->cErrors, STAMTYPE_COUNTER, "/MM/Page/Low/cErrors", STAMUNIT_ERRORS,"Number of errors for the <4GB pool.");
89
90	/** @todo init a mutex? */
91	return VINF_SUCCESS;
92	}
93
94
95	/**
96	* Release all locks and free the allocated memory.
97	*
98	* @param pVM VM handle.
99	* @thread The Emulation Thread.
100	*/
101	void mmr3PagePoolTerm(PVM pVM)
102	{
103	if (pVM->mm.s.pPagePool)
104	{
105	/*
106	* Unlock all memory held by subpools and free the memory.
107	* (The MM Heap will free the memory used for internal stuff.)
108	*/
109	Assert(!pVM->mm.s.pPagePool->fLow);
110	PMMPAGESUBPOOL pSubPool = pVM->mm.s.pPagePool->pHead;
111	while (pSubPool)
112	{
113	int rc = SUPPageUnlock(pSubPool->pvPages);
114	AssertMsgRC(rc, ("SUPPageUnlock(%p) failed with rc=%d\n", pSubPool->pvPages, rc));
115	rc = SUPPageFree(pSubPool->pvPages, pSubPool->cPages);
116	AssertMsgRC(rc, ("SUPPageFree(%p) failed with rc=%d\n", pSubPool->pvPages, rc));
117	pSubPool->pvPages = NULL;
118
119	/* next */
120	pSubPool = pSubPool->pNext;
121	}
122	pVM->mm.s.pPagePool = NULL;
123	}
124
125	if (pVM->mm.s.pPagePoolLow)
126	{
127	/*
128	* Free the memory.
129	*/
130	Assert(pVM->mm.s.pPagePoolLow->fLow);
131	PMMPAGESUBPOOL pSubPool = pVM->mm.s.pPagePoolLow->pHead;
132	while (pSubPool)
133	{
134	int rc = SUPLowFree(pSubPool->pvPages, pSubPool->cPages);
135	AssertMsgRC(rc, ("SUPPageFree(%p) failed with rc=%d\n", pSubPool->pvPages, rc));
136	pSubPool->pvPages = NULL;
137
138	/* next */
139	pSubPool = pSubPool->pNext;
140	}
141	pVM->mm.s.pPagePoolLow = NULL;
142	}
143	}
144
145
146	/**
147	* Allocates a page from the page pool.
148	*
149	* @returns Pointer to allocated page(s).
150	* @returns NULL on failure.
151	* @param pPool Pointer to the page pool.
152	* @thread The Emulation Thread.
153	*/
154	DECLINLINE(void *) mmr3PagePoolAlloc(PMMPAGEPOOL pPool)
155	{
156	VM_ASSERT_EMT(pPool->pVM);
157	STAM_COUNTER_INC(&pPool->cAllocCalls);
158
159	/*
160	* Walk free list.
161	*/
162	if (pPool->pHeadFree)
163	{
164	PMMPAGESUBPOOL pSub = pPool->pHeadFree;
165	/* decrement free count and unlink if no more free entries. */
166	if (!--pSub->cPagesFree)
167	pPool->pHeadFree = pSub->pNextFree;
168	#ifdef VBOX_WITH_STATISTICS
169	pPool->cFreePages--;
170	#endif
171
172	/* find free spot in bitmap. */
173	#ifdef USE_INLINE_ASM_BIT_OPS
174	const int iPage = ASMBitFirstClear(pSub->auBitmap, pSub->cPages);
175	if (iPage >= 0)
176	{
177	Assert(!ASMBitTest(pSub->auBitmap, iPage));
178	ASMBitSet(pSub->auBitmap, iPage);
179	return (char )pSub->pvPages + PAGE_SIZE iPage;
180	}
181	#else
182	unsigned *pu = &pSub->auBitmap[0];
183	unsigned puEnd = &pSub->auBitmap[pSub->cPages / (sizeof(pSub->auBitmap) 8)];
184	while (pu < puEnd)
185	{
186	unsigned u;
187	if ((u = *pu) != ~0U)
188	{
189	unsigned iBit = 0;
190	unsigned uMask = 1;
191	while (iBit < sizeof(pSub->auBitmap[0]) * 8)
192	{
193	if (!(u & uMask))
194	{
195	*pu \|= uMask;
196	return (char *)pSub->pvPages
197	+ PAGE_SIZE * (iBit + ((char )pu - (char )&pSub->auBitmap[0]) * 8);
198	}
199	iBit++;
200	uMask <<= 1;
201	}
202	STAM_COUNTER_INC(&pPool->cErrors);
203	AssertMsgFailed(("how odd, expected to find a free bit in %#x, but didn't\n", u));
204	}
205	/* next */
206	pu++;
207	}
208	#endif
209	STAM_COUNTER_INC(&pPool->cErrors);
210	#ifdef VBOX_WITH_STATISTICS
211	pPool->cFreePages++;
212	#endif
213	AssertMsgFailed(("how strange, expected to find a free bit in %p, but didn't (%d pages supposed to be free!)\n", pSub, pSub->cPagesFree + 1));
214	}
215
216	/*
217	* Allocate new subpool.
218	*/
219	unsigned cPages = !pPool->fLow ? 128 : 32;
220	PMMPAGESUBPOOL pSub;
221	int rc = MMHyperAlloc(pPool->pVM,
222	RT_OFFSETOF(MMPAGESUBPOOL, auBitmap[cPages / (sizeof(pSub->auBitmap[0] * 8))])
223	+ (sizeof(SUPPAGE) + sizeof(MMPPLOOKUPHCPHYS)) * cPages
224	+ sizeof(MMPPLOOKUPHCPTR),
225	0,
226	MM_TAG_MM_PAGE,
227	(void **)&pSub);
228	if (VBOX_FAILURE(rc))
229	return NULL;
230
231	PSUPPAGE paPhysPages = (PSUPPAGE)&pSub->auBitmap[cPages / (sizeof(pSub->auBitmap[0]) * 8)];
232	Assert((uintptr_t)paPhysPages >= (uintptr_t)&pSub->auBitmap[1]);
233	if (!pPool->fLow)
234	{
235	/*
236	* Allocate and lock the pages.
237	*/
238	rc = SUPPageAlloc(cPages, &pSub->pvPages);
239	if (VBOX_SUCCESS(rc))
240	{
241	rc = SUPPageLock(pSub->pvPages, cPages, paPhysPages);
242	if (VBOX_FAILURE(rc))
243	{
244	SUPPageFree(pSub->pvPages, cPages);
245	rc = VMSetError(pPool->pVM, rc, RT_SRC_POS,
246	N_("Failed to lock host %zd bytes of memory (out of memory)"), (size_t)cPages << PAGE_SHIFT);
247	}
248	}
249	}
250	else
251	rc = SUPLowAlloc(cPages, &pSub->pvPages, NULL, paPhysPages);
252	if (VBOX_SUCCESS(rc))
253	{
254	/*
255	* Setup the sub structure and allocate the requested page.
256	*/
257	pSub->cPages = cPages;
258	pSub->cPagesFree= cPages - 1;
259	pSub->paPhysPages = paPhysPages;
260	memset(pSub->auBitmap, 0, cPages / 8);
261	/* allocate first page. */
262	pSub->auBitmap[0] \|= 1;
263	/* link into free chain. */
264	pSub->pNextFree = pPool->pHeadFree;
265	pPool->pHeadFree= pSub;
266	/* link into main chain. */
267	pSub->pNext = pPool->pHead;
268	pPool->pHead = pSub;
269	/* update pool statistics. */
270	pPool->cSubPools++;
271	pPool->cPages += cPages;
272	#ifdef VBOX_WITH_STATISTICS
273	pPool->cFreePages += cPages - 1;
274	#endif
275
276	/*
277	* Initialize the physical pages with backpointer to subpool.
278	*/
279	unsigned i = cPages;
280	while (i-- > 0)
281	{
282	AssertMsg(paPhysPages[i].Phys && !(paPhysPages[i].Phys & PAGE_OFFSET_MASK),
283	("i=%d Phys=%d\n", i, paPhysPages[i].Phys));
284	paPhysPages[i].uReserved = (RTHCUINTPTR)pSub;
285	}
286
287	/*
288	* Initialize the physical lookup record with backpointers to the physical pages.
289	*/
290	PMMPPLOOKUPHCPHYS paLookupPhys = (PMMPPLOOKUPHCPHYS)&paPhysPages[cPages];
291	i = cPages;
292	while (i-- > 0)
293	{
294	paLookupPhys[i].pPhysPage = &paPhysPages[i];
295	paLookupPhys[i].Core.Key = paPhysPages[i].Phys;
296	RTAvlHCPhysInsert(&pPool->pLookupPhys, &paLookupPhys[i].Core);
297	}
298
299	/*
300	* And the one record for virtual memory lookup.
301	*/
302	PMMPPLOOKUPHCPTR pLookupVirt = (PMMPPLOOKUPHCPTR)&paLookupPhys[cPages];
303	pLookupVirt->pSubPool = pSub;
304	pLookupVirt->Core.Key = pSub->pvPages;
305	RTAvlPVInsert(&pPool->pLookupVirt, &pLookupVirt->Core);
306
307	/* return allocated page (first). */
308	return pSub->pvPages;
309	}
310
311	MMR3HeapFree(pSub);
312	STAM_COUNTER_INC(&pPool->cErrors);
313	if (pPool->fLow)
314	VMSetError(pPool->pVM, rc, RT_SRC_POS,
315	N_("Failed to expand page pool for memory below 4GB. current size: %d pages"),
316	pPool->cPages);
317	AssertMsgFailed(("Failed to expand pool%s. rc=%Vrc poolsize=%d\n",
318	pPool->fLow ? " (<4GB)" : "", rc, pPool->cPages));
319	return NULL;
320	}
321
322
323	/**
324	* Frees a page from the page pool.
325	*
326	* @param pPool Pointer to the page pool.
327	* @param pv Pointer to the page to free.
328	* I.e. pointer returned by mmr3PagePoolAlloc().
329	* @thread The Emulation Thread.
330	*/
331	DECLINLINE(void) mmr3PagePoolFree(PMMPAGEPOOL pPool, void *pv)
332	{
333	VM_ASSERT_EMT(pPool->pVM);
334	STAM_COUNTER_INC(&pPool->cFreeCalls);
335
336	/*
337	* Lookup the virtual address.
338	*/
339	PMMPPLOOKUPHCPTR pLookup = (PMMPPLOOKUPHCPTR)RTAvlPVGetBestFit(&pPool->pLookupVirt, pv, false);
340	if ( !pLookup
341	\|\| (char )pv >= (char )pLookup->pSubPool->pvPages + (pLookup->pSubPool->cPages << PAGE_SHIFT)
342	)
343	{
344	STAM_COUNTER_INC(&pPool->cErrors);
345	AssertMsgFailed(("invalid pointer %p\n", pv));
346	return;
347	}
348
349	/*
350	* Free the page.
351	*/
352	PMMPAGESUBPOOL pSubPool = pLookup->pSubPool;
353	/* clear bitmap bit */
354	const unsigned iPage = ((char )pv - (char )pSubPool->pvPages) >> PAGE_SHIFT;
355	#ifdef USE_INLINE_ASM_BIT_OPS
356	Assert(ASMBitTest(pSubPool->auBitmap, iPage));
357	ASMBitClear(pSubPool->auBitmap, iPage);
358	#else
359	unsigned iBit = iPage % (sizeof(pSubPool->auBitmap[0]) * 8);
360	unsigned iIndex = iPage / (sizeof(pSubPool->auBitmap[0]) * 8);
361	pSubPool->auBitmap[iIndex] &= ~(1 << iBit);
362	#endif
363	/* update stats. */
364	pSubPool->cPagesFree++;
365	#ifdef VBOX_WITH_STATISTICS
366	pPool->cFreePages++;
367	#endif
368	if (pSubPool->cPagesFree == 1)
369	{
370	pSubPool->pNextFree = pPool->pHeadFree;
371	pPool->pHeadFree = pSubPool;
372	}
373	}
374
375
376	/**
377	* Allocates a page from the page pool.
378	*
379	* This function may returns pages which has physical addresses any
380	* where. If you require a page to be within the first 4GB of physical
381	* memory, use MMR3PageAllocLow().
382	*
383	* @returns Pointer to the allocated page page.
384	* @returns NULL on failure.
385	* @param pVM VM handle.
386	* @thread The Emulation Thread.
387	*/
388	MMR3DECL(void *) MMR3PageAlloc(PVM pVM)
389	{
390	return mmr3PagePoolAlloc(pVM->mm.s.pPagePool);
391	}
392
393
394	/**
395	* Allocates a page from the page pool and return its physical address.
396	*
397	* This function may returns pages which has physical addresses any
398	* where. If you require a page to be within the first 4GB of physical
399	* memory, use MMR3PageAllocLow().
400	*
401	* @returns Pointer to the allocated page page.
402	* @returns NIL_RTHCPHYS on failure.
403	* @param pVM VM handle.
404	* @thread The Emulation Thread.
405	*/
406	MMR3DECL(RTHCPHYS) MMR3PageAllocPhys(PVM pVM)
407	{
408	/** @todo optimize this, it's the most common case now. */
409	void *pv = mmr3PagePoolAlloc(pVM->mm.s.pPagePool);
410	if (pv)
411	return mmPagePoolPtr2Phys(pVM->mm.s.pPagePool, pv);
412	return NIL_RTHCPHYS;
413	}
414
415
416	/**
417	* Frees a page allocated from the page pool by MMR3PageAlloc() or
418	* MMR3PageAllocPhys().
419	*
420	* @param pVM VM handle.
421	* @param pvPage Pointer to the page.
422	* @thread The Emulation Thread.
423	*/
424	MMR3DECL(void) MMR3PageFree(PVM pVM, void *pvPage)
425	{
426	mmr3PagePoolFree(pVM->mm.s.pPagePool, pvPage);
427	}
428
429
430	/**
431	* Allocates a page from the low page pool.
432	*
433	* @returns Pointer to the allocated page.
434	* @returns NULL on failure.
435	* @param pVM VM handle.
436	* @thread The Emulation Thread.
437	*/
438	MMR3DECL(void *) MMR3PageAllocLow(PVM pVM)
439	{
440	return mmr3PagePoolAlloc(pVM->mm.s.pPagePoolLow);
441	}
442
443
444	/**
445	* Frees a page allocated from the page pool by MMR3PageAllocLow().
446	*
447	* @param pVM VM handle.
448	* @param pvPage Pointer to the page.
449	* @thread The Emulation Thread.
450	*/
451	MMR3DECL(void) MMR3PageFreeLow(PVM pVM, void *pvPage)
452	{
453	mmr3PagePoolFree(pVM->mm.s.pPagePoolLow, pvPage);
454	}
455
456
457	/**
458	* Free a page allocated from the page pool by physical address.
459	* This works for pages allocated by MMR3PageAlloc(), MMR3PageAllocPhys()
460	* and MMR3PageAllocLow().
461	*
462	* @param pVM VM handle.
463	* @param HCPhysPage The physical address of the page to be freed.
464	* @thread The Emulation Thread.
465	*/
466	MMR3DECL(void) MMR3PageFreeByPhys(PVM pVM, RTHCPHYS HCPhysPage)
467	{
468	void *pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePool, HCPhysPage);
469	if (!pvPage)
470	pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePoolLow, HCPhysPage);
471	if (pvPage)
472	mmr3PagePoolFree(pVM->mm.s.pPagePool, pvPage);
473	else
474	AssertMsgFailed(("Invalid address HCPhysPT=%#x\n", HCPhysPage));
475	}
476
477
478	/**
479	* Gets the HC pointer to the dummy page.
480	*
481	* The dummy page is used as a place holder to prevent potential bugs
482	* from doing really bad things to the system.
483	*
484	* @returns Pointer to the dummy page.
485	* @param pVM VM handle.
486	* @thread The Emulation Thread.
487	*/
488	MMR3DECL(void *) MMR3PageDummyHCPtr(PVM pVM)
489	{
490	VM_ASSERT_EMT(pVM);
491	if (!pVM->mm.s.pvDummyPage)
492	{
493	pVM->mm.s.pvDummyPage = mmr3PagePoolAlloc(pVM->mm.s.pPagePool);
494	AssertRelease(pVM->mm.s.pvDummyPage);
495	pVM->mm.s.HCPhysDummyPage = mmPagePoolPtr2Phys(pVM->mm.s.pPagePool, pVM->mm.s.pvDummyPage);
496	}
497	return pVM->mm.s.pvDummyPage;
498	}
499
500
501	/**
502	* Gets the HC Phys to the dummy page.
503	*
504	* The dummy page is used as a place holder to prevent potential bugs
505	* from doing really bad things to the system.
506	*
507	* @returns Pointer to the dummy page.
508	* @param pVM VM handle.
509	* @thread The Emulation Thread.
510	*/
511	MMR3DECL(RTHCPHYS) MMR3PageDummyHCPhys(PVM pVM)
512	{
513	VM_ASSERT_EMT(pVM);
514	if (!pVM->mm.s.pvDummyPage)
515	MMR3PageDummyHCPtr(pVM);
516	return pVM->mm.s.HCPhysDummyPage;
517	}
518

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/MMPagePool.cpp@ 6298

Download in other formats: