MMAllHyper.cpp@ 44577

Last change on this file since 44577 was 44528, checked in by vboxsync, 12 years ago
header (C) fixes
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 47.1 KB

Line
1	/* $Id: MMAllHyper.cpp 44528 2013-02-04 14:27:54Z vboxsync $ */
2	/** @file
3	* MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
23	#include <VBox/vmm/mm.h>
24	#include <VBox/vmm/stam.h>
25	#include "MMInternal.h"
26	#include <VBox/vmm/vm.h>
27
28	#include <VBox/err.h>
29	#include <VBox/param.h>
30	#include <iprt/assert.h>
31	#include <VBox/log.h>
32	#include <iprt/asm.h>
33	#include <iprt/string.h>
34
35
36	/*******************************************************************************
37	* Defined Constants And Macros *
38	*******************************************************************************/
39	#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
40	#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
41	#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
42	#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
43
44	#define ASSERT_OFFPREV(pHeap, pChunk) \
45	do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
46	Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
47	AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
48	\|\| (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
49	("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
50	} while (0)
51
52	#define ASSERT_OFFNEXT(pHeap, pChunk) \
53	do { ASSERT_ALIGN((pChunk)->offNext); \
54	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
55	} while (0)
56
57	#define ASSERT_OFFHEAP(pHeap, pChunk) \
58	do { Assert((pChunk)->offHeap); \
59	AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
60	("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
61	Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
62	} while (0)
63
64	#ifdef VBOX_WITH_STATISTICS
65	#define ASSERT_OFFSTAT(pHeap, pChunk) \
66	do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
67	Assert(!(pChunk)->offStat); \
68	else if ((pChunk)->offStat) \
69	{ \
70	Assert((pChunk)->offStat); \
71	AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
72	uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
73	AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
74	("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
75	} \
76	} while (0)
77	#else
78	#define ASSERT_OFFSTAT(pHeap, pChunk) \
79	do { Assert(!(pChunk)->offStat); \
80	} while (0)
81	#endif
82
83	#define ASSERT_CHUNK(pHeap, pChunk) \
84	do { ASSERT_OFFNEXT(pHeap, pChunk); \
85	ASSERT_OFFPREV(pHeap, pChunk); \
86	ASSERT_OFFHEAP(pHeap, pChunk); \
87	ASSERT_OFFSTAT(pHeap, pChunk); \
88	} while (0)
89	#define ASSERT_CHUNK_USED(pHeap, pChunk) \
90	do { ASSERT_OFFNEXT(pHeap, pChunk); \
91	ASSERT_OFFPREV(pHeap, pChunk); \
92	Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
93	} while (0)
94
95	#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
96	do { ASSERT_ALIGN((pChunk)->offPrev); \
97	ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
98	Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) \|\| !(pChunk)->offPrev); \
99	AssertMsg( (pChunk)->offPrev \
100	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
101	("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
102	(pHeap)->offFreeHead)); \
103	} while (0)
104
105	#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
106	do { ASSERT_ALIGN((pChunk)->offNext); \
107	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
108	Assert((pChunk)->offNext != (pChunk)->core.offNext \|\| !(pChunk)->offNext); \
109	AssertMsg( (pChunk)->offNext \
110	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
111	("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
112	(pHeap)->offFreeTail)); \
113	} while (0)
114
115	#define ASSERT_FREE_CB(pHeap, pChunk) \
116	do { ASSERT_ALIGN((pChunk)->cb); \
117	Assert((pChunk)->cb > 0); \
118	if ((pChunk)->core.offNext) \
119	AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
120	("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
121	else \
122	ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
123	} while (0)
124
125	#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
126	do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
127	Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
128	ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
129	ASSERT_FREE_OFFPREV(pHeap, pChunk); \
130	ASSERT_FREE_CB(pHeap, pChunk); \
131	} while (0)
132
133
134	/*******************************************************************************
135	* Internal Functions *
136	*******************************************************************************/
137	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
138	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
139	#ifdef VBOX_WITH_STATISTICS
140	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
141	#ifdef IN_RING3
142	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
143	#endif
144	#endif
145	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
146	#ifdef MMHYPER_HEAP_STRICT
147	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
148	#endif
149
150
151
152	/**
153	* Locks the hypervisor heap.
154	* This might call back to Ring-3 in order to deal with lock contention in GC and R3.
155	*
156	* @param pVM Pointer to the VM.
157	*/
158	static int mmHyperLock(PVM pVM)
159	{
160	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
161
162	#ifdef IN_RING3
163	if (!PDMCritSectIsInitialized(&pHeap->Lock))
164	return VINF_SUCCESS; /* early init */
165	#else
166	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
167	#endif
168	int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
169	#if defined(IN_RC) \|\| defined(IN_RING0)
170	if (rc == VERR_SEM_BUSY)
171	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
172	#endif
173	AssertRC(rc);
174	return rc;
175	}
176
177
178	/**
179	* Unlocks the hypervisor heap.
180	*
181	* @param pVM Pointer to the VM.
182	*/
183	static void mmHyperUnlock(PVM pVM)
184	{
185	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
186
187	#ifdef IN_RING3
188	if (!PDMCritSectIsInitialized(&pHeap->Lock))
189	return; /* early init */
190	#endif
191	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
192	PDMCritSectLeave(&pHeap->Lock);
193	}
194
195	/**
196	* Allocates memory in the Hypervisor (RC VMM) area.
197	* The returned memory is of course zeroed.
198	*
199	* @returns VBox status code.
200	* @param pVM Pointer to the VM.
201	* @param cb Number of bytes to allocate.
202	* @param uAlignment Required memory alignment in bytes.
203	* Values are 0,8,16,32,64 and PAGE_SIZE.
204	* 0 -> default alignment, i.e. 8 bytes.
205	* @param enmTag The statistics tag.
206	* @param ppv Where to store the address to the allocated
207	* memory.
208	*/
209	static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
210	{
211	AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
212
213	/*
214	* Validate input and adjust it to reasonable values.
215	*/
216	if (!uAlignment \|\| uAlignment < MMHYPER_HEAP_ALIGN_MIN)
217	uAlignment = MMHYPER_HEAP_ALIGN_MIN;
218	uint32_t cbAligned;
219	switch (uAlignment)
220	{
221	case 8:
222	case 16:
223	case 32:
224	case 64:
225	cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
226	if (!cbAligned \|\| cbAligned < cb)
227	{
228	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
229	AssertMsgFailed(("Nice try.\n"));
230	return VERR_INVALID_PARAMETER;
231	}
232	break;
233
234	case PAGE_SIZE:
235	AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
236	cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
237	if (!cbAligned)
238	{
239	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
240	AssertMsgFailed(("Nice try.\n"));
241	return VERR_INVALID_PARAMETER;
242	}
243	break;
244
245	default:
246	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
247	AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
248	return VERR_INVALID_PARAMETER;
249	}
250
251
252	/*
253	* Get heap and statisticsStatistics.
254	*/
255	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
256	#ifdef VBOX_WITH_STATISTICS
257	PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
258	if (!pStat)
259	{
260	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
261	AssertMsgFailed(("Failed to allocate statistics!\n"));
262	return VERR_MM_HYPER_NO_MEMORY;
263	}
264	#endif
265	if (uAlignment < PAGE_SIZE)
266	{
267	/*
268	* Allocate a chunk.
269	*/
270	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
271	if (pChunk)
272	{
273	#ifdef VBOX_WITH_STATISTICS
274	const uint32_t cbChunk = pChunk->offNext
275	? pChunk->offNext
276	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
277	pStat->cbAllocated += (uint32_t)cbChunk;
278	pStat->cbCurAllocated += (uint32_t)cbChunk;
279	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
280	pStat->cbMaxAllocated = pStat->cbCurAllocated;
281	pStat->cAllocations++;
282	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
283	#else
284	pChunk->offStat = 0;
285	#endif
286	void *pv = pChunk + 1;
287	*ppv = pv;
288	ASMMemZero32(pv, cbAligned);
289	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
290	return VINF_SUCCESS;
291	}
292	}
293	else
294	{
295	/*
296	* Allocate page aligned memory.
297	*/
298	void *pv = mmHyperAllocPages(pHeap, cbAligned);
299	if (pv)
300	{
301	#ifdef VBOX_WITH_STATISTICS
302	pStat->cbAllocated += cbAligned;
303	pStat->cbCurAllocated += cbAligned;
304	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
305	pStat->cbMaxAllocated = pStat->cbCurAllocated;
306	pStat->cAllocations++;
307	#endif
308	*ppv = pv;
309	/* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
310	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
311	return VINF_SUCCESS;
312	}
313	}
314
315	#ifdef VBOX_WITH_STATISTICS
316	pStat->cAllocations++;
317	pStat->cFailures++;
318	#endif
319	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
320	AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
321	return VERR_MM_HYPER_NO_MEMORY;
322	}
323
324	/**
325	* Wrapper for mmHyperAllocInternal
326	*/
327	VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
328	{
329	int rc;
330
331	rc = mmHyperLock(pVM);
332	AssertRCReturn(rc, rc);
333
334	LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
335
336	rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
337
338	mmHyperUnlock(pVM);
339	return rc;
340	}
341
342	/**
343	* Allocates a chunk of memory from the specified heap.
344	* The caller validates the parameters of this request.
345	*
346	* @returns Pointer to the allocated chunk.
347	* @returns NULL on failure.
348	* @param pHeap The heap.
349	* @param cb Size of the memory block to allocate.
350	* @param uAlignment The alignment specifications for the allocated block.
351	* @internal
352	*/
353	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
354	{
355	Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
356	#ifdef MMHYPER_HEAP_STRICT
357	mmHyperHeapCheck(pHeap);
358	#endif
359	#ifdef MMHYPER_HEAP_STRICT_FENCE
360	uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
361	cb += cbFence;
362	#endif
363
364	/*
365	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
366	*/
367	if (pHeap->offFreeHead == NIL_OFFSET)
368	return NULL;
369
370	/*
371	* Small alignments - from the front of the heap.
372	*
373	* Must split off free chunks at the end to prevent messing up the
374	* last free node which we take the page aligned memory from the top of.
375	*/
376	PMMHYPERCHUNK pRet = NULL;
377	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
378	while (pFree)
379	{
380	ASSERT_CHUNK_FREE(pHeap, pFree);
381	if (pFree->cb >= cb)
382	{
383	unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
384	if (offAlign)
385	offAlign = uAlignment - offAlign;
386	if (!offAlign \|\| pFree->cb - offAlign >= cb)
387	{
388	Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
389
390	/*
391	* Adjust the node in front.
392	* Because of multiple alignments we need to special case allocation of the first block.
393	*/
394	if (offAlign)
395	{
396	MMHYPERCHUNKFREE Free = *pFree;
397	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
398	{
399	/* just add a bit of memory to it. */
400	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
401	pPrev->core.offNext += offAlign;
402	AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
403	Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
404	}
405	else
406	{
407	/* make new head node, mark it USED for simplicity. */
408	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
409	Assert(pPrev == &pFree->core);
410	pPrev->offPrev = 0;
411	MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
412	pPrev->offNext = offAlign;
413	Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
414
415	}
416	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
417	pHeap->cbFree -= offAlign;
418
419	/* Recreate pFree node and adjusting everything... */
420	pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
421	*pFree = Free;
422
423	pFree->cb -= offAlign;
424	if (pFree->core.offNext)
425	{
426	pFree->core.offNext -= offAlign;
427	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
428	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
429	ASSERT_CHUNK(pHeap, pNext);
430	}
431	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
432	MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
433
434	if (pFree->offNext)
435	{
436	pFree->offNext -= offAlign;
437	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
438	pNext->offPrev = -(int32_t)pFree->offNext;
439	ASSERT_CHUNK_FREE(pHeap, pNext);
440	}
441	else
442	pHeap->offFreeTail += offAlign;
443	if (pFree->offPrev)
444	{
445	pFree->offPrev -= offAlign;
446	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
447	pPrev->offNext = -pFree->offPrev;
448	ASSERT_CHUNK_FREE(pHeap, pPrev);
449	}
450	else
451	pHeap->offFreeHead += offAlign;
452	pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
453	pFree->core.offStat = 0;
454	ASSERT_CHUNK_FREE(pHeap, pFree);
455	Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
456	}
457
458	/*
459	* Split off a new FREE chunk?
460	*/
461	if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
462	{
463	/*
464	* Move the FREE chunk up to make room for the new USED chunk.
465	*/
466	const int off = cb + sizeof(MMHYPERCHUNK);
467	PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
468	pNew = pFree;
469	pNew->cb -= off;
470	if (pNew->core.offNext)
471	{
472	pNew->core.offNext -= off;
473	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
474	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
475	ASSERT_CHUNK(pHeap, pNext);
476	}
477	pNew->core.offPrev = -off;
478	MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
479
480	if (pNew->offNext)
481	{
482	pNew->offNext -= off;
483	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
484	pNext->offPrev = -(int32_t)pNew->offNext;
485	ASSERT_CHUNK_FREE(pHeap, pNext);
486	}
487	else
488	pHeap->offFreeTail += off;
489	if (pNew->offPrev)
490	{
491	pNew->offPrev -= off;
492	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
493	pPrev->offNext = -pNew->offPrev;
494	ASSERT_CHUNK_FREE(pHeap, pPrev);
495	}
496	else
497	pHeap->offFreeHead += off;
498	pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
499	pNew->core.offStat = 0;
500	ASSERT_CHUNK_FREE(pHeap, pNew);
501
502	/*
503	* Update the old FREE node making it a USED node.
504	*/
505	pFree->core.offNext = off;
506	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
507
508
509	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
510	pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
511	pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
512	pRet = &pFree->core;
513	ASSERT_CHUNK(pHeap, &pFree->core);
514	Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
515	}
516	else
517	{
518	/*
519	* Link out of free list.
520	*/
521	if (pFree->offNext)
522	{
523	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
524	if (pFree->offPrev)
525	{
526	pNext->offPrev += pFree->offPrev;
527	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
528	pPrev->offNext += pFree->offNext;
529	ASSERT_CHUNK_FREE(pHeap, pPrev);
530	}
531	else
532	{
533	pHeap->offFreeHead += pFree->offNext;
534	pNext->offPrev = 0;
535	}
536	ASSERT_CHUNK_FREE(pHeap, pNext);
537	}
538	else
539	{
540	if (pFree->offPrev)
541	{
542	pHeap->offFreeTail += pFree->offPrev;
543	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
544	pPrev->offNext = 0;
545	ASSERT_CHUNK_FREE(pHeap, pPrev);
546	}
547	else
548	{
549	pHeap->offFreeHead = NIL_OFFSET;
550	pHeap->offFreeTail = NIL_OFFSET;
551	}
552	}
553
554	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
555	pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
556	pHeap->cbFree -= pFree->cb;
557	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
558	pRet = &pFree->core;
559	ASSERT_CHUNK(pHeap, &pFree->core);
560	Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
561	}
562	Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
563	break;
564	}
565	}
566
567	/* next */
568	pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
569	}
570
571	#ifdef MMHYPER_HEAP_STRICT_FENCE
572	uint32_t pu32End = (uint32_t )((uint8_t *)(pRet + 1) + cb);
573	uint32_t *pu32EndReal = pRet->offNext
574	? (uint32_t )((uint8_t )pRet + pRet->offNext)
575	: (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
576	cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
577	ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
578	pu32EndReal[-1] = cbFence;
579	#endif
580	#ifdef MMHYPER_HEAP_STRICT
581	mmHyperHeapCheck(pHeap);
582	#endif
583	return pRet;
584	}
585
586
587	/**
588	* Allocates one or more pages of memory from the specified heap.
589	* The caller validates the parameters of this request.
590	*
591	* @returns Pointer to the allocated chunk.
592	* @returns NULL on failure.
593	* @param pHeap The heap.
594	* @param cb Size of the memory block to allocate.
595	* @internal
596	*/
597	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
598	{
599	Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
600
601	#ifdef MMHYPER_HEAP_STRICT
602	mmHyperHeapCheck(pHeap);
603	#endif
604
605	/*
606	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
607	*/
608	if (pHeap->offFreeHead == NIL_OFFSET)
609	return NULL;
610
611	/*
612	* Page aligned chunks.
613	*
614	* Page aligned chunks can only be allocated from the last FREE chunk.
615	* This is for reasons of simplicity and fragmentation. Page aligned memory
616	* must also be allocated in page aligned sizes. Page aligned memory cannot
617	* be freed either.
618	*
619	* So, for this to work, the last FREE chunk needs to end on a page aligned
620	* boundary.
621	*/
622	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
623	ASSERT_CHUNK_FREE(pHeap, pFree);
624	if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
625	\|\| pFree->cb + sizeof(MMHYPERCHUNK) < cb)
626	{
627	Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
628	return NULL;
629	}
630
631	void *pvRet;
632	if (pFree->cb > cb)
633	{
634	/*
635	* Simple, just cut the top of the free node and return it.
636	*/
637	pFree->cb -= cb;
638	pvRet = (char *)(&pFree->core + 1) + pFree->cb;
639	AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
640	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
641	pHeap->cbFree -= cb;
642	ASSERT_CHUNK_FREE(pHeap, pFree);
643	Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
644	}
645	else
646	{
647	/*
648	* Unlink the FREE node.
649	*/
650	pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
651	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
652	pHeap->cbFree -= pFree->cb;
653
654	/* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
655	if (pvRet != (void *)pFree)
656	{
657	AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the beep did someone manage to allocated up all the heap with page aligned memory?!?\n"));
658	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
659	pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
660	AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
661	#ifdef VBOX_WITH_STATISTICS
662	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
663	pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
664	pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
665	#endif
666	Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
667	}
668
669	/* unlink from FREE chain. */
670	if (pFree->offPrev)
671	{
672	pHeap->offFreeTail += pFree->offPrev;
673	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
674	}
675	else
676	{
677	pHeap->offFreeTail = NIL_OFFSET;
678	pHeap->offFreeHead = NIL_OFFSET;
679	}
680	Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
681	}
682	pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
683	Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
684
685	#ifdef MMHYPER_HEAP_STRICT
686	mmHyperHeapCheck(pHeap);
687	#endif
688	return pvRet;
689	}
690
691	#ifdef VBOX_WITH_STATISTICS
692
693	/**
694	* Get the statistic record for a tag.
695	*
696	* @returns Pointer to a stat record.
697	* @returns NULL on failure.
698	* @param pHeap The heap.
699	* @param enmTag The tag.
700	*/
701	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
702	{
703	/* try look it up first. */
704	PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
705	if (!pStat)
706	{
707	/* try allocate a new one */
708	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
709	if (!pChunk)
710	return NULL;
711	pStat = (PMMHYPERSTAT)(pChunk + 1);
712	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
713
714	ASMMemZero32(pStat, sizeof(*pStat));
715	pStat->Core.Key = enmTag;
716	RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
717	}
718	if (!pStat->fRegistered)
719	{
720	# ifdef IN_RING3
721	mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
722	# else
723	/** @todo schedule a R3 action. */
724	# endif
725	}
726	return pStat;
727	}
728
729
730	# ifdef IN_RING3
731	/**
732	* Registers statistics with STAM.
733	*
734	*/
735	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
736	{
737	if (pStat->fRegistered)
738	return;
739	const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
740	STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
741	STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
742	STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
743	STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
744	STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
745	STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
746	STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
747	pStat->fRegistered = true;
748	}
749	# endif /* IN_RING3 */
750
751	#endif /* VBOX_WITH_STATISTICS */
752
753
754	/**
755	* Free memory allocated using MMHyperAlloc().
756	* The caller validates the parameters of this request.
757	*
758	* @returns VBox status code.
759	* @param pVM Pointer to the VM.
760	* @param pv The memory to free.
761	* @remark Try avoid free hyper memory.
762	*/
763	static int mmHyperFreeInternal(PVM pVM, void *pv)
764	{
765	Log2(("MMHyperFree: pv=%p\n", pv));
766	if (!pv)
767	return VINF_SUCCESS;
768	AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
769	("Invalid pointer %p!\n", pv),
770	VERR_INVALID_POINTER);
771
772	/*
773	* Get the heap and stats.
774	* Validate the chunk at the same time.
775	*/
776	PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
777
778	AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
779	\|\| RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
780	("%p: offNext=%#RX32\n", pv, pChunk->offNext),
781	VERR_INVALID_POINTER);
782
783	AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
784	("%p: Not used!\n", pv),
785	VERR_INVALID_POINTER);
786
787	int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
788	AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
789	&& !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
790	("%p: offPrev=%#RX32!\n", pv, offPrev),
791	VERR_INVALID_POINTER);
792
793	/* statistics */
794	#ifdef VBOX_WITH_STATISTICS
795	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
796	AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
797	&& pChunk->offStat,
798	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
799	VERR_INVALID_POINTER);
800	#else
801	AssertMsgReturn(!pChunk->offStat,
802	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
803	VERR_INVALID_POINTER);
804	#endif
805
806	/* The heap structure. */
807	PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
808	AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
809	&& pChunk->offHeap,
810	("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
811	VERR_INVALID_POINTER);
812
813	AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
814	("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
815	VERR_INVALID_POINTER);
816	Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
817
818	/* Some more verifications using additional info from pHeap. */
819	AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
820	("%p: offPrev=%#RX32!\n", pv, offPrev),
821	VERR_INVALID_POINTER);
822
823	AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
824	("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
825	VERR_INVALID_POINTER);
826
827	AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
828	("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
829	(char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
830	VERR_INVALID_POINTER);
831
832	#ifdef MMHYPER_HEAP_STRICT
833	mmHyperHeapCheck(pHeap);
834	#endif
835
836	#if defined(VBOX_WITH_STATISTICS) \|\| defined(MMHYPER_HEAP_FREE_POISON)
837	/* calc block size. */
838	const uint32_t cbChunk = pChunk->offNext
839	? pChunk->offNext
840	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
841	#endif
842	#ifdef MMHYPER_HEAP_FREE_POISON
843	/* poison the block */
844	memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
845	#endif
846
847	#ifdef MMHYPER_HEAP_FREE_DELAY
848	# ifdef MMHYPER_HEAP_FREE_POISON
849	/*
850	* Check poison.
851	*/
852	unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
853	while (i-- > 0)
854	if (pHeap->aDelayedFrees[i].offChunk)
855	{
856	PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
857	const size_t cb = pCur->offNext
858	? pCur->offNext - sizeof(*pCur)
859	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t )pCur - sizeof(pCur);
860	uint8_t pab = (uint8_t )(pCur + 1);
861	for (unsigned off = 0; off < cb; off++)
862	AssertReleaseMsg(pab[off] == 0xCB,
863	("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
864	pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
865	}
866	# endif /* MMHYPER_HEAP_FREE_POISON */
867
868	/*
869	* Delayed freeing.
870	*/
871	int rc = VINF_SUCCESS;
872	if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
873	{
874	PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
875	rc = mmHyperFree(pHeap, pChunkFree);
876	}
877	pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
878	pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
879	pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
880
881	#else /* !MMHYPER_HEAP_FREE_POISON */
882	/*
883	* Call the worker.
884	*/
885	int rc = mmHyperFree(pHeap, pChunk);
886	#endif /* !MMHYPER_HEAP_FREE_POISON */
887
888	/*
889	* Update statistics.
890	*/
891	#ifdef VBOX_WITH_STATISTICS
892	pStat->cFrees++;
893	if (RT_SUCCESS(rc))
894	{
895	pStat->cbFreed += cbChunk;
896	pStat->cbCurAllocated -= cbChunk;
897	}
898	else
899	pStat->cFailures++;
900	#endif
901
902	return rc;
903	}
904
905
906	/**
907	* Wrapper for mmHyperFreeInternal
908	*/
909	VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
910	{
911	int rc;
912
913	rc = mmHyperLock(pVM);
914	AssertRCReturn(rc, rc);
915
916	LogFlow(("MMHyperFree %p\n", pv));
917
918	rc = mmHyperFreeInternal(pVM, pv);
919
920	mmHyperUnlock(pVM);
921	return rc;
922	}
923
924
925	/**
926	* Free memory a memory chunk.
927	*
928	* @returns VBox status code.
929	* @param pHeap The heap.
930	* @param pChunk The memory chunk to free.
931	*/
932	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
933	{
934	Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
935	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
936
937	/*
938	* Insert into the free list (which is sorted on address).
939	*
940	* We'll search towards the end of the heap to locate the
941	* closest FREE chunk.
942	*/
943	PMMHYPERCHUNKFREE pLeft = NULL;
944	PMMHYPERCHUNKFREE pRight = NULL;
945	if (pHeap->offFreeTail != NIL_OFFSET)
946	{
947	if (pFree->core.offNext)
948	{
949	pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
950	ASSERT_CHUNK(pHeap, &pRight->core);
951	while (!MMHYPERCHUNK_ISFREE(&pRight->core))
952	{
953	if (!pRight->core.offNext)
954	{
955	pRight = NULL;
956	break;
957	}
958	pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
959	ASSERT_CHUNK(pHeap, &pRight->core);
960	}
961	}
962	if (!pRight)
963	pRight = (PMMHYPERCHUNKFREE)((char )pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /* @todo this can't be correct! 'pLeft = .. ; else' I think */
964	if (pRight)
965	{
966	ASSERT_CHUNK_FREE(pHeap, pRight);
967	if (pRight->offPrev)
968	{
969	pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
970	ASSERT_CHUNK_FREE(pHeap, pLeft);
971	}
972	}
973	}
974	if (pLeft == pFree)
975	{
976	AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
977	return VERR_INVALID_POINTER;
978	}
979	pChunk->offStat = 0;
980
981	/*
982	* Head free chunk list?
983	*/
984	if (!pLeft)
985	{
986	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
987	pFree->offPrev = 0;
988	pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
989	if (pRight)
990	{
991	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
992	pRight->offPrev = -(int32_t)pFree->offNext;
993	}
994	else
995	{
996	pFree->offNext = 0;
997	pHeap->offFreeTail = pHeap->offFreeHead;
998	}
999	Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
1000	}
1001	else
1002	{
1003	/*
1004	* Can we merge with left hand free chunk?
1005	*/
1006	if ((char )pLeft + pLeft->core.offNext == (char )pFree)
1007	{
1008	if (pFree->core.offNext)
1009	{
1010	pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
1011	MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
1012	}
1013	else
1014	pLeft->core.offNext = 0;
1015	pFree = pLeft;
1016	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1017	pHeap->cbFree -= pLeft->cb;
1018	Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
1019	}
1020	/*
1021	* No, just link it into the free list then.
1022	*/
1023	else
1024	{
1025	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1026	pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
1027	pLeft->offNext = -pFree->offPrev;
1028	if (pRight)
1029	{
1030	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1031	pRight->offPrev = -(int32_t)pFree->offNext;
1032	}
1033	else
1034	{
1035	pFree->offNext = 0;
1036	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1037	}
1038	Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
1039	}
1040	}
1041
1042	/*
1043	* Can we merge with right hand free chunk?
1044	*/
1045	if (pRight && (char )pRight == (char )pFree + pFree->core.offNext)
1046	{
1047	/* core */
1048	if (pRight->core.offNext)
1049	{
1050	pFree->core.offNext += pRight->core.offNext;
1051	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
1052	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
1053	ASSERT_CHUNK(pHeap, pNext);
1054	}
1055	else
1056	pFree->core.offNext = 0;
1057
1058	/* free */
1059	if (pRight->offNext)
1060	{
1061	pFree->offNext += pRight->offNext;
1062	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1063	}
1064	else
1065	{
1066	pFree->offNext = 0;
1067	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1068	}
1069	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1070	pHeap->cbFree -= pRight->cb;
1071	Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1072	}
1073
1074	/* calculate the size. */
1075	if (pFree->core.offNext)
1076	pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1077	else
1078	pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1079	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1080	pHeap->cbFree += pFree->cb;
1081	ASSERT_CHUNK_FREE(pHeap, pFree);
1082
1083	#ifdef MMHYPER_HEAP_STRICT
1084	mmHyperHeapCheck(pHeap);
1085	#endif
1086	return VINF_SUCCESS;
1087	}
1088
1089
1090	#if defined(DEBUG) \|\| defined(MMHYPER_HEAP_STRICT)
1091	/**
1092	* Dumps a heap chunk to the log.
1093	*
1094	* @param pHeap Pointer to the heap.
1095	* @param pCur Pointer to the chunk.
1096	*/
1097	static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1098	{
1099	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1100	{
1101	if (pCur->core.offStat)
1102	{
1103	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1104	const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1105	#ifdef IN_RING3
1106	Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1107	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1108	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1109	mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
1110	#else
1111	Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1112	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1113	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1114	(MMTAG)pStat->Core.Key, pszSelf));
1115	#endif
1116	}
1117	else
1118	Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1119	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1120	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1121	}
1122	else
1123	Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1124	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1125	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1126	}
1127	#endif /* DEBUG \|\| MMHYPER_HEAP_STRICT */
1128
1129
1130	#ifdef MMHYPER_HEAP_STRICT
1131	/**
1132	* Internal consistency check.
1133	*/
1134	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1135	{
1136	PMMHYPERCHUNKFREE pPrev = NULL;
1137	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1138	for (;;)
1139	{
1140	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1141	ASSERT_CHUNK_USED(pHeap, &pCur->core);
1142	else
1143	ASSERT_CHUNK_FREE(pHeap, pCur);
1144	if (pPrev)
1145	AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1146	("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1147
1148	# ifdef MMHYPER_HEAP_STRICT_FENCE
1149	uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1150	if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1151	&& off < pHeap->offPageAligned)
1152	{
1153	uint32_t cbCur = pCur->core.offNext
1154	? pCur->core.offNext
1155	: pHeap->cbHeap - off;
1156	uint32_t pu32End = ((uint32_t )((uint8_t *)pCur + cbCur));
1157	uint32_t cbFence = pu32End[-1];
1158	if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1159	\|\| cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1160	{
1161	mmHyperHeapDumpOne(pHeap, pCur);
1162	Assert(cbFence < cbCur - sizeof(*pCur));
1163	Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1164	}
1165
1166	uint32_t pu32Bad = ASMMemIsAllU32((uint8_t )pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1167	if (RT_UNLIKELY(pu32Bad))
1168	{
1169	mmHyperHeapDumpOne(pHeap, pCur);
1170	Assert(!pu32Bad);
1171	}
1172	}
1173	# endif
1174
1175	/* next */
1176	if (!pCur->core.offNext)
1177	break;
1178	pPrev = pCur;
1179	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1180	}
1181	}
1182	#endif
1183
1184
1185	/**
1186	* Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1187	* defined at build time.
1188	*
1189	* @param pVM Pointer to the VM.
1190	*/
1191	VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1192	{
1193	#ifdef MMHYPER_HEAP_STRICT
1194	int rc;
1195
1196	rc = mmHyperLock(pVM);
1197	AssertRC(rc);
1198	mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1199	mmHyperUnlock(pVM);
1200	#endif
1201	}
1202
1203
1204	#ifdef DEBUG
1205	/**
1206	* Dumps the hypervisor heap to Log.
1207	* @param pVM Pointer to the VM.
1208	*/
1209	VMMDECL(void) MMHyperHeapDump(PVM pVM)
1210	{
1211	Log(("MMHyperHeapDump: * heap dump - start *\n"));
1212	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1213	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1214	for (;;)
1215	{
1216	mmHyperHeapDumpOne(pHeap, pCur);
1217
1218	/* next */
1219	if (!pCur->core.offNext)
1220	break;
1221	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1222	}
1223	Log(("MMHyperHeapDump: * heap dump - end *\n"));
1224	}
1225	#endif
1226
1227
1228	/**
1229	* Query the amount of free memory in the hypervisor heap.
1230	*
1231	* @returns Number of free bytes in the hypervisor heap.
1232	*/
1233	VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1234	{
1235	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1236	}
1237
1238	/**
1239	* Query the size the hypervisor heap.
1240	*
1241	* @returns The size of the hypervisor heap in bytes.
1242	*/
1243	VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1244	{
1245	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1246	}
1247
1248
1249	/**
1250	* Query the address and size the hypervisor memory area.
1251	*
1252	* @returns Base address of the hypervisor area.
1253	* @param pVM Pointer to the VM.
1254	* @param pcb Where to store the size of the hypervisor area. (out)
1255	*/
1256	VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1257	{
1258	if (pcb)
1259	*pcb = pVM->mm.s.cbHyperArea;
1260	return pVM->mm.s.pvHyperAreaGC;
1261	}
1262
1263
1264	/**
1265	* Checks if an address is within the hypervisor memory area.
1266	*
1267	* @returns true if inside.
1268	* @returns false if outside.
1269	* @param pVM Pointer to the VM.
1270	* @param GCPtr The pointer to check.
1271	*/
1272	VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1273	{
1274	return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1275	}
1276

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 44577

Download in other formats: