VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 13038

Last change on this file since 13038 was 12989, checked in by vboxsync, 16 years ago

VMM + VBox/cdefs.h: consolidated all the XYZ*DECLS of the VMM into VMM*DECL. Removed dead DECL and IN_XYZ* macros.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 45.5 KB
Line 
1/* $Id: MMAllHyper.cpp 12989 2008-10-06 02:15:39Z vboxsync $ */
2/** @file
3 * MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
27#include <VBox/mm.h>
28#include <VBox/stam.h>
29#include "MMInternal.h"
30#include <VBox/vm.h>
31
32#include <VBox/err.h>
33#include <VBox/param.h>
34#include <iprt/assert.h>
35#include <VBox/log.h>
36#include <iprt/asm.h>
37#include <iprt/string.h>
38
39
40/*******************************************************************************
41* Defined Constants And Macros *
42*******************************************************************************/
43#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
44#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
45#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
46#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
47
48#define ASSERT_OFFPREV(pHeap, pChunk) \
49 do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
50 Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
51 AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
52 || (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
53 ("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
54 } while (0)
55
56#define ASSERT_OFFNEXT(pHeap, pChunk) \
57 do { ASSERT_ALIGN((pChunk)->offNext); \
58 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
59 } while (0)
60
61#define ASSERT_OFFHEAP(pHeap, pChunk) \
62 do { Assert((pChunk)->offHeap); \
63 AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
64 ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
65 Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
66 } while (0)
67
68#ifdef VBOX_WITH_STATISTICS
69#define ASSERT_OFFSTAT(pHeap, pChunk) \
70 do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
71 Assert(!(pChunk)->offStat); \
72 else if ((pChunk)->offStat) \
73 { \
74 Assert((pChunk)->offStat); \
75 AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
76 uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
77 AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
78 ("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
79 } \
80 } while (0)
81#else
82#define ASSERT_OFFSTAT(pHeap, pChunk) \
83 do { Assert(!(pChunk)->offStat); \
84 } while (0)
85#endif
86
87#define ASSERT_CHUNK(pHeap, pChunk) \
88 do { ASSERT_OFFNEXT(pHeap, pChunk); \
89 ASSERT_OFFPREV(pHeap, pChunk); \
90 ASSERT_OFFHEAP(pHeap, pChunk); \
91 ASSERT_OFFSTAT(pHeap, pChunk); \
92 } while (0)
93#define ASSERT_CHUNK_USED(pHeap, pChunk) \
94 do { ASSERT_OFFNEXT(pHeap, pChunk); \
95 ASSERT_OFFPREV(pHeap, pChunk); \
96 Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
97 } while (0)
98
99#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
100 do { ASSERT_ALIGN((pChunk)->offPrev); \
101 ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
102 Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) || !(pChunk)->offPrev); \
103 AssertMsg( (pChunk)->offPrev \
104 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
105 ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
106 (pHeap)->offFreeHead)); \
107 } while (0)
108
109#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
110 do { ASSERT_ALIGN((pChunk)->offNext); \
111 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
112 Assert((pChunk)->offNext != (pChunk)->core.offNext || !(pChunk)->offNext); \
113 AssertMsg( (pChunk)->offNext \
114 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
115 ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
116 (pHeap)->offFreeTail)); \
117 } while (0)
118
119#define ASSERT_FREE_CB(pHeap, pChunk) \
120 do { ASSERT_ALIGN((pChunk)->cb); \
121 Assert((pChunk)->cb > 0); \
122 if ((pChunk)->core.offNext) \
123 AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
124 ("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
125 else \
126 ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
127 } while (0)
128
129#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
130 do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
131 Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
132 ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
133 ASSERT_FREE_OFFPREV(pHeap, pChunk); \
134 ASSERT_FREE_CB(pHeap, pChunk); \
135 } while (0)
136
137
138/*******************************************************************************
139* Internal Functions *
140*******************************************************************************/
141static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
142static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
143#ifdef VBOX_WITH_STATISTICS
144static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
145#ifdef IN_RING3
146static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
147#endif
148#endif
149static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
150#ifdef MMHYPER_HEAP_STRICT
151static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
152#endif
153
154
155/**
156 * Allocates memory in the Hypervisor (RC VMM) area.
157 * The returned memory is of course zeroed.
158 *
159 * @returns VBox status code.
160 * @param pVM The VM to operate on.
161 * @param cb Number of bytes to allocate.
162 * @param uAlignment Required memory alignment in bytes.
163 * Values are 0,8,16,32 and PAGE_SIZE.
164 * 0 -> default alignment, i.e. 8 bytes.
165 * @param enmTag The statistics tag.
166 * @param ppv Where to store the address to the allocated
167 * memory.
168 * @remark This is assumed not to be used at times when serialization is required.
169 */
170VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
171{
172 AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
173
174 /*
175 * Validate input and adjust it to reasonable values.
176 */
177 if (!uAlignment || uAlignment < MMHYPER_HEAP_ALIGN_MIN)
178 uAlignment = MMHYPER_HEAP_ALIGN_MIN;
179 uint32_t cbAligned;
180 switch (uAlignment)
181 {
182 case 8:
183 case 16:
184 case 32:
185 cbAligned = RT_ALIGN(cb, MMHYPER_HEAP_ALIGN_MIN);
186 if (!cbAligned || cbAligned < cb)
187 {
188 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
189 AssertMsgFailed(("Nice try.\n"));
190 return VERR_INVALID_PARAMETER;
191 }
192 break;
193
194 case PAGE_SIZE:
195 AssertMsg(RT_ALIGN(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
196 cbAligned = RT_ALIGN(cb, PAGE_SIZE);
197 if (!cbAligned)
198 {
199 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
200 AssertMsgFailed(("Nice try.\n"));
201 return VERR_INVALID_PARAMETER;
202 }
203 break;
204
205 default:
206 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
207 AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
208 return VERR_INVALID_PARAMETER;
209 }
210
211
212 /*
213 * Get heap and statisticsStatistics.
214 */
215 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
216#ifdef VBOX_WITH_STATISTICS
217 PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
218 if (!pStat)
219 {
220 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
221 AssertMsgFailed(("Failed to allocate statistics!\n"));
222 return VERR_MM_HYPER_NO_MEMORY;
223 }
224#endif
225 if (uAlignment < PAGE_SIZE)
226 {
227 /*
228 * Allocate a chunk.
229 */
230 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
231 if (pChunk)
232 {
233#ifdef VBOX_WITH_STATISTICS
234 const uint32_t cbChunk = pChunk->offNext
235 ? pChunk->offNext
236 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
237 pStat->cbAllocated += (uint32_t)cbChunk;
238 pStat->cbCurAllocated += (uint32_t)cbChunk;
239 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
240 pStat->cbMaxAllocated = pStat->cbCurAllocated;
241 pStat->cAllocations++;
242 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
243#else
244 pChunk->offStat = 0;
245#endif
246 void *pv = pChunk + 1;
247 *ppv = pv;
248 ASMMemZero32(pv, cbAligned);
249 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
250 return VINF_SUCCESS;
251 }
252 }
253 else
254 {
255 /*
256 * Allocate page aligned memory.
257 */
258 void *pv = mmHyperAllocPages(pHeap, cbAligned);
259 if (pv)
260 {
261#ifdef VBOX_WITH_STATISTICS
262 pStat->cbAllocated += cbAligned;
263 pStat->cbCurAllocated += cbAligned;
264 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
265 pStat->cbMaxAllocated = pStat->cbCurAllocated;
266 pStat->cAllocations++;
267#endif
268 *ppv = pv;
269 /* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPPageAlloc zeros it. */
270 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
271 return VINF_SUCCESS;
272 }
273 }
274
275#ifdef VBOX_WITH_STATISTICS
276 pStat->cAllocations++;
277 pStat->cFailures++;
278#endif
279 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
280 AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
281 return VERR_MM_HYPER_NO_MEMORY;
282}
283
284
285
286/**
287 * Allocates a chunk of memory from the specified heap.
288 * The caller validates the parameters of this request.
289 *
290 * @returns Pointer to the allocated chunk.
291 * @returns NULL on failure.
292 * @param pHeap The heap.
293 * @param cb Size of the memory block to allocate.
294 * @param uAlignment The alignment specifications for the allocated block.
295 * @internal
296 */
297static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
298{
299 Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
300#ifdef MMHYPER_HEAP_STRICT
301 mmHyperHeapCheck(pHeap);
302#endif
303#ifdef MMHYPER_HEAP_STRICT_FENCE
304 uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
305 cb += cbFence;
306#endif
307
308 /*
309 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
310 */
311 if (pHeap->offFreeHead == NIL_OFFSET)
312 return NULL;
313
314 /*
315 * Small alignments - from the front of the heap.
316 *
317 * Must split off free chunks at the end to prevent messing up the
318 * last free node which we take the page aligned memory from the top of.
319 */
320 PMMHYPERCHUNK pRet = NULL;
321 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
322 while (pFree)
323 {
324 ASSERT_CHUNK_FREE(pHeap, pFree);
325 if (pFree->cb >= cb)
326 {
327 unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
328 if (offAlign)
329 offAlign = uAlignment - offAlign;
330 if (!offAlign || pFree->cb - offAlign >= cb)
331 {
332 Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
333
334 /*
335 * Adjust the node in front.
336 * Because of multiple alignments we need to special case allocation of the first block.
337 */
338 if (offAlign)
339 {
340 MMHYPERCHUNKFREE Free = *pFree;
341 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
342 {
343 /* just add a bit of memory to it. */
344 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
345 pPrev->core.offNext += offAlign;
346 AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
347 Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
348 }
349 else
350 {
351 /* make new head node, mark it USED for simplisity. */
352 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
353 Assert(pPrev == &pFree->core);
354 pPrev->offPrev = 0;
355 MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
356 pPrev->offNext = offAlign;
357 Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
358
359 }
360 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
361 pHeap->cbFree -= offAlign;
362
363 /* Recreate pFree node and adjusting everything... */
364 pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
365 *pFree = Free;
366
367 pFree->cb -= offAlign;
368 if (pFree->core.offNext)
369 {
370 pFree->core.offNext -= offAlign;
371 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
372 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
373 ASSERT_CHUNK(pHeap, pNext);
374 }
375 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
376 MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
377
378 if (pFree->offNext)
379 {
380 pFree->offNext -= offAlign;
381 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
382 pNext->offPrev = -(int32_t)pFree->offNext;
383 ASSERT_CHUNK_FREE(pHeap, pNext);
384 }
385 else
386 pHeap->offFreeTail += offAlign;
387 if (pFree->offPrev)
388 {
389 pFree->offPrev -= offAlign;
390 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
391 pPrev->offNext = -pFree->offPrev;
392 ASSERT_CHUNK_FREE(pHeap, pPrev);
393 }
394 else
395 pHeap->offFreeHead += offAlign;
396 pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
397 pFree->core.offStat = 0;
398 ASSERT_CHUNK_FREE(pHeap, pFree);
399 Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
400 }
401
402 /*
403 * Split off a new FREE chunk?
404 */
405 if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
406 {
407 /*
408 * Move the FREE chunk up to make room for the new USED chunk.
409 */
410 const int off = cb + sizeof(MMHYPERCHUNK);
411 PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
412 *pNew = *pFree;
413 pNew->cb -= off;
414 if (pNew->core.offNext)
415 {
416 pNew->core.offNext -= off;
417 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
418 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
419 ASSERT_CHUNK(pHeap, pNext);
420 }
421 pNew->core.offPrev = -off;
422 MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
423
424 if (pNew->offNext)
425 {
426 pNew->offNext -= off;
427 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
428 pNext->offPrev = -(int32_t)pNew->offNext;
429 ASSERT_CHUNK_FREE(pHeap, pNext);
430 }
431 else
432 pHeap->offFreeTail += off;
433 if (pNew->offPrev)
434 {
435 pNew->offPrev -= off;
436 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
437 pPrev->offNext = -pNew->offPrev;
438 ASSERT_CHUNK_FREE(pHeap, pPrev);
439 }
440 else
441 pHeap->offFreeHead += off;
442 pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
443 pNew->core.offStat = 0;
444 ASSERT_CHUNK_FREE(pHeap, pNew);
445
446 /*
447 * Update the old FREE node making it a USED node.
448 */
449 pFree->core.offNext = off;
450 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
451
452
453 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
454 pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
455 pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
456 pRet = &pFree->core;
457 ASSERT_CHUNK(pHeap, &pFree->core);
458 Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
459 }
460 else
461 {
462 /*
463 * Link out of free list.
464 */
465 if (pFree->offNext)
466 {
467 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
468 if (pFree->offPrev)
469 {
470 pNext->offPrev += pFree->offPrev;
471 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
472 pPrev->offNext += pFree->offNext;
473 ASSERT_CHUNK_FREE(pHeap, pPrev);
474 }
475 else
476 {
477 pHeap->offFreeHead += pFree->offNext;
478 pNext->offPrev = 0;
479 }
480 ASSERT_CHUNK_FREE(pHeap, pNext);
481 }
482 else
483 {
484 if (pFree->offPrev)
485 {
486 pHeap->offFreeTail += pFree->offPrev;
487 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
488 pPrev->offNext = 0;
489 ASSERT_CHUNK_FREE(pHeap, pPrev);
490 }
491 else
492 {
493 pHeap->offFreeHead = NIL_OFFSET;
494 pHeap->offFreeTail = NIL_OFFSET;
495 }
496 }
497
498 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
499 pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
500 pHeap->cbFree -= pFree->cb;
501 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
502 pRet = &pFree->core;
503 ASSERT_CHUNK(pHeap, &pFree->core);
504 Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
505 }
506 Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
507 break;
508 }
509 }
510
511 /* next */
512 pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
513 }
514
515#ifdef MMHYPER_HEAP_STRICT_FENCE
516 uint32_t *pu32End = (uint32_t *)((uint8_t *)(pRet + 1) + cb);
517 uint32_t *pu32EndReal = pRet->offNext
518 ? (uint32_t *)((uint8_t *)pRet + pRet->offNext)
519 : (uint32_t *)(pHeap->CTXSUFF(pbHeap) + pHeap->cbHeap);
520 cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
521 ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
522 pu32EndReal[-1] = cbFence;
523#endif
524#ifdef MMHYPER_HEAP_STRICT
525 mmHyperHeapCheck(pHeap);
526#endif
527 return pRet;
528}
529
530
531/**
532 * Allocates one or more pages of memory from the specified heap.
533 * The caller validates the parameters of this request.
534 *
535 * @returns Pointer to the allocated chunk.
536 * @returns NULL on failure.
537 * @param pHeap The heap.
538 * @param cb Size of the memory block to allocate.
539 * @internal
540 */
541static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
542{
543 Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
544
545#ifdef MMHYPER_HEAP_STRICT
546 mmHyperHeapCheck(pHeap);
547#endif
548
549 /*
550 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
551 */
552 if (pHeap->offFreeHead == NIL_OFFSET)
553 return NULL;
554
555 /*
556 * Page aligned chunks.
557 *
558 * Page aligned chunks can only be allocated from the last FREE chunk.
559 * This is for reasons of simplicity and fragmentation. Page aligned memory
560 * must also be allocated in page aligned sizes. Page aligned memory cannot
561 * be freed either.
562 *
563 * So, for this to work, the last FREE chunk needs to end on a page aligned
564 * boundrary.
565 */
566 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
567 ASSERT_CHUNK_FREE(pHeap, pFree);
568 if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
569 || pFree->cb + sizeof(MMHYPERCHUNK) < cb)
570 {
571 Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
572 return NULL;
573 }
574
575 void *pvRet;
576 if (pFree->cb > cb)
577 {
578 /*
579 * Simple, just cut the top of the free node and return it.
580 */
581 pFree->cb -= cb;
582 pvRet = (char *)(&pFree->core + 1) + pFree->cb;
583 AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
584 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
585 pHeap->cbFree -= cb;
586 ASSERT_CHUNK_FREE(pHeap, pFree);
587 Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
588 }
589 else
590 {
591 /*
592 * Unlink the FREE node.
593 */
594 pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
595 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
596 pHeap->cbFree -= pFree->cb;
597
598 /* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
599 if (pvRet != (void *)pFree)
600 {
601 AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the *beep* did someone manage to allocated up all the heap with page aligned memory?!?\n"));
602 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
603 pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
604 AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
605#ifdef VBOX_WITH_STATISTICS
606 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
607 pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
608 pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
609#endif
610 Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
611 }
612
613 /* unlink from FREE chain. */
614 if (pFree->offPrev)
615 {
616 pHeap->offFreeTail += pFree->offPrev;
617 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
618 }
619 else
620 {
621 pHeap->offFreeTail = NIL_OFFSET;
622 pHeap->offFreeHead = NIL_OFFSET;
623 }
624 Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
625 }
626 pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
627 Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
628
629#ifdef MMHYPER_HEAP_STRICT
630 mmHyperHeapCheck(pHeap);
631#endif
632 return pvRet;
633}
634
635#ifdef VBOX_WITH_STATISTICS
636
637/**
638 * Get the statistic record for a tag.
639 *
640 * @returns Pointer to a stat record.
641 * @returns NULL on failure.
642 * @param pHeap The heap.
643 * @param enmTag The tag.
644 */
645static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
646{
647 /* try look it up first. */
648 PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
649 if (!pStat)
650 {
651 /* try allocate a new one */
652 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
653 if (!pChunk)
654 return NULL;
655 pStat = (PMMHYPERSTAT)(pChunk + 1);
656 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
657
658 ASMMemZero32(pStat, sizeof(*pStat));
659 pStat->Core.Key = enmTag;
660 RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
661 }
662 if (!pStat->fRegistered)
663 {
664# ifdef IN_RING3
665 mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
666# else
667 /** @todo schedule a R3 action. */
668# endif
669 }
670 return pStat;
671}
672
673
674# ifdef IN_RING3
675/**
676 * Registers statistics with STAM.
677 *
678 */
679static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
680{
681 if (pStat->fRegistered)
682 return;
683 const char *pszTag = mmR3GetTagName((MMTAG)pStat->Core.Key);
684 STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
685 STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
686 STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
687 STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
688 STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
689 STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s/cbCurAllocated", pszTag);
690 STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
691 pStat->fRegistered = true;
692}
693# endif /* IN_RING3 */
694
695#endif /* VBOX_WITH_STATISTICS */
696
697
698/**
699 * Free memory allocated using MMHyperAlloc().
700 * The caller validates the parameters of this request.
701 *
702 * @returns VBox status code.
703 * @param pVM The VM to operate on.
704 * @param pv The memory to free.
705 * @remark Try avoid free hyper memory.
706 */
707VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
708{
709 Log2(("MMHyperFree: pv=%p\n", pv));
710 if (!pv)
711 return VINF_SUCCESS;
712 AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
713 ("Invalid pointer %p!\n", pv),
714 VERR_INVALID_POINTER);
715
716 /*
717 * Get the heap and stats.
718 * Validate the chunk at the same time.
719 */
720 PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
721
722 AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
723 || RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
724 ("%p: offNext=%#RX32\n", pv, pChunk->offNext),
725 VERR_INVALID_POINTER);
726
727 AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
728 ("%p: Not used!\n", pv),
729 VERR_INVALID_POINTER);
730
731 int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
732 AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
733 && !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
734 ("%p: offPrev=%#RX32!\n", pv, offPrev),
735 VERR_INVALID_POINTER);
736
737 /* statistics */
738#ifdef VBOX_WITH_STATISTICS
739 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
740 AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
741 && pChunk->offStat,
742 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
743 VERR_INVALID_POINTER);
744#else
745 AssertMsgReturn(!pChunk->offStat,
746 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
747 VERR_INVALID_POINTER);
748#endif
749
750 /* The heap structure. */
751 PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
752 AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
753 && pChunk->offHeap,
754 ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
755 VERR_INVALID_POINTER);
756
757 AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
758 ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
759 VERR_INVALID_POINTER);
760Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
761
762 /* Some more verifications using additional info from pHeap. */
763 AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
764 ("%p: offPrev=%#RX32!\n", pv, offPrev),
765 VERR_INVALID_POINTER);
766
767 AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
768 ("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
769 VERR_INVALID_POINTER);
770
771 AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
772 ("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
773 (char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
774 VERR_INVALID_POINTER);
775
776#ifdef MMHYPER_HEAP_STRICT
777 mmHyperHeapCheck(pHeap);
778#endif
779
780#if defined(VBOX_WITH_STATISTICS) || defined(MMHYPER_HEAP_FREE_POISON)
781 /* calc block size. */
782 const uint32_t cbChunk = pChunk->offNext
783 ? pChunk->offNext
784 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
785#endif
786#ifdef MMHYPER_HEAP_FREE_POISON
787 /* poison the block */
788 memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
789#endif
790
791#ifdef MMHYPER_HEAP_FREE_DELAY
792# ifdef MMHYPER_HEAP_FREE_POISON
793 /*
794 * Check poison.
795 */
796 unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
797 while (i-- > 0)
798 if (pHeap->aDelayedFrees[i].offChunk)
799 {
800 PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
801 const size_t cb = pCur->offNext
802 ? pCur->offNext - sizeof(*pCur)
803 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
804 uint8_t *pab = (uint8_t *)(pCur + 1);
805 for (unsigned off = 0; off < cb; off++)
806 AssertReleaseMsg(pab[off] == 0xCB,
807 ("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
808 pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
809 }
810# endif /* MMHYPER_HEAP_FREE_POISON */
811
812 /*
813 * Delayed freeing.
814 */
815 int rc = VINF_SUCCESS;
816 if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
817 {
818 PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
819 rc = mmHyperFree(pHeap, pChunkFree);
820 }
821 pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
822 pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
823 pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
824
825#else /* !MMHYPER_HEAP_FREE_POISON */
826 /*
827 * Call the worker.
828 */
829 int rc = mmHyperFree(pHeap, pChunk);
830#endif /* !MMHYPER_HEAP_FREE_POISON */
831
832 /*
833 * Update statistics.
834 */
835#ifdef VBOX_WITH_STATISTICS
836 pStat->cFrees++;
837 if (VBOX_SUCCESS(rc))
838 {
839 pStat->cbFreed += cbChunk;
840 pStat->cbCurAllocated -= cbChunk;
841 }
842 else
843 pStat->cFailures++;
844#endif
845
846 return rc;
847}
848
849
850/**
851 * Free memory a memory chunk.
852 *
853 * @returns VBox status code.
854 * @param pHeap The heap.
855 * @param pChunk The memory chunk to free.
856 */
857static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
858{
859 Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
860 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
861
862 /*
863 * Insert into the free list (which is sorted on address).
864 *
865 * We'll search towards the end of the heap to locate the
866 * closest FREE chunk.
867 */
868 PMMHYPERCHUNKFREE pLeft = NULL;
869 PMMHYPERCHUNKFREE pRight = NULL;
870 if (pHeap->offFreeTail != NIL_OFFSET)
871 {
872 if (pFree->core.offNext)
873 {
874 pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
875 ASSERT_CHUNK(pHeap, &pRight->core);
876 while (!MMHYPERCHUNK_ISFREE(&pRight->core))
877 {
878 if (!pRight->core.offNext)
879 {
880 pRight = NULL;
881 break;
882 }
883 pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
884 ASSERT_CHUNK(pHeap, &pRight->core);
885 }
886 }
887 if (!pRight)
888 pRight = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
889 if (pRight)
890 {
891 ASSERT_CHUNK_FREE(pHeap, pRight);
892 if (pRight->offPrev)
893 {
894 pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
895 ASSERT_CHUNK_FREE(pHeap, pLeft);
896 }
897 }
898 }
899 if (pLeft == pFree)
900 {
901 AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
902 return VERR_INVALID_POINTER;
903 }
904 pChunk->offStat = 0;
905
906 /*
907 * Head free chunk list?
908 */
909 if (!pLeft)
910 {
911 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
912 pFree->offPrev = 0;
913 pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
914 if (pRight)
915 {
916 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
917 pRight->offPrev = -(int32_t)pFree->offNext;
918 }
919 else
920 {
921 pFree->offNext = 0;
922 pHeap->offFreeTail = pHeap->offFreeHead;
923 }
924 Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
925 }
926 else
927 {
928 /*
929 * Can we merge with left hand free chunk?
930 */
931 if ((char *)pLeft + pLeft->core.offNext == (char *)pFree)
932 {
933 if (pFree->core.offNext)
934 {
935 pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
936 MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
937 }
938 else
939 pLeft->core.offNext = 0;
940 pFree = pLeft;
941 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
942 pHeap->cbFree -= pLeft->cb;
943 Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
944 }
945 /*
946 * No, just link it into the free list then.
947 */
948 else
949 {
950 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
951 pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
952 pLeft->offNext = -pFree->offPrev;
953 if (pRight)
954 {
955 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
956 pRight->offPrev = -(int32_t)pFree->offNext;
957 }
958 else
959 {
960 pFree->offNext = 0;
961 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
962 }
963 Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
964 }
965 }
966
967 /*
968 * Can we merge with right hand free chunk?
969 */
970 if (pRight && (char *)pRight == (char *)pFree + pFree->core.offNext)
971 {
972 /* core */
973 if (pRight->core.offNext)
974 {
975 pFree->core.offNext += pRight->core.offNext;
976 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
977 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
978 ASSERT_CHUNK(pHeap, pNext);
979 }
980 else
981 pFree->core.offNext = 0;
982
983 /* free */
984 if (pRight->offNext)
985 {
986 pFree->offNext += pRight->offNext;
987 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
988 }
989 else
990 {
991 pFree->offNext = 0;
992 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
993 }
994 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
995 pHeap->cbFree -= pRight->cb;
996 Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
997 }
998
999 /* calculate the size. */
1000 if (pFree->core.offNext)
1001 pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1002 else
1003 pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1004 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1005 pHeap->cbFree += pFree->cb;
1006 ASSERT_CHUNK_FREE(pHeap, pFree);
1007
1008#ifdef MMHYPER_HEAP_STRICT
1009 mmHyperHeapCheck(pHeap);
1010#endif
1011 return VINF_SUCCESS;
1012}
1013
1014
1015#if defined(DEBUG) || defined(MMHYPER_HEAP_STRICT)
1016/**
1017 * Dumps a heap chunk to the log.
1018 *
1019 * @param pHeap Pointer to the heap.
1020 * @param pCur Pointer to the chunk.
1021 */
1022static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1023{
1024 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1025 {
1026 if (pCur->core.offStat)
1027 {
1028 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1029 const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1030#ifdef IN_RING3
1031 Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1032 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1033 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1034 mmR3GetTagName((MMTAG)pStat->Core.Key), pszSelf));
1035#else
1036 Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1037 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1038 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1039 (MMTAG)pStat->Core.Key, pszSelf));
1040#endif
1041 }
1042 else
1043 Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1044 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1045 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1046 }
1047 else
1048 Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1049 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1050 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1051}
1052#endif /* DEBUG || MMHYPER_HEAP_STRICT */
1053
1054
1055#ifdef MMHYPER_HEAP_STRICT
1056/**
1057 * Internal consitency check.
1058 */
1059static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1060{
1061 PMMHYPERCHUNKFREE pPrev = NULL;
1062 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1063 for (;;)
1064 {
1065 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1066 ASSERT_CHUNK_USED(pHeap, &pCur->core);
1067 else
1068 ASSERT_CHUNK_FREE(pHeap, pCur);
1069 if (pPrev)
1070 AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1071 ("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1072
1073# ifdef MMHYPER_HEAP_STRICT_FENCE
1074 uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1075 if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1076 && off < pHeap->offPageAligned)
1077 {
1078 uint32_t cbCur = pCur->core.offNext
1079 ? pCur->core.offNext
1080 : pHeap->cbHeap - off;
1081 uint32_t *pu32End = ((uint32_t *)((uint8_t *)pCur + cbCur));
1082 uint32_t cbFence = pu32End[-1];
1083 if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1084 || cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1085 {
1086 mmHyperHeapDumpOne(pHeap, pCur);
1087 Assert(cbFence < cbCur - sizeof(*pCur));
1088 Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1089 }
1090
1091 uint32_t *pu32Bad = ASMMemIsAllU32((uint8_t *)pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1092 if (RT_UNLIKELY(pu32Bad))
1093 {
1094 mmHyperHeapDumpOne(pHeap, pCur);
1095 Assert(!pu32Bad);
1096 }
1097 }
1098# endif
1099
1100 /* next */
1101 if (!pCur->core.offNext)
1102 break;
1103 pPrev = pCur;
1104 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1105 }
1106}
1107#endif
1108
1109
1110/**
1111 * Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1112 * defined at build time.
1113 *
1114 * @param pVM Pointer to the shared VM structure.
1115 */
1116VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1117{
1118#ifdef MMHYPER_HEAP_STRICT
1119 mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1120#endif
1121}
1122
1123
1124#ifdef DEBUG
1125/**
1126 * Dumps the hypervisor heap to Log.
1127 * @param pVM VM Handle.
1128 */
1129VMMDECL(void) MMHyperHeapDump(PVM pVM)
1130{
1131 Log(("MMHyperHeapDump: *** heap dump - start ***\n"));
1132 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1133 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1134 for (;;)
1135 {
1136 mmHyperHeapDumpOne(pHeap, pCur);
1137
1138 /* next */
1139 if (!pCur->core.offNext)
1140 break;
1141 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1142 }
1143 Log(("MMHyperHeapDump: *** heap dump - end ***\n"));
1144}
1145#endif
1146
1147
1148/**
1149 * Query the amount of free memory in the hypervisor heap.
1150 *
1151 * @returns Number of free bytes in the hypervisor heap.
1152 */
1153VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1154{
1155 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1156}
1157
1158/**
1159 * Query the size the hypervisor heap.
1160 *
1161 * @returns The size of the hypervisor heap in bytes.
1162 */
1163VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1164{
1165 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1166}
1167
1168
1169/**
1170 * Query the address and size the hypervisor memory area.
1171 *
1172 * @returns Base address of the hypervisor area.
1173 * @param pVM VM Handle.
1174 * @param pcb Where to store the size of the hypervisor area. (out)
1175 */
1176VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1177{
1178 if (pcb)
1179 *pcb = pVM->mm.s.cbHyperArea;
1180 return pVM->mm.s.pvHyperAreaGC;
1181}
1182
1183
1184/**
1185 * Checks if an address is within the hypervisor memory area.
1186 *
1187 * @returns true if inside.
1188 * @returns false if outside.
1189 * @param pVM VM handle.
1190 * @param GCPtr The pointer to check.
1191 */
1192VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1193{
1194 return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1195}
1196
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette