VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 22997

Last change on this file since 22997 was 22026, checked in by vboxsync, 15 years ago

space
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 47.2 KB
Line 
1/* $Id: MMAllHyper.cpp 22026 2009-08-06 11:36:27Z vboxsync $ */
2/** @file
3 * MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
27#include <VBox/mm.h>
28#include <VBox/stam.h>
29#include "MMInternal.h"
30#include <VBox/vm.h>
31
32#include <VBox/err.h>
33#include <VBox/param.h>
34#include <iprt/assert.h>
35#include <VBox/log.h>
36#include <iprt/asm.h>
37#include <iprt/string.h>
38
39
40/*******************************************************************************
41* Defined Constants And Macros *
42*******************************************************************************/
43#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
44#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
45#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
46#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
47
48#define ASSERT_OFFPREV(pHeap, pChunk) \
49 do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
50 Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
51 AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
52 || (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
53 ("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
54 } while (0)
55
56#define ASSERT_OFFNEXT(pHeap, pChunk) \
57 do { ASSERT_ALIGN((pChunk)->offNext); \
58 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
59 } while (0)
60
61#define ASSERT_OFFHEAP(pHeap, pChunk) \
62 do { Assert((pChunk)->offHeap); \
63 AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
64 ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
65 Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
66 } while (0)
67
68#ifdef VBOX_WITH_STATISTICS
69#define ASSERT_OFFSTAT(pHeap, pChunk) \
70 do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
71 Assert(!(pChunk)->offStat); \
72 else if ((pChunk)->offStat) \
73 { \
74 Assert((pChunk)->offStat); \
75 AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
76 uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
77 AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
78 ("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
79 } \
80 } while (0)
81#else
82#define ASSERT_OFFSTAT(pHeap, pChunk) \
83 do { Assert(!(pChunk)->offStat); \
84 } while (0)
85#endif
86
87#define ASSERT_CHUNK(pHeap, pChunk) \
88 do { ASSERT_OFFNEXT(pHeap, pChunk); \
89 ASSERT_OFFPREV(pHeap, pChunk); \
90 ASSERT_OFFHEAP(pHeap, pChunk); \
91 ASSERT_OFFSTAT(pHeap, pChunk); \
92 } while (0)
93#define ASSERT_CHUNK_USED(pHeap, pChunk) \
94 do { ASSERT_OFFNEXT(pHeap, pChunk); \
95 ASSERT_OFFPREV(pHeap, pChunk); \
96 Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
97 } while (0)
98
99#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
100 do { ASSERT_ALIGN((pChunk)->offPrev); \
101 ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
102 Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) || !(pChunk)->offPrev); \
103 AssertMsg( (pChunk)->offPrev \
104 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
105 ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
106 (pHeap)->offFreeHead)); \
107 } while (0)
108
109#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
110 do { ASSERT_ALIGN((pChunk)->offNext); \
111 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
112 Assert((pChunk)->offNext != (pChunk)->core.offNext || !(pChunk)->offNext); \
113 AssertMsg( (pChunk)->offNext \
114 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
115 ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
116 (pHeap)->offFreeTail)); \
117 } while (0)
118
119#define ASSERT_FREE_CB(pHeap, pChunk) \
120 do { ASSERT_ALIGN((pChunk)->cb); \
121 Assert((pChunk)->cb > 0); \
122 if ((pChunk)->core.offNext) \
123 AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
124 ("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
125 else \
126 ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
127 } while (0)
128
129#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
130 do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
131 Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
132 ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
133 ASSERT_FREE_OFFPREV(pHeap, pChunk); \
134 ASSERT_FREE_CB(pHeap, pChunk); \
135 } while (0)
136
137
138/*******************************************************************************
139* Internal Functions *
140*******************************************************************************/
141static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
142static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
143#ifdef VBOX_WITH_STATISTICS
144static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
145#ifdef IN_RING3
146static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
147#endif
148#endif
149static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
150#ifdef MMHYPER_HEAP_STRICT
151static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
152#endif
153
154
155
156/**
157 * Locks the hypervisor heap.
158 * This might call back to Ring-3 in order to deal with lock contention in GC and R3.
159 *
160 * @param pVM The VM handle.
161 */
162static int mmHyperLock(PVM pVM)
163{
164 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
165
166#ifdef IN_RING3
167 if (!PDMCritSectIsInitialized(&pHeap->Lock))
168 return VINF_SUCCESS; /* early init */
169#else
170 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
171#endif
172 int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
173#if defined(IN_RC) || defined(IN_RING0)
174 if (rc == VERR_SEM_BUSY)
175 rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
176#endif
177 AssertRC(rc);
178 return rc;
179}
180
181
182/**
183 * Unlocks the hypervisor heap.
184 *
185 * @param pVM The VM handle.
186 */
187static void mmHyperUnlock(PVM pVM)
188{
189 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
190
191#ifdef IN_RING3
192 if (!PDMCritSectIsInitialized(&pHeap->Lock))
193 return; /* early init */
194#endif
195 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
196 PDMCritSectLeave(&pHeap->Lock);
197}
198
199/**
200 * Allocates memory in the Hypervisor (RC VMM) area.
201 * The returned memory is of course zeroed.
202 *
203 * @returns VBox status code.
204 * @param pVM The VM to operate on.
205 * @param cb Number of bytes to allocate.
206 * @param uAlignment Required memory alignment in bytes.
207 * Values are 0,8,16,32 and PAGE_SIZE.
208 * 0 -> default alignment, i.e. 8 bytes.
209 * @param enmTag The statistics tag.
210 * @param ppv Where to store the address to the allocated
211 * memory.
212 */
213static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
214{
215 AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
216
217 /*
218 * Validate input and adjust it to reasonable values.
219 */
220 if (!uAlignment || uAlignment < MMHYPER_HEAP_ALIGN_MIN)
221 uAlignment = MMHYPER_HEAP_ALIGN_MIN;
222 uint32_t cbAligned;
223 switch (uAlignment)
224 {
225 case 8:
226 case 16:
227 case 32:
228 cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
229 if (!cbAligned || cbAligned < cb)
230 {
231 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
232 AssertMsgFailed(("Nice try.\n"));
233 return VERR_INVALID_PARAMETER;
234 }
235 break;
236
237 case PAGE_SIZE:
238 AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
239 cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
240 if (!cbAligned)
241 {
242 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
243 AssertMsgFailed(("Nice try.\n"));
244 return VERR_INVALID_PARAMETER;
245 }
246 break;
247
248 default:
249 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
250 AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
251 return VERR_INVALID_PARAMETER;
252 }
253
254
255 /*
256 * Get heap and statisticsStatistics.
257 */
258 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
259#ifdef VBOX_WITH_STATISTICS
260 PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
261 if (!pStat)
262 {
263 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
264 AssertMsgFailed(("Failed to allocate statistics!\n"));
265 return VERR_MM_HYPER_NO_MEMORY;
266 }
267#endif
268 if (uAlignment < PAGE_SIZE)
269 {
270 /*
271 * Allocate a chunk.
272 */
273 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
274 if (pChunk)
275 {
276#ifdef VBOX_WITH_STATISTICS
277 const uint32_t cbChunk = pChunk->offNext
278 ? pChunk->offNext
279 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
280 pStat->cbAllocated += (uint32_t)cbChunk;
281 pStat->cbCurAllocated += (uint32_t)cbChunk;
282 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
283 pStat->cbMaxAllocated = pStat->cbCurAllocated;
284 pStat->cAllocations++;
285 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
286#else
287 pChunk->offStat = 0;
288#endif
289 void *pv = pChunk + 1;
290 *ppv = pv;
291 ASMMemZero32(pv, cbAligned);
292 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
293 return VINF_SUCCESS;
294 }
295 }
296 else
297 {
298 /*
299 * Allocate page aligned memory.
300 */
301 void *pv = mmHyperAllocPages(pHeap, cbAligned);
302 if (pv)
303 {
304#ifdef VBOX_WITH_STATISTICS
305 pStat->cbAllocated += cbAligned;
306 pStat->cbCurAllocated += cbAligned;
307 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
308 pStat->cbMaxAllocated = pStat->cbCurAllocated;
309 pStat->cAllocations++;
310#endif
311 *ppv = pv;
312 /* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
313 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
314 return VINF_SUCCESS;
315 }
316 }
317
318#ifdef VBOX_WITH_STATISTICS
319 pStat->cAllocations++;
320 pStat->cFailures++;
321#endif
322 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
323 AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
324 return VERR_MM_HYPER_NO_MEMORY;
325}
326
327/**
328 * Wrapper for mmHyperAllocInternal
329 */
330VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
331{
332 int rc;
333
334 rc = mmHyperLock(pVM);
335 AssertRCReturn(rc, rc);
336
337 LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
338
339 rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
340
341 mmHyperUnlock(pVM);
342 return rc;
343}
344
345/**
346 * Allocates a chunk of memory from the specified heap.
347 * The caller validates the parameters of this request.
348 *
349 * @returns Pointer to the allocated chunk.
350 * @returns NULL on failure.
351 * @param pHeap The heap.
352 * @param cb Size of the memory block to allocate.
353 * @param uAlignment The alignment specifications for the allocated block.
354 * @internal
355 */
356static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
357{
358 Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
359#ifdef MMHYPER_HEAP_STRICT
360 mmHyperHeapCheck(pHeap);
361#endif
362#ifdef MMHYPER_HEAP_STRICT_FENCE
363 uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
364 cb += cbFence;
365#endif
366
367 /*
368 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
369 */
370 if (pHeap->offFreeHead == NIL_OFFSET)
371 return NULL;
372
373 /*
374 * Small alignments - from the front of the heap.
375 *
376 * Must split off free chunks at the end to prevent messing up the
377 * last free node which we take the page aligned memory from the top of.
378 */
379 PMMHYPERCHUNK pRet = NULL;
380 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
381 while (pFree)
382 {
383 ASSERT_CHUNK_FREE(pHeap, pFree);
384 if (pFree->cb >= cb)
385 {
386 unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
387 if (offAlign)
388 offAlign = uAlignment - offAlign;
389 if (!offAlign || pFree->cb - offAlign >= cb)
390 {
391 Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
392
393 /*
394 * Adjust the node in front.
395 * Because of multiple alignments we need to special case allocation of the first block.
396 */
397 if (offAlign)
398 {
399 MMHYPERCHUNKFREE Free = *pFree;
400 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
401 {
402 /* just add a bit of memory to it. */
403 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
404 pPrev->core.offNext += offAlign;
405 AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
406 Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
407 }
408 else
409 {
410 /* make new head node, mark it USED for simplisity. */
411 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
412 Assert(pPrev == &pFree->core);
413 pPrev->offPrev = 0;
414 MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
415 pPrev->offNext = offAlign;
416 Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
417
418 }
419 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
420 pHeap->cbFree -= offAlign;
421
422 /* Recreate pFree node and adjusting everything... */
423 pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
424 *pFree = Free;
425
426 pFree->cb -= offAlign;
427 if (pFree->core.offNext)
428 {
429 pFree->core.offNext -= offAlign;
430 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
431 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
432 ASSERT_CHUNK(pHeap, pNext);
433 }
434 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
435 MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
436
437 if (pFree->offNext)
438 {
439 pFree->offNext -= offAlign;
440 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
441 pNext->offPrev = -(int32_t)pFree->offNext;
442 ASSERT_CHUNK_FREE(pHeap, pNext);
443 }
444 else
445 pHeap->offFreeTail += offAlign;
446 if (pFree->offPrev)
447 {
448 pFree->offPrev -= offAlign;
449 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
450 pPrev->offNext = -pFree->offPrev;
451 ASSERT_CHUNK_FREE(pHeap, pPrev);
452 }
453 else
454 pHeap->offFreeHead += offAlign;
455 pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
456 pFree->core.offStat = 0;
457 ASSERT_CHUNK_FREE(pHeap, pFree);
458 Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
459 }
460
461 /*
462 * Split off a new FREE chunk?
463 */
464 if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
465 {
466 /*
467 * Move the FREE chunk up to make room for the new USED chunk.
468 */
469 const int off = cb + sizeof(MMHYPERCHUNK);
470 PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
471 *pNew = *pFree;
472 pNew->cb -= off;
473 if (pNew->core.offNext)
474 {
475 pNew->core.offNext -= off;
476 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
477 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
478 ASSERT_CHUNK(pHeap, pNext);
479 }
480 pNew->core.offPrev = -off;
481 MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
482
483 if (pNew->offNext)
484 {
485 pNew->offNext -= off;
486 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
487 pNext->offPrev = -(int32_t)pNew->offNext;
488 ASSERT_CHUNK_FREE(pHeap, pNext);
489 }
490 else
491 pHeap->offFreeTail += off;
492 if (pNew->offPrev)
493 {
494 pNew->offPrev -= off;
495 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
496 pPrev->offNext = -pNew->offPrev;
497 ASSERT_CHUNK_FREE(pHeap, pPrev);
498 }
499 else
500 pHeap->offFreeHead += off;
501 pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
502 pNew->core.offStat = 0;
503 ASSERT_CHUNK_FREE(pHeap, pNew);
504
505 /*
506 * Update the old FREE node making it a USED node.
507 */
508 pFree->core.offNext = off;
509 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
510
511
512 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
513 pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
514 pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
515 pRet = &pFree->core;
516 ASSERT_CHUNK(pHeap, &pFree->core);
517 Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
518 }
519 else
520 {
521 /*
522 * Link out of free list.
523 */
524 if (pFree->offNext)
525 {
526 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
527 if (pFree->offPrev)
528 {
529 pNext->offPrev += pFree->offPrev;
530 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
531 pPrev->offNext += pFree->offNext;
532 ASSERT_CHUNK_FREE(pHeap, pPrev);
533 }
534 else
535 {
536 pHeap->offFreeHead += pFree->offNext;
537 pNext->offPrev = 0;
538 }
539 ASSERT_CHUNK_FREE(pHeap, pNext);
540 }
541 else
542 {
543 if (pFree->offPrev)
544 {
545 pHeap->offFreeTail += pFree->offPrev;
546 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
547 pPrev->offNext = 0;
548 ASSERT_CHUNK_FREE(pHeap, pPrev);
549 }
550 else
551 {
552 pHeap->offFreeHead = NIL_OFFSET;
553 pHeap->offFreeTail = NIL_OFFSET;
554 }
555 }
556
557 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
558 pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
559 pHeap->cbFree -= pFree->cb;
560 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
561 pRet = &pFree->core;
562 ASSERT_CHUNK(pHeap, &pFree->core);
563 Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
564 }
565 Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
566 break;
567 }
568 }
569
570 /* next */
571 pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
572 }
573
574#ifdef MMHYPER_HEAP_STRICT_FENCE
575 uint32_t *pu32End = (uint32_t *)((uint8_t *)(pRet + 1) + cb);
576 uint32_t *pu32EndReal = pRet->offNext
577 ? (uint32_t *)((uint8_t *)pRet + pRet->offNext)
578 : (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
579 cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
580 ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
581 pu32EndReal[-1] = cbFence;
582#endif
583#ifdef MMHYPER_HEAP_STRICT
584 mmHyperHeapCheck(pHeap);
585#endif
586 return pRet;
587}
588
589
590/**
591 * Allocates one or more pages of memory from the specified heap.
592 * The caller validates the parameters of this request.
593 *
594 * @returns Pointer to the allocated chunk.
595 * @returns NULL on failure.
596 * @param pHeap The heap.
597 * @param cb Size of the memory block to allocate.
598 * @internal
599 */
600static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
601{
602 Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
603
604#ifdef MMHYPER_HEAP_STRICT
605 mmHyperHeapCheck(pHeap);
606#endif
607
608 /*
609 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
610 */
611 if (pHeap->offFreeHead == NIL_OFFSET)
612 return NULL;
613
614 /*
615 * Page aligned chunks.
616 *
617 * Page aligned chunks can only be allocated from the last FREE chunk.
618 * This is for reasons of simplicity and fragmentation. Page aligned memory
619 * must also be allocated in page aligned sizes. Page aligned memory cannot
620 * be freed either.
621 *
622 * So, for this to work, the last FREE chunk needs to end on a page aligned
623 * boundrary.
624 */
625 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
626 ASSERT_CHUNK_FREE(pHeap, pFree);
627 if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
628 || pFree->cb + sizeof(MMHYPERCHUNK) < cb)
629 {
630 Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
631 return NULL;
632 }
633
634 void *pvRet;
635 if (pFree->cb > cb)
636 {
637 /*
638 * Simple, just cut the top of the free node and return it.
639 */
640 pFree->cb -= cb;
641 pvRet = (char *)(&pFree->core + 1) + pFree->cb;
642 AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
643 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
644 pHeap->cbFree -= cb;
645 ASSERT_CHUNK_FREE(pHeap, pFree);
646 Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
647 }
648 else
649 {
650 /*
651 * Unlink the FREE node.
652 */
653 pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
654 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
655 pHeap->cbFree -= pFree->cb;
656
657 /* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
658 if (pvRet != (void *)pFree)
659 {
660 AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the *beep* did someone manage to allocated up all the heap with page aligned memory?!?\n"));
661 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
662 pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
663 AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
664#ifdef VBOX_WITH_STATISTICS
665 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
666 pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
667 pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
668#endif
669 Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
670 }
671
672 /* unlink from FREE chain. */
673 if (pFree->offPrev)
674 {
675 pHeap->offFreeTail += pFree->offPrev;
676 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
677 }
678 else
679 {
680 pHeap->offFreeTail = NIL_OFFSET;
681 pHeap->offFreeHead = NIL_OFFSET;
682 }
683 Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
684 }
685 pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
686 Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
687
688#ifdef MMHYPER_HEAP_STRICT
689 mmHyperHeapCheck(pHeap);
690#endif
691 return pvRet;
692}
693
694#ifdef VBOX_WITH_STATISTICS
695
696/**
697 * Get the statistic record for a tag.
698 *
699 * @returns Pointer to a stat record.
700 * @returns NULL on failure.
701 * @param pHeap The heap.
702 * @param enmTag The tag.
703 */
704static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
705{
706 /* try look it up first. */
707 PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
708 if (!pStat)
709 {
710 /* try allocate a new one */
711 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
712 if (!pChunk)
713 return NULL;
714 pStat = (PMMHYPERSTAT)(pChunk + 1);
715 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
716
717 ASMMemZero32(pStat, sizeof(*pStat));
718 pStat->Core.Key = enmTag;
719 RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
720 }
721 if (!pStat->fRegistered)
722 {
723# ifdef IN_RING3
724 mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
725# else
726 /** @todo schedule a R3 action. */
727# endif
728 }
729 return pStat;
730}
731
732
733# ifdef IN_RING3
734/**
735 * Registers statistics with STAM.
736 *
737 */
738static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
739{
740 if (pStat->fRegistered)
741 return;
742 const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
743 STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
744 STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
745 STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
746 STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
747 STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
748 STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
749 STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
750 pStat->fRegistered = true;
751}
752# endif /* IN_RING3 */
753
754#endif /* VBOX_WITH_STATISTICS */
755
756
757/**
758 * Free memory allocated using MMHyperAlloc().
759 * The caller validates the parameters of this request.
760 *
761 * @returns VBox status code.
762 * @param pVM The VM to operate on.
763 * @param pv The memory to free.
764 * @remark Try avoid free hyper memory.
765 */
766static int mmHyperFreeInternal(PVM pVM, void *pv)
767{
768 Log2(("MMHyperFree: pv=%p\n", pv));
769 if (!pv)
770 return VINF_SUCCESS;
771 AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
772 ("Invalid pointer %p!\n", pv),
773 VERR_INVALID_POINTER);
774
775 /*
776 * Get the heap and stats.
777 * Validate the chunk at the same time.
778 */
779 PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
780
781 AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
782 || RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
783 ("%p: offNext=%#RX32\n", pv, pChunk->offNext),
784 VERR_INVALID_POINTER);
785
786 AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
787 ("%p: Not used!\n", pv),
788 VERR_INVALID_POINTER);
789
790 int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
791 AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
792 && !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
793 ("%p: offPrev=%#RX32!\n", pv, offPrev),
794 VERR_INVALID_POINTER);
795
796 /* statistics */
797#ifdef VBOX_WITH_STATISTICS
798 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
799 AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
800 && pChunk->offStat,
801 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
802 VERR_INVALID_POINTER);
803#else
804 AssertMsgReturn(!pChunk->offStat,
805 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
806 VERR_INVALID_POINTER);
807#endif
808
809 /* The heap structure. */
810 PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
811 AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
812 && pChunk->offHeap,
813 ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
814 VERR_INVALID_POINTER);
815
816 AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
817 ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
818 VERR_INVALID_POINTER);
819 Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
820
821 /* Some more verifications using additional info from pHeap. */
822 AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
823 ("%p: offPrev=%#RX32!\n", pv, offPrev),
824 VERR_INVALID_POINTER);
825
826 AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
827 ("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
828 VERR_INVALID_POINTER);
829
830 AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
831 ("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
832 (char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
833 VERR_INVALID_POINTER);
834
835#ifdef MMHYPER_HEAP_STRICT
836 mmHyperHeapCheck(pHeap);
837#endif
838
839#if defined(VBOX_WITH_STATISTICS) || defined(MMHYPER_HEAP_FREE_POISON)
840 /* calc block size. */
841 const uint32_t cbChunk = pChunk->offNext
842 ? pChunk->offNext
843 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
844#endif
845#ifdef MMHYPER_HEAP_FREE_POISON
846 /* poison the block */
847 memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
848#endif
849
850#ifdef MMHYPER_HEAP_FREE_DELAY
851# ifdef MMHYPER_HEAP_FREE_POISON
852 /*
853 * Check poison.
854 */
855 unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
856 while (i-- > 0)
857 if (pHeap->aDelayedFrees[i].offChunk)
858 {
859 PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
860 const size_t cb = pCur->offNext
861 ? pCur->offNext - sizeof(*pCur)
862 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
863 uint8_t *pab = (uint8_t *)(pCur + 1);
864 for (unsigned off = 0; off < cb; off++)
865 AssertReleaseMsg(pab[off] == 0xCB,
866 ("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
867 pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
868 }
869# endif /* MMHYPER_HEAP_FREE_POISON */
870
871 /*
872 * Delayed freeing.
873 */
874 int rc = VINF_SUCCESS;
875 if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
876 {
877 PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
878 rc = mmHyperFree(pHeap, pChunkFree);
879 }
880 pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
881 pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
882 pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
883
884#else /* !MMHYPER_HEAP_FREE_POISON */
885 /*
886 * Call the worker.
887 */
888 int rc = mmHyperFree(pHeap, pChunk);
889#endif /* !MMHYPER_HEAP_FREE_POISON */
890
891 /*
892 * Update statistics.
893 */
894#ifdef VBOX_WITH_STATISTICS
895 pStat->cFrees++;
896 if (RT_SUCCESS(rc))
897 {
898 pStat->cbFreed += cbChunk;
899 pStat->cbCurAllocated -= cbChunk;
900 }
901 else
902 pStat->cFailures++;
903#endif
904
905 return rc;
906}
907
908
909/**
910 * Wrapper for mmHyperFreeInternal
911 */
912VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
913{
914 int rc;
915
916 rc = mmHyperLock(pVM);
917 AssertRCReturn(rc, rc);
918
919 LogFlow(("MMHyperFree %p\n", pv));
920
921 rc = mmHyperFreeInternal(pVM, pv);
922
923 mmHyperUnlock(pVM);
924 return rc;
925}
926
927
928/**
929 * Free memory a memory chunk.
930 *
931 * @returns VBox status code.
932 * @param pHeap The heap.
933 * @param pChunk The memory chunk to free.
934 */
935static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
936{
937 Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
938 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
939
940 /*
941 * Insert into the free list (which is sorted on address).
942 *
943 * We'll search towards the end of the heap to locate the
944 * closest FREE chunk.
945 */
946 PMMHYPERCHUNKFREE pLeft = NULL;
947 PMMHYPERCHUNKFREE pRight = NULL;
948 if (pHeap->offFreeTail != NIL_OFFSET)
949 {
950 if (pFree->core.offNext)
951 {
952 pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
953 ASSERT_CHUNK(pHeap, &pRight->core);
954 while (!MMHYPERCHUNK_ISFREE(&pRight->core))
955 {
956 if (!pRight->core.offNext)
957 {
958 pRight = NULL;
959 break;
960 }
961 pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
962 ASSERT_CHUNK(pHeap, &pRight->core);
963 }
964 }
965 if (!pRight)
966 pRight = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
967 if (pRight)
968 {
969 ASSERT_CHUNK_FREE(pHeap, pRight);
970 if (pRight->offPrev)
971 {
972 pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
973 ASSERT_CHUNK_FREE(pHeap, pLeft);
974 }
975 }
976 }
977 if (pLeft == pFree)
978 {
979 AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
980 return VERR_INVALID_POINTER;
981 }
982 pChunk->offStat = 0;
983
984 /*
985 * Head free chunk list?
986 */
987 if (!pLeft)
988 {
989 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
990 pFree->offPrev = 0;
991 pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
992 if (pRight)
993 {
994 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
995 pRight->offPrev = -(int32_t)pFree->offNext;
996 }
997 else
998 {
999 pFree->offNext = 0;
1000 pHeap->offFreeTail = pHeap->offFreeHead;
1001 }
1002 Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
1003 }
1004 else
1005 {
1006 /*
1007 * Can we merge with left hand free chunk?
1008 */
1009 if ((char *)pLeft + pLeft->core.offNext == (char *)pFree)
1010 {
1011 if (pFree->core.offNext)
1012 {
1013 pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
1014 MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
1015 }
1016 else
1017 pLeft->core.offNext = 0;
1018 pFree = pLeft;
1019 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1020 pHeap->cbFree -= pLeft->cb;
1021 Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
1022 }
1023 /*
1024 * No, just link it into the free list then.
1025 */
1026 else
1027 {
1028 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1029 pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
1030 pLeft->offNext = -pFree->offPrev;
1031 if (pRight)
1032 {
1033 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1034 pRight->offPrev = -(int32_t)pFree->offNext;
1035 }
1036 else
1037 {
1038 pFree->offNext = 0;
1039 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1040 }
1041 Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
1042 }
1043 }
1044
1045 /*
1046 * Can we merge with right hand free chunk?
1047 */
1048 if (pRight && (char *)pRight == (char *)pFree + pFree->core.offNext)
1049 {
1050 /* core */
1051 if (pRight->core.offNext)
1052 {
1053 pFree->core.offNext += pRight->core.offNext;
1054 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
1055 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
1056 ASSERT_CHUNK(pHeap, pNext);
1057 }
1058 else
1059 pFree->core.offNext = 0;
1060
1061 /* free */
1062 if (pRight->offNext)
1063 {
1064 pFree->offNext += pRight->offNext;
1065 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1066 }
1067 else
1068 {
1069 pFree->offNext = 0;
1070 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1071 }
1072 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1073 pHeap->cbFree -= pRight->cb;
1074 Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1075 }
1076
1077 /* calculate the size. */
1078 if (pFree->core.offNext)
1079 pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1080 else
1081 pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1082 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1083 pHeap->cbFree += pFree->cb;
1084 ASSERT_CHUNK_FREE(pHeap, pFree);
1085
1086#ifdef MMHYPER_HEAP_STRICT
1087 mmHyperHeapCheck(pHeap);
1088#endif
1089 return VINF_SUCCESS;
1090}
1091
1092
1093#if defined(DEBUG) || defined(MMHYPER_HEAP_STRICT)
1094/**
1095 * Dumps a heap chunk to the log.
1096 *
1097 * @param pHeap Pointer to the heap.
1098 * @param pCur Pointer to the chunk.
1099 */
1100static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1101{
1102 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1103 {
1104 if (pCur->core.offStat)
1105 {
1106 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1107 const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1108#ifdef IN_RING3
1109 Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1110 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1111 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1112 mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
1113#else
1114 Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1115 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1116 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1117 (MMTAG)pStat->Core.Key, pszSelf));
1118#endif
1119 }
1120 else
1121 Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1122 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1123 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1124 }
1125 else
1126 Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1127 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1128 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1129}
1130#endif /* DEBUG || MMHYPER_HEAP_STRICT */
1131
1132
1133#ifdef MMHYPER_HEAP_STRICT
1134/**
1135 * Internal consitency check.
1136 */
1137static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1138{
1139 PMMHYPERCHUNKFREE pPrev = NULL;
1140 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1141 for (;;)
1142 {
1143 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1144 ASSERT_CHUNK_USED(pHeap, &pCur->core);
1145 else
1146 ASSERT_CHUNK_FREE(pHeap, pCur);
1147 if (pPrev)
1148 AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1149 ("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1150
1151# ifdef MMHYPER_HEAP_STRICT_FENCE
1152 uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1153 if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1154 && off < pHeap->offPageAligned)
1155 {
1156 uint32_t cbCur = pCur->core.offNext
1157 ? pCur->core.offNext
1158 : pHeap->cbHeap - off;
1159 uint32_t *pu32End = ((uint32_t *)((uint8_t *)pCur + cbCur));
1160 uint32_t cbFence = pu32End[-1];
1161 if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1162 || cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1163 {
1164 mmHyperHeapDumpOne(pHeap, pCur);
1165 Assert(cbFence < cbCur - sizeof(*pCur));
1166 Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1167 }
1168
1169 uint32_t *pu32Bad = ASMMemIsAllU32((uint8_t *)pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1170 if (RT_UNLIKELY(pu32Bad))
1171 {
1172 mmHyperHeapDumpOne(pHeap, pCur);
1173 Assert(!pu32Bad);
1174 }
1175 }
1176# endif
1177
1178 /* next */
1179 if (!pCur->core.offNext)
1180 break;
1181 pPrev = pCur;
1182 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1183 }
1184}
1185#endif
1186
1187
1188/**
1189 * Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1190 * defined at build time.
1191 *
1192 * @param pVM Pointer to the shared VM structure.
1193 */
1194VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1195{
1196#ifdef MMHYPER_HEAP_STRICT
1197 int rc;
1198
1199 rc = mmHyperLock(pVM);
1200 AssertRC(rc);
1201 mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1202 mmHyperUnlock(pVM);
1203#endif
1204}
1205
1206
1207#ifdef DEBUG
1208/**
1209 * Dumps the hypervisor heap to Log.
1210 * @param pVM VM Handle.
1211 */
1212VMMDECL(void) MMHyperHeapDump(PVM pVM)
1213{
1214 Log(("MMHyperHeapDump: *** heap dump - start ***\n"));
1215 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1216 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1217 for (;;)
1218 {
1219 mmHyperHeapDumpOne(pHeap, pCur);
1220
1221 /* next */
1222 if (!pCur->core.offNext)
1223 break;
1224 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1225 }
1226 Log(("MMHyperHeapDump: *** heap dump - end ***\n"));
1227}
1228#endif
1229
1230
1231/**
1232 * Query the amount of free memory in the hypervisor heap.
1233 *
1234 * @returns Number of free bytes in the hypervisor heap.
1235 */
1236VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1237{
1238 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1239}
1240
1241/**
1242 * Query the size the hypervisor heap.
1243 *
1244 * @returns The size of the hypervisor heap in bytes.
1245 */
1246VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1247{
1248 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1249}
1250
1251
1252/**
1253 * Query the address and size the hypervisor memory area.
1254 *
1255 * @returns Base address of the hypervisor area.
1256 * @param pVM VM Handle.
1257 * @param pcb Where to store the size of the hypervisor area. (out)
1258 */
1259VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1260{
1261 if (pcb)
1262 *pcb = pVM->mm.s.cbHyperArea;
1263 return pVM->mm.s.pvHyperAreaGC;
1264}
1265
1266
1267/**
1268 * Checks if an address is within the hypervisor memory area.
1269 *
1270 * @returns true if inside.
1271 * @returns false if outside.
1272 * @param pVM VM handle.
1273 * @param GCPtr The pointer to check.
1274 */
1275VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1276{
1277 return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1278}
1279
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette