alloc-ef-r0drv.cpp@ 58269

Last change on this file since 58269 was 58269, checked in by vboxsync, 9 years ago
iprt/r0drv: untested electrical heap alloc functions.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 28.3 KB

Line
1	/* $Id: alloc-ef-r0drv.cpp 58269 2015-10-15 18:29:21Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Allocation, electric fence for ring-0 drivers.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "internal/iprt.h"
32	#include <iprt/mem.h>
33	#include <iprt/memobj.h>
34	#include <iprt/log.h>
35	#include <iprt/asm.h>
36	#include <iprt/thread.h>
37	#include <VBox/sup.h>
38	#include <iprt/err.h>
39
40	#include <iprt/alloc.h>
41	#include <iprt/assert.h>
42	#include <iprt/param.h>
43	#include <iprt/string.h>
44
45
46
47	/*******************************************************************************
48	* Defined Constants And Macros *
49	*******************************************************************************/
50	#if defined(DOXYGEN_RUNNING)
51	# define RTR0MEM_EF_IN_FRONT
52	#endif
53
54	/** @def RTR0MEM_EF_SIZE
55	* The size of the fence. This must be page aligned.
56	*/
57	#define RTR0MEM_EF_SIZE PAGE_SIZE
58
59	/** @def RTR0MEM_EF_ALIGNMENT
60	* The allocation alignment, power of two of course.
61	*
62	* Use this for working around misaligned sizes, usually stemming from
63	* allocating a string or something after the main structure. When you
64	* encounter this, please fix the allocation to RTMemAllocVar or RTMemAllocZVar.
65	*/
66	#if 0
67	# define RTR0MEM_EF_ALIGNMENT (ARCH_BITS / 8)
68	#else
69	# define RTR0MEM_EF_ALIGNMENT 1
70	#endif
71
72	/** @def RTR0MEM_EF_IN_FRONT
73	* Define this to put the fence up in front of the block.
74	* The default (when this isn't defined) is to up it up after the block.
75	*/
76	//# define RTR0MEM_EF_IN_FRONT
77
78	/** @def RTR0MEM_EF_FREE_DELAYED
79	* This define will enable free() delay and protection of the freed data
80	* while it's being delayed. The value of RTR0MEM_EF_FREE_DELAYED defines
81	* the threshold of the delayed blocks.
82	* Delayed blocks does not consume any physical memory, only virtual address space.
83	*/
84	#define RTR0MEM_EF_FREE_DELAYED (20 * _1M)
85
86	/** @def RTR0MEM_EF_FREE_FILL
87	* This define will enable memset(,RTR0MEM_EF_FREE_FILL,)'ing the user memory
88	* in the block before freeing/decommitting it. This is useful in GDB since GDB
89	* appears to be able to read the content of the page even after it's been
90	* decommitted.
91	*/
92	#define RTR0MEM_EF_FREE_FILL 'f'
93
94	/** @def RTR0MEM_EF_FILLER
95	* This define will enable memset(,RTR0MEM_EF_FILLER,)'ing the allocated
96	* memory when the API doesn't require it to be zero'd.
97	*/
98	#define RTR0MEM_EF_FILLER 0xef
99
100	/** @def RTR0MEM_EF_NOMAN_FILLER
101	* This define will enable memset(,RTR0MEM_EF_NOMAN_FILLER,)'ing the
102	* unprotected but not allocated area of memory, the so called no man's land.
103	*/
104	#define RTR0MEM_EF_NOMAN_FILLER 0xaa
105
106	/** @def RTR0MEM_EF_FENCE_FILLER
107	* This define will enable memset(,RTR0MEM_EF_FENCE_FILLER,)'ing the
108	* fence itself, as debuggers can usually read them.
109	*/
110	#define RTR0MEM_EF_FENCE_FILLER 0xcc
111
112
113	/*******************************************************************************
114	* Header Files *
115	*******************************************************************************/
116	#ifdef RT_OS_WINDOWS
117	# include <Windows.h>
118	#else
119	# include <sys/mman.h>
120	#endif
121	#include <iprt/avl.h>
122	#include <iprt/thread.h>
123
124
125	/*******************************************************************************
126	* Structures and Typedefs *
127	*******************************************************************************/
128	/**
129	* Allocation types.
130	*/
131	typedef enum RTMEMTYPE
132	{
133	RTMEMTYPE_RTMEMALLOC,
134	RTMEMTYPE_RTMEMALLOCZ,
135	RTMEMTYPE_RTMEMREALLOC,
136	RTMEMTYPE_RTMEMFREE,
137
138	RTMEMTYPE_NEW,
139	RTMEMTYPE_NEW_ARRAY,
140	RTMEMTYPE_DELETE,
141	RTMEMTYPE_DELETE_ARRAY
142	} RTMEMTYPE;
143
144	/**
145	* Node tracking a memory allocation.
146	*/
147	typedef struct RTR0MEMEFBLOCK
148	{
149	/** Avl node code, key is the user block pointer. */
150	AVLPVNODECORE Core;
151	/** Allocation type. */
152	RTMEMTYPE enmType;
153	/** The memory object. */
154	RTR0MEMOBJ hMemObj;
155	/** The unaligned size of the block. */
156	size_t cbUnaligned;
157	/** The aligned size of the block. */
158	size_t cbAligned;
159	/** The allocation tag (read-only string). */
160	const char *pszTag;
161	/** The return address of the allocator function. */
162	void *pvCaller;
163	/** Line number of the alloc call. */
164	unsigned iLine;
165	/** File from within the allocation was made. */
166	const char *pszFile;
167	/** Function from within the allocation was made. */
168	const char *pszFunction;
169	} RTR0MEMEFBLOCK, *PRTR0MEMEFBLOCK;
170
171
172
173	/*********************************************************************************************************************************
174	* Global Variables *
175	*********************************************************************************************************************************/
176	/** Spinlock protecting the all the block's globals. */
177	static volatile uint32_t g_BlocksLock;
178	/** Tree tracking the allocations. */
179	static AVLPVTREE g_BlocksTree;
180
181	#ifdef RTR0MEM_EF_FREE_DELAYED
182	/** Tail of the delayed blocks. */
183	static volatile PRTR0MEMEFBLOCK g_pBlocksDelayHead;
184	/** Tail of the delayed blocks. */
185	static volatile PRTR0MEMEFBLOCK g_pBlocksDelayTail;
186	/** Number of bytes in the delay list (includes fences). */
187	static volatile size_t g_cbBlocksDelay;
188	#endif /* RTR0MEM_EF_FREE_DELAYED */
189
190	/** Array of pointers free watches for. */
191	void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
192	/** Enable logging of all freed memory. */
193	bool gfRTMemFreeLog = false;
194
195
196	/*********************************************************************************************************************************
197	* Internal Functions *
198	*********************************************************************************************************************************/
199
200
201	/**
202	* @callback_method_impl{FNRTSTROUTPUT}
203	*/
204	static DECLCALLBACK(size_t) rtR0MemEfWrite(void pvArg, const char pachChars, size_t cbChars)
205	{
206	if (cbChars)
207	{
208	RTLogWriteDebugger(pachChars, cbChars);
209	RTLogWriteStdOut(pachChars, cbChars);
210	RTLogWriteUser(pachChars, cbChars);
211	}
212	return cbChars;
213	}
214
215
216	/**
217	* Complains about something.
218	*/
219	static void rtR0MemComplain(const char pszOp, const char pszFormat, ...)
220	{
221	va_list args;
222	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem error: %s: ", pszOp);
223	va_start(args, pszFormat);
224	RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
225	va_end(args);
226	RTAssertDoPanic();
227	}
228
229	/**
230	* Log an event.
231	*/
232	DECLINLINE(void) rtR0MemLog(const char pszOp, const char pszFormat, ...)
233	{
234	#if 0
235	va_list args;
236	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem info: %s: ", pszOp);
237	va_start(args, pszFormat);
238	RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
239	va_end(args);
240	#else
241	NOREF(pszOp); NOREF(pszFormat);
242	#endif
243	}
244
245
246
247	/**
248	* Acquires the lock.
249	*/
250	DECLINLINE(RTCCUINTREG) rtR0MemBlockLock(void)
251	{
252	RTCCUINTREG uRet;
253	unsigned c = 0;
254	if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
255	{
256	for (;;)
257	{
258	uRet = ASMIntDisableFlags();
259	if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
260	break;
261	ASMSetFlags(uRet);
262	RTThreadSleepNoLog(((++c) >> 2) & 31);
263	}
264	}
265	else
266	{
267	for (;;)
268	{
269	uRet = ASMIntDisableFlags();
270	if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
271	break;
272	ASMSetFlags(uRet);
273	ASMNopPause();
274	if (++c & 3)
275	ASMNopPause();
276	}
277	}
278	return uRet;
279	}
280
281
282	/**
283	* Releases the lock.
284	*/
285	DECLINLINE(void) rtR0MemBlockUnlock(RTCCUINTREG fSavedIntFlags)
286	{
287	Assert(g_BlocksLock == 1);
288	ASMAtomicXchgU32(&g_BlocksLock, 0);
289	ASMSetFlags(fSavedIntFlags);
290	}
291
292
293	/**
294	* Creates a block.
295	*/
296	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
297	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
298	{
299	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTMemAlloc(sizeof(*pBlock));
300	if (pBlock)
301	{
302	pBlock->enmType = enmType;
303	pBlock->cbUnaligned = cbUnaligned;
304	pBlock->cbAligned = cbAligned;
305	pBlock->pszTag = pszTag;
306	pBlock->pvCaller = pvCaller;
307	pBlock->iLine = iLine;
308	pBlock->pszFile = pszFile;
309	pBlock->pszFunction = pszFunction;
310	}
311	return pBlock;
312	}
313
314
315	/**
316	* Frees a block.
317	*/
318	DECLINLINE(void) rtR0MemBlockFree(PRTR0MEMEFBLOCK pBlock)
319	{
320	RTMemFree(pBlock);
321	}
322
323
324	/**
325	* Insert a block from the tree.
326	*/
327	DECLINLINE(void) rtR0MemBlockInsert(PRTR0MEMEFBLOCK pBlock, void *pv, RTR0MEMOBJ hMemObj)
328	{
329	pBlock->Core.Key = pv;
330	pBlock->hMemObj = hMemObj;
331	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
332	bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
333	rtR0MemBlockUnlock(fSavedIntFlags);
334	AssertRelease(fRc);
335	}
336
337
338	/**
339	* Remove a block from the tree and returns it to the caller.
340	*/
341	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockRemove(void *pv)
342	{
343	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
344	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
345	rtR0MemBlockUnlock(fSavedIntFlags);
346	return pBlock;
347	}
348
349
350	/**
351	* Gets a block.
352	*/
353	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockGet(void *pv)
354	{
355	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
356	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
357	rtR0MemBlockUnlock(fSavedIntFlags);
358	return pBlock;
359	}
360
361
362	/**
363	* Dumps one allocation.
364	*/
365	static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
366	{
367	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
368	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "%p %08lx(+%02lx) %p\n",
369	pBlock->Core.Key,
370	(unsigned long)pBlock->cbUnaligned,
371	(unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
372	pBlock->pvCaller);
373	NOREF(pvUser);
374	return 0;
375	}
376
377
378	/**
379	* Dumps the allocated blocks.
380	* This is something which you should call from gdb.
381	*/
382	extern "C" void RTMemDump(void);
383	void RTMemDump(void)
384	{
385	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "address size(alg) caller\n");
386	RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
387	}
388
389	#ifdef RTR0MEM_EF_FREE_DELAYED
390
391	/**
392	* Insert a delayed block.
393	*/
394	DECLINLINE(void) rtR0MemBlockDelayInsert(PRTR0MEMEFBLOCK pBlock)
395	{
396	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
397	pBlock->Core.pRight = NULL;
398	pBlock->Core.pLeft = NULL;
399	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
400	if (g_pBlocksDelayHead)
401	{
402	g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock;
403	pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead;
404	g_pBlocksDelayHead = pBlock;
405	}
406	else
407	{
408	g_pBlocksDelayTail = pBlock;
409	g_pBlocksDelayHead = pBlock;
410	}
411	g_cbBlocksDelay += cbBlock;
412	rtR0MemBlockUnlock(fSavedIntFlags);
413	}
414
415	/**
416	* Removes a delayed block.
417	*/
418	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockDelayRemove(void)
419	{
420	PRTR0MEMEFBLOCK pBlock = NULL;
421	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
422	if (g_cbBlocksDelay > RTR0MEM_EF_FREE_DELAYED)
423	{
424	pBlock = g_pBlocksDelayTail;
425	if (pBlock)
426	{
427	g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
428	if (pBlock->Core.pLeft)
429	pBlock->Core.pLeft->pRight = NULL;
430	else
431	g_pBlocksDelayHead = NULL;
432	g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
433	}
434	}
435	rtR0MemBlockUnlock(fSavedIntFlags);
436	return pBlock;
437	}
438
439	#endif /* RTR0MEM_EF_FREE_DELAYED */
440
441
442	static void rtR0MemFreeBlock(PRTR0MEMEFBLOCK pBlock, const char *pszOp)
443	{
444	void *pv = pBlock->Core.Key;
445	# ifdef RTR0MEM_EF_IN_FRONT
446	void pvBlock = (char )pv - RTR0MEM_EF_SIZE;
447	# else
448	void pvBlock = (void )((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
449	# endif
450	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
451
452	int rc = RTR0MemObjProtect(pBlock->hMemObj, 0 /offSub/, cbBlock, RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
453	if (RT_FAILURE(rc))
454	rtR0MemComplain(pszOp, "RTR0MemObjProtect([%p], 0, %#x, RTMEM_PROT_READ \| RTMEM_PROT_WRITE) -> %Rrc\n",
455	pvBlock, cbBlock, rc);
456
457	rc = RTR0MemObjFree(pBlock->hMemObj, true /fFreeMappings/);
458	if (RT_FAILURE(rc))
459	rtR0MemComplain(pszOp, "RTR0MemObjFree([%p LB %#x]) -> %Rrc\n", pvBlock, cbBlock, rc);
460	pBlock->hMemObj = NIL_RTR0MEMOBJ;
461
462	rtR0MemBlockFree(pBlock);
463	}
464
465
466	/**
467	* Initialize call, we shouldn't fail here.
468	*/
469	void rtR0MemEfInit(void)
470	{
471
472	}
473
474	/**
475	* @callback_method_impl{AVLPVCALLBACK}
476	*/
477	static DECLCALLBACK(int) rtR0MemEfDestroyBlock(PAVLPVNODECORE pNode, void *pvUser)
478	{
479	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
480
481	/* Note! pszFile and pszFunction may be invalid at this point. */
482	rtR0MemComplain("rtR0MemEfDestroyBlock", "Leaking %zu bytes at %p (iLine=%u pvCaller=%p)\n",
483	pBlock->cbAligned, pBlock->Core.Key, pBlock->iLine, pBlock->pvCaller);
484
485	rtR0MemFreeBlock(pBlock, "rtR0MemEfDestroyBlock");
486
487	NOREF(pvUser);
488	return VINF_SUCCESS;
489	}
490
491
492	/**
493	* Termination call.
494	*
495	* Will check and free memory.
496	*/
497	void rtR0MemEfTerm(void)
498	{
499	#ifdef RTR0MEM_EF_FREE_DELAYED
500	/*
501	* Release delayed frees.
502	*/
503	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
504	for (;;)
505	{
506	PRTR0MEMEFBLOCK pBlock = g_pBlocksDelayTail;
507	g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
508	if (pBlock->Core.pLeft)
509	pBlock->Core.pLeft->pRight = NULL;
510	else
511	g_pBlocksDelayHead = NULL;
512	rtR0MemBlockUnlock(fSavedIntFlags);
513
514	rtR0MemFreeBlock(pBlock, "rtR0MemEfTerm");
515
516	rtR0MemBlockLock();
517	}
518	g_cbBlocksDelay = 0;
519	rtR0MemBlockUnlock(fSavedIntFlags);
520	#endif
521
522	/*
523	* Complain about leaks. Then release them.
524	*/
525	RTAvlPVDestroy(&g_BlocksTree, rtR0MemEfDestroyBlock, NULL);
526	}
527
528
529	/**
530	* Internal allocator.
531	*/
532	static void * rtR0MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
533	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
534	{
535	/*
536	* Sanity.
537	*/
538	if ( RT_ALIGN_Z(RTR0MEM_EF_SIZE, PAGE_SIZE) != RTR0MEM_EF_SIZE
539	&& RTR0MEM_EF_SIZE <= 0)
540	{
541	rtR0MemComplain(pszOp, "Invalid E-fence size! %#x\n", RTR0MEM_EF_SIZE);
542	return NULL;
543	}
544	if (!cbUnaligned)
545	{
546	#if 1
547	rtR0MemComplain(pszOp, "Request of ZERO bytes allocation!\n");
548	return NULL;
549	#else
550	cbAligned = cbUnaligned = 1;
551	#endif
552	}
553
554	#ifndef RTR0MEM_EF_IN_FRONT
555	/* Alignment decreases fence accuracy, but this is at least partially
556	* counteracted by filling and checking the alignment padding. When the
557	* fence is in front then then no extra alignment is needed. */
558	cbAligned = RT_ALIGN_Z(cbAligned, RTR0MEM_EF_ALIGNMENT);
559	#endif
560
561	/*
562	* Allocate the trace block.
563	*/
564	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
565	if (!pBlock)
566	{
567	rtR0MemComplain(pszOp, "Failed to allocate trace block!\n");
568	return NULL;
569	}
570
571	/*
572	* Allocate a block with page alignment space + the size of the E-fence.
573	*/
574	void *pvBlock = NULL;
575	RTR0MEMOBJ hMemObj;
576	size_t cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
577	int rc = RTR0MemObjAllocPage(&hMemObj, cbBlock, false /fExecutable/);
578	if (RT_SUCCESS(rc))
579	pvBlock = RTR0MemObjAddress(hMemObj);
580	if (pvBlock)
581	{
582	/*
583	* Calc the start of the fence and the user block
584	* and then change the page protection of the fence.
585	*/
586	#ifdef RTR0MEM_EF_IN_FRONT
587	void *pvEFence = pvBlock;
588	void pv = (char )pvEFence + RTR0MEM_EF_SIZE;
589	# ifdef RTR0MEM_EF_NOMAN_FILLER
590	memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbUnaligned);
591	# endif
592	#else
593	void pvEFence = (char )pvBlock + (cbBlock - RTR0MEM_EF_SIZE);
594	void pv = (char )pvEFence - cbAligned;
595	# ifdef RTR0MEM_EF_NOMAN_FILLER
596	memset(pvBlock, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbAligned);
597	memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbAligned - cbUnaligned);
598	# endif
599	#endif
600
601	#ifdef RTR0MEM_EF_FENCE_FILLER
602	memset(pvEFence, RTR0MEM_EF_FENCE_FILLER, RTR0MEM_EF_SIZE);
603	#endif
604	rc = RTR0MemObjProtect(hMemObj, (uint8_t )pvEFence - (uint8_t )pvBlock, RTR0MEM_EF_SIZE, RTMEM_PROT_NONE);
605	if (!rc)
606	{
607	rtR0MemBlockInsert(pBlock, pv, hMemObj);
608	if (enmType == RTMEMTYPE_RTMEMALLOCZ)
609	memset(pv, 0, cbUnaligned);
610	#ifdef RTR0MEM_EF_FILLER
611	else
612	memset(pv, RTR0MEM_EF_FILLER, cbUnaligned);
613	#endif
614
615	rtR0MemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
616	return pv;
617	}
618	rtR0MemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTR0MEM_EF_SIZE, rc);
619	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
620	}
621	else
622	{
623	rtR0MemComplain(pszOp, "Failed to allocated %zu (%zu) bytes (rc=%Rrc).\n", cbBlock, cbUnaligned, rc);
624	if (RT_SUCCESS(rc))
625	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
626	}
627
628	rtR0MemBlockFree(pBlock);
629	return NULL;
630	}
631
632
633	/**
634	* Internal free.
635	*/
636	static void rtR0MemFree(const char pszOp, RTMEMTYPE enmType, void pv, void *pvCaller, RT_SRC_POS_DECL)
637	{
638	NOREF(enmType); RT_SRC_POS_NOREF();
639
640	/*
641	* Simple case.
642	*/
643	if (!pv)
644	return;
645
646	/*
647	* Check watch points.
648	*/
649	for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++)
650	if (gapvRTMemFreeWatch[i] == pv)
651	RTAssertDoPanic();
652
653	/*
654	* Find the block.
655	*/
656	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockRemove(pv);
657	if (pBlock)
658	{
659	if (gfRTMemFreeLog)
660	RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
661
662	#ifdef RTR0MEM_EF_NOMAN_FILLER
663	/*
664	* Check whether the no man's land is untouched.
665	*/
666	# ifdef RTR0MEM_EF_IN_FRONT
667	void pvWrong = ASMMemIsAll8((char )pv + pBlock->cbUnaligned,
668	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned,
669	RTR0MEM_EF_NOMAN_FILLER);
670	# else
671	/* Alignment must match allocation alignment in rtMemAlloc(). */
672	void pvWrong = ASMMemIsAll8((char )pv + pBlock->cbUnaligned,
673	pBlock->cbAligned - pBlock->cbUnaligned,
674	RTR0MEM_EF_NOMAN_FILLER);
675	if (pvWrong)
676	RTAssertDoPanic();
677	pvWrong = ASMMemIsAll8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK),
678	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned,
679	RTR0MEM_EF_NOMAN_FILLER);
680	# endif
681	if (pvWrong)
682	RTAssertDoPanic();
683	#endif
684
685	#ifdef RTR0MEM_EF_FREE_FILL
686	/*
687	* Fill the user part of the block.
688	*/
689	memset(pv, RTR0MEM_EF_FREE_FILL, pBlock->cbUnaligned);
690	#endif
691
692	#if defined(RTR0MEM_EF_FREE_DELAYED) && RTR0MEM_EF_FREE_DELAYED > 0
693	/*
694	* We're doing delayed freeing.
695	* That means we'll expand the E-fence to cover the entire block.
696	*/
697	# ifdef RTR0MEM_EF_IN_FRONT
698	int rc = RTR0MemObjProtect(pBlock->hMemObj, RTR0MEM_EF_SIZE /offSub/, pBlock->cbAligned, RTMEM_PROT_NONE);
699	# else
700	int rc = RTR0MemObjProtect(pBlock->hMemObj, 0 /offSub/, pBlock->cbAligned, RTMEM_PROT_NONE);
701	# endif
702	if (RT_SUCCESS(rc))
703	{
704	/*
705	* Insert it into the free list and process pending frees.
706	*/
707	rtR0MemBlockDelayInsert(pBlock);
708	while ((pBlock = rtR0MemBlockDelayRemove()) != NULL)
709	rtR0MemFreeBlock(pBlock, pszOp);
710	}
711	else
712	rtR0MemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
713
714	#else /* !RTR0MEM_EF_FREE_DELAYED */
715	rtR0MemFreeBlock(pBlock, pszOp);
716	#endif /* !RTR0MEM_EF_FREE_DELAYED */
717	}
718	else
719	rtR0MemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
720	}
721
722
723	/**
724	* Internal realloc.
725	*/
726	static void rtR0MemRealloc(const char pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
727	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
728	{
729	/*
730	* Allocate new and copy.
731	*/
732	if (!pvOld)
733	return rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
734	if (!cbNew)
735	{
736	rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
737	return NULL;
738	}
739
740	/*
741	* Get the block, allocate the new, copy the data, free the old one.
742	*/
743	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockGet(pvOld);
744	if (pBlock)
745	{
746	void *pvRet = rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
747	if (pvRet)
748	{
749	memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
750	rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
751	}
752	return pvRet;
753	}
754	rtR0MemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
755	return NULL;
756	}
757
758
759
760
761	RTDECL(void ) RTMemEfTmpAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
762	{
763	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
764	}
765
766
767	RTDECL(void ) RTMemEfTmpAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
768	{
769	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
770	}
771
772
773	RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
774	{
775	if (pv)
776	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
777	}
778
779
780	RTDECL(void ) RTMemEfAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
781	{
782	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
783	}
784
785
786	RTDECL(void ) RTMemEfAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
787	{
788	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
789	}
790
791
792	RTDECL(void ) RTMemEfAllocVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
793	{
794	size_t cbAligned;
795	if (cbUnaligned >= 16)
796	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
797	else
798	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
799	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
800	}
801
802
803	RTDECL(void ) RTMemEfAllocZVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
804	{
805	size_t cbAligned;
806	if (cbUnaligned >= 16)
807	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
808	else
809	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
810	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
811	}
812
813
814	RTDECL(void ) RTMemEfRealloc(void pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
815	{
816	return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
817	}
818
819
820	RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
821	{
822	if (pv)
823	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
824	}
825
826
827	RTDECL(void ) RTMemEfDup(const void pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
828	{
829	void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS);
830	if (pvDst)
831	memcpy(pvDst, pvSrc, cb);
832	return pvDst;
833	}
834
835
836	RTDECL(void ) RTMemEfDupEx(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
837	{
838	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS);
839	if (pvDst)
840	{
841	memcpy(pvDst, pvSrc, cbSrc);
842	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
843	}
844	return pvDst;
845	}
846
847
848
849
850	/*
851	*
852	* The NP (no position) versions.
853	*
854	*/
855
856
857
858	RTDECL(void ) RTMemEfTmpAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
859	{
860	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
861	}
862
863
864	RTDECL(void ) RTMemEfTmpAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
865	{
866	return rtR0MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
867	}
868
869
870	RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF
871	{
872	if (pv)
873	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
874	}
875
876
877	RTDECL(void ) RTMemEfAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
878	{
879	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
880	}
881
882
883	RTDECL(void ) RTMemEfAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
884	{
885	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
886	}
887
888
889	RTDECL(void ) RTMemEfAllocVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
890	{
891	size_t cbAligned;
892	if (cbUnaligned >= 16)
893	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
894	else
895	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
896	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
897	}
898
899
900	RTDECL(void ) RTMemEfAllocZVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
901	{
902	size_t cbAligned;
903	if (cbUnaligned >= 16)
904	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
905	else
906	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
907	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
908	}
909
910
911	RTDECL(void ) RTMemEfReallocNP(void pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
912	{
913	return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
914	}
915
916
917	RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF
918	{
919	if (pv)
920	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
921	}
922
923
924	RTDECL(void ) RTMemEfDupNP(const void pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF
925	{
926	void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL);
927	if (pvDst)
928	memcpy(pvDst, pvSrc, cb);
929	return pvDst;
930	}
931
932
933	RTDECL(void ) RTMemEfDupExNP(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF
934	{
935	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL);
936	if (pvDst)
937	{
938	memcpy(pvDst, pvSrc, cbSrc);
939	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
940	}
941	return pvDst;
942	}
943

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp@ 58269

Download in other formats: