alloc-ef-r0drv.cpp@ 61572

Last change on this file since 61572 was 60603, checked in by vboxsync, 9 years ago
Assorted compile fixes for FreeBSD
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 28.6 KB

Line
1	/* $Id: alloc-ef-r0drv.cpp 60603 2016-04-20 14:32:19Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Allocation, electric fence for ring-0 drivers.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#define RTMEM_NO_WRAP_TO_EF_APIS
32	#include "internal/iprt.h"
33	#include <iprt/mem.h>
34
35	#include <iprt/alloc.h>
36	#include <iprt/asm.h>
37	#include <iprt/asm-amd64-x86.h>
38	#include <iprt/assert.h>
39	#include <iprt/err.h>
40	#include <iprt/log.h>
41	#include <iprt/memobj.h>
42	#include <iprt/param.h>
43	#include <iprt/string.h>
44	#include <iprt/thread.h>
45
46	#include "internal/mem.h"
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	#if defined(DOXYGEN_RUNNING)
53	# define RTR0MEM_EF_IN_FRONT
54	#endif
55
56	/** @def RTR0MEM_EF_SIZE
57	* The size of the fence. This must be page aligned.
58	*/
59	#define RTR0MEM_EF_SIZE PAGE_SIZE
60
61	/** @def RTR0MEM_EF_ALIGNMENT
62	* The allocation alignment, power of two of course.
63	*
64	* Use this for working around misaligned sizes, usually stemming from
65	* allocating a string or something after the main structure. When you
66	* encounter this, please fix the allocation to RTMemAllocVar or RTMemAllocZVar.
67	*/
68	#if 0
69	# define RTR0MEM_EF_ALIGNMENT (ARCH_BITS / 8)
70	#else
71	# define RTR0MEM_EF_ALIGNMENT 1
72	#endif
73
74	/** @def RTR0MEM_EF_IN_FRONT
75	* Define this to put the fence up in front of the block.
76	* The default (when this isn't defined) is to up it up after the block.
77	*/
78	//# define RTR0MEM_EF_IN_FRONT
79
80	/** @def RTR0MEM_EF_FREE_DELAYED
81	* This define will enable free() delay and protection of the freed data
82	* while it's being delayed. The value of RTR0MEM_EF_FREE_DELAYED defines
83	* the threshold of the delayed blocks.
84	* Delayed blocks does not consume any physical memory, only virtual address space.
85	*/
86	#define RTR0MEM_EF_FREE_DELAYED (20 * _1M)
87
88	/** @def RTR0MEM_EF_FREE_FILL
89	* This define will enable memset(,RTR0MEM_EF_FREE_FILL,)'ing the user memory
90	* in the block before freeing/decommitting it. This is useful in GDB since GDB
91	* appears to be able to read the content of the page even after it's been
92	* decommitted.
93	*/
94	#define RTR0MEM_EF_FREE_FILL 'f'
95
96	/** @def RTR0MEM_EF_FILLER
97	* This define will enable memset(,RTR0MEM_EF_FILLER,)'ing the allocated
98	* memory when the API doesn't require it to be zero'd.
99	*/
100	#define RTR0MEM_EF_FILLER 0xef
101
102	/** @def RTR0MEM_EF_NOMAN_FILLER
103	* This define will enable memset(,RTR0MEM_EF_NOMAN_FILLER,)'ing the
104	* unprotected but not allocated area of memory, the so called no man's land.
105	*/
106	#define RTR0MEM_EF_NOMAN_FILLER 0xaa
107
108	/** @def RTR0MEM_EF_FENCE_FILLER
109	* This define will enable memset(,RTR0MEM_EF_FENCE_FILLER,)'ing the
110	* fence itself, as debuggers can usually read them.
111	*/
112	#define RTR0MEM_EF_FENCE_FILLER 0xcc
113
114
115	/*******************************************************************************
116	* Header Files *
117	*******************************************************************************/
118	#ifdef RT_OS_WINDOWS
119	# include <Windows.h>
120	#elif !defined(RT_OS_FREEBSD)
121	# include <sys/mman.h>
122	#endif
123	#include <iprt/avl.h>
124	#include <iprt/thread.h>
125
126
127	/*******************************************************************************
128	* Structures and Typedefs *
129	*******************************************************************************/
130	/**
131	* Allocation types.
132	*/
133	typedef enum RTMEMTYPE
134	{
135	RTMEMTYPE_RTMEMALLOC,
136	RTMEMTYPE_RTMEMALLOCZ,
137	RTMEMTYPE_RTMEMREALLOC,
138	RTMEMTYPE_RTMEMFREE,
139
140	RTMEMTYPE_NEW,
141	RTMEMTYPE_NEW_ARRAY,
142	RTMEMTYPE_DELETE,
143	RTMEMTYPE_DELETE_ARRAY
144	} RTMEMTYPE;
145
146	/**
147	* Node tracking a memory allocation.
148	*/
149	typedef struct RTR0MEMEFBLOCK
150	{
151	/** Avl node code, key is the user block pointer. */
152	AVLPVNODECORE Core;
153	/** Allocation type. */
154	RTMEMTYPE enmType;
155	/** The memory object. */
156	RTR0MEMOBJ hMemObj;
157	/** The unaligned size of the block. */
158	size_t cbUnaligned;
159	/** The aligned size of the block. */
160	size_t cbAligned;
161	/** The allocation tag (read-only string). */
162	const char *pszTag;
163	/** The return address of the allocator function. */
164	void *pvCaller;
165	/** Line number of the alloc call. */
166	unsigned iLine;
167	/** File from within the allocation was made. */
168	const char *pszFile;
169	/** Function from within the allocation was made. */
170	const char *pszFunction;
171	} RTR0MEMEFBLOCK, *PRTR0MEMEFBLOCK;
172
173
174
175	/*********************************************************************************************************************************
176	* Global Variables *
177	*********************************************************************************************************************************/
178	/** Spinlock protecting the all the block's globals. */
179	static volatile uint32_t g_BlocksLock;
180	/** Tree tracking the allocations. */
181	static AVLPVTREE g_BlocksTree;
182
183	#ifdef RTR0MEM_EF_FREE_DELAYED
184	/** Tail of the delayed blocks. */
185	static volatile PRTR0MEMEFBLOCK g_pBlocksDelayHead;
186	/** Tail of the delayed blocks. */
187	static volatile PRTR0MEMEFBLOCK g_pBlocksDelayTail;
188	/** Number of bytes in the delay list (includes fences). */
189	static volatile size_t g_cbBlocksDelay;
190	#endif /* RTR0MEM_EF_FREE_DELAYED */
191
192	/** Array of pointers free watches for. */
193	void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
194	/** Enable logging of all freed memory. */
195	bool gfRTMemFreeLog = false;
196
197
198	/*********************************************************************************************************************************
199	* Internal Functions *
200	*********************************************************************************************************************************/
201
202
203	/**
204	* @callback_method_impl{FNRTSTROUTPUT}
205	*/
206	static DECLCALLBACK(size_t) rtR0MemEfWrite(void pvArg, const char pachChars, size_t cbChars)
207	{
208	if (cbChars)
209	{
210	RTLogWriteDebugger(pachChars, cbChars);
211	RTLogWriteStdOut(pachChars, cbChars);
212	RTLogWriteUser(pachChars, cbChars);
213	}
214	return cbChars;
215	}
216
217
218	/**
219	* Complains about something.
220	*/
221	static void rtR0MemComplain(const char pszOp, const char pszFormat, ...)
222	{
223	va_list args;
224	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem error: %s: ", pszOp);
225	va_start(args, pszFormat);
226	RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
227	va_end(args);
228	RTAssertDoPanic();
229	}
230
231	/**
232	* Log an event.
233	*/
234	DECLINLINE(void) rtR0MemLog(const char pszOp, const char pszFormat, ...)
235	{
236	#if 0
237	va_list args;
238	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem info: %s: ", pszOp);
239	va_start(args, pszFormat);
240	RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
241	va_end(args);
242	#else
243	NOREF(pszOp); NOREF(pszFormat);
244	#endif
245	}
246
247
248
249	/**
250	* Acquires the lock.
251	*/
252	DECLINLINE(RTCCUINTREG) rtR0MemBlockLock(void)
253	{
254	RTCCUINTREG uRet;
255	unsigned c = 0;
256	if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
257	{
258	for (;;)
259	{
260	uRet = ASMIntDisableFlags();
261	if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
262	break;
263	ASMSetFlags(uRet);
264	RTThreadSleepNoLog(((++c) >> 2) & 31);
265	}
266	}
267	else
268	{
269	for (;;)
270	{
271	uRet = ASMIntDisableFlags();
272	if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
273	break;
274	ASMSetFlags(uRet);
275	ASMNopPause();
276	if (++c & 3)
277	ASMNopPause();
278	}
279	}
280	return uRet;
281	}
282
283
284	/**
285	* Releases the lock.
286	*/
287	DECLINLINE(void) rtR0MemBlockUnlock(RTCCUINTREG fSavedIntFlags)
288	{
289	Assert(g_BlocksLock == 1);
290	ASMAtomicXchgU32(&g_BlocksLock, 0);
291	ASMSetFlags(fSavedIntFlags);
292	}
293
294
295	/**
296	* Creates a block.
297	*/
298	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
299	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
300	{
301	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTMemAlloc(sizeof(*pBlock));
302	if (pBlock)
303	{
304	pBlock->enmType = enmType;
305	pBlock->cbUnaligned = cbUnaligned;
306	pBlock->cbAligned = cbAligned;
307	pBlock->pszTag = pszTag;
308	pBlock->pvCaller = pvCaller;
309	pBlock->iLine = iLine;
310	pBlock->pszFile = pszFile;
311	pBlock->pszFunction = pszFunction;
312	}
313	return pBlock;
314	}
315
316
317	/**
318	* Frees a block.
319	*/
320	DECLINLINE(void) rtR0MemBlockFree(PRTR0MEMEFBLOCK pBlock)
321	{
322	RTMemFree(pBlock);
323	}
324
325
326	/**
327	* Insert a block from the tree.
328	*/
329	DECLINLINE(void) rtR0MemBlockInsert(PRTR0MEMEFBLOCK pBlock, void *pv, RTR0MEMOBJ hMemObj)
330	{
331	pBlock->Core.Key = pv;
332	pBlock->hMemObj = hMemObj;
333	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
334	bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
335	rtR0MemBlockUnlock(fSavedIntFlags);
336	AssertRelease(fRc);
337	}
338
339
340	/**
341	* Remove a block from the tree and returns it to the caller.
342	*/
343	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockRemove(void *pv)
344	{
345	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
346	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
347	rtR0MemBlockUnlock(fSavedIntFlags);
348	return pBlock;
349	}
350
351
352	/**
353	* Gets a block.
354	*/
355	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockGet(void *pv)
356	{
357	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
358	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
359	rtR0MemBlockUnlock(fSavedIntFlags);
360	return pBlock;
361	}
362
363
364	/**
365	* Dumps one allocation.
366	*/
367	static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
368	{
369	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
370	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "%p %08lx(+%02lx) %p\n",
371	pBlock->Core.Key,
372	(unsigned long)pBlock->cbUnaligned,
373	(unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
374	pBlock->pvCaller);
375	NOREF(pvUser);
376	return 0;
377	}
378
379
380	/**
381	* Dumps the allocated blocks.
382	* This is something which you should call from gdb.
383	*/
384	RT_C_DECLS_BEGIN
385	void RTMemDump(void);
386	RT_C_DECLS_END
387
388	void RTMemDump(void)
389	{
390	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "address size(alg) caller\n");
391	RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
392	}
393
394	#ifdef RTR0MEM_EF_FREE_DELAYED
395
396	/**
397	* Insert a delayed block.
398	*/
399	DECLINLINE(void) rtR0MemBlockDelayInsert(PRTR0MEMEFBLOCK pBlock)
400	{
401	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
402	pBlock->Core.pRight = NULL;
403	pBlock->Core.pLeft = NULL;
404	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
405	if (g_pBlocksDelayHead)
406	{
407	g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock;
408	pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead;
409	g_pBlocksDelayHead = pBlock;
410	}
411	else
412	{
413	g_pBlocksDelayTail = pBlock;
414	g_pBlocksDelayHead = pBlock;
415	}
416	g_cbBlocksDelay += cbBlock;
417	rtR0MemBlockUnlock(fSavedIntFlags);
418	}
419
420	/**
421	* Removes a delayed block.
422	*/
423	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockDelayRemove(void)
424	{
425	PRTR0MEMEFBLOCK pBlock = NULL;
426	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
427	if (g_cbBlocksDelay > RTR0MEM_EF_FREE_DELAYED)
428	{
429	pBlock = g_pBlocksDelayTail;
430	if (pBlock)
431	{
432	g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
433	if (pBlock->Core.pLeft)
434	pBlock->Core.pLeft->pRight = NULL;
435	else
436	g_pBlocksDelayHead = NULL;
437	g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
438	}
439	}
440	rtR0MemBlockUnlock(fSavedIntFlags);
441	return pBlock;
442	}
443
444	#endif /* RTR0MEM_EF_FREE_DELAYED */
445
446
447	static void rtR0MemFreeBlock(PRTR0MEMEFBLOCK pBlock, const char *pszOp)
448	{
449	void *pv = pBlock->Core.Key;
450	# ifdef RTR0MEM_EF_IN_FRONT
451	void pvBlock = (char )pv - RTR0MEM_EF_SIZE;
452	# else
453	void pvBlock = (void )((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
454	# endif
455	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
456
457	int rc = RTR0MemObjProtect(pBlock->hMemObj, 0 /offSub/, RT_ALIGN_Z(cbBlock, PAGE_SIZE), RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
458	if (RT_FAILURE(rc))
459	rtR0MemComplain(pszOp, "RTR0MemObjProtect([%p], 0, %#x, RTMEM_PROT_READ \| RTMEM_PROT_WRITE) -> %Rrc\n",
460	pvBlock, cbBlock, rc);
461
462	rc = RTR0MemObjFree(pBlock->hMemObj, true /fFreeMappings/);
463	if (RT_FAILURE(rc))
464	rtR0MemComplain(pszOp, "RTR0MemObjFree([%p LB %#x]) -> %Rrc\n", pvBlock, cbBlock, rc);
465	pBlock->hMemObj = NIL_RTR0MEMOBJ;
466
467	rtR0MemBlockFree(pBlock);
468	}
469
470
471	/**
472	* Initialize call, we shouldn't fail here.
473	*/
474	void rtR0MemEfInit(void)
475	{
476
477	}
478
479	/**
480	* @callback_method_impl{AVLPVCALLBACK}
481	*/
482	static DECLCALLBACK(int) rtR0MemEfDestroyBlock(PAVLPVNODECORE pNode, void *pvUser)
483	{
484	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
485
486	/* Note! pszFile and pszFunction may be invalid at this point. */
487	rtR0MemComplain("rtR0MemEfDestroyBlock", "Leaking %zu bytes at %p (iLine=%u pvCaller=%p)\n",
488	pBlock->cbAligned, pBlock->Core.Key, pBlock->iLine, pBlock->pvCaller);
489
490	rtR0MemFreeBlock(pBlock, "rtR0MemEfDestroyBlock");
491
492	NOREF(pvUser);
493	return VINF_SUCCESS;
494	}
495
496
497	/**
498	* Termination call.
499	*
500	* Will check and free memory.
501	*/
502	void rtR0MemEfTerm(void)
503	{
504	#ifdef RTR0MEM_EF_FREE_DELAYED
505	/*
506	* Release delayed frees.
507	*/
508	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
509	for (;;)
510	{
511	PRTR0MEMEFBLOCK pBlock = g_pBlocksDelayTail;
512	if (pBlock)
513	{
514	g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
515	if (pBlock->Core.pLeft)
516	pBlock->Core.pLeft->pRight = NULL;
517	else
518	g_pBlocksDelayHead = NULL;
519	rtR0MemBlockUnlock(fSavedIntFlags);
520
521	rtR0MemFreeBlock(pBlock, "rtR0MemEfTerm");
522
523	rtR0MemBlockLock();
524	}
525	else
526	break;
527	}
528	g_cbBlocksDelay = 0;
529	rtR0MemBlockUnlock(fSavedIntFlags);
530	#endif
531
532	/*
533	* Complain about leaks. Then release them.
534	*/
535	RTAvlPVDestroy(&g_BlocksTree, rtR0MemEfDestroyBlock, NULL);
536	}
537
538
539	/**
540	* Internal allocator.
541	*/
542	static void * rtR0MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
543	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
544	{
545	/*
546	* Sanity.
547	*/
548	if ( RT_ALIGN_Z(RTR0MEM_EF_SIZE, PAGE_SIZE) != RTR0MEM_EF_SIZE
549	&& RTR0MEM_EF_SIZE <= 0)
550	{
551	rtR0MemComplain(pszOp, "Invalid E-fence size! %#x\n", RTR0MEM_EF_SIZE);
552	return NULL;
553	}
554	if (!cbUnaligned)
555	{
556	#if 1
557	rtR0MemComplain(pszOp, "Request of ZERO bytes allocation!\n");
558	return NULL;
559	#else
560	cbAligned = cbUnaligned = 1;
561	#endif
562	}
563
564	#ifndef RTR0MEM_EF_IN_FRONT
565	/* Alignment decreases fence accuracy, but this is at least partially
566	* counteracted by filling and checking the alignment padding. When the
567	* fence is in front then then no extra alignment is needed. */
568	cbAligned = RT_ALIGN_Z(cbAligned, RTR0MEM_EF_ALIGNMENT);
569	#endif
570
571	/*
572	* Allocate the trace block.
573	*/
574	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
575	if (!pBlock)
576	{
577	rtR0MemComplain(pszOp, "Failed to allocate trace block!\n");
578	return NULL;
579	}
580
581	/*
582	* Allocate a block with page alignment space + the size of the E-fence.
583	*/
584	void *pvBlock = NULL;
585	RTR0MEMOBJ hMemObj;
586	size_t cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
587	int rc = RTR0MemObjAllocPage(&hMemObj, cbBlock, false /fExecutable/);
588	if (RT_SUCCESS(rc))
589	pvBlock = RTR0MemObjAddress(hMemObj);
590	if (pvBlock)
591	{
592	/*
593	* Calc the start of the fence and the user block
594	* and then change the page protection of the fence.
595	*/
596	#ifdef RTR0MEM_EF_IN_FRONT
597	void *pvEFence = pvBlock;
598	void pv = (char )pvEFence + RTR0MEM_EF_SIZE;
599	# ifdef RTR0MEM_EF_NOMAN_FILLER
600	memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbUnaligned);
601	# endif
602	#else
603	void pvEFence = (char )pvBlock + (cbBlock - RTR0MEM_EF_SIZE);
604	void pv = (char )pvEFence - cbAligned;
605	# ifdef RTR0MEM_EF_NOMAN_FILLER
606	memset(pvBlock, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbAligned);
607	memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbAligned - cbUnaligned);
608	# endif
609	#endif
610
611	#ifdef RTR0MEM_EF_FENCE_FILLER
612	memset(pvEFence, RTR0MEM_EF_FENCE_FILLER, RTR0MEM_EF_SIZE);
613	#endif
614	rc = RTR0MemObjProtect(hMemObj, (uint8_t )pvEFence - (uint8_t )pvBlock, RTR0MEM_EF_SIZE, RTMEM_PROT_NONE);
615	if (!rc)
616	{
617	rtR0MemBlockInsert(pBlock, pv, hMemObj);
618	if (enmType == RTMEMTYPE_RTMEMALLOCZ)
619	memset(pv, 0, cbUnaligned);
620	#ifdef RTR0MEM_EF_FILLER
621	else
622	memset(pv, RTR0MEM_EF_FILLER, cbUnaligned);
623	#endif
624
625	rtR0MemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
626	return pv;
627	}
628	rtR0MemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTR0MEM_EF_SIZE, rc);
629	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
630	}
631	else
632	{
633	rtR0MemComplain(pszOp, "Failed to allocated %zu (%zu) bytes (rc=%Rrc).\n", cbBlock, cbUnaligned, rc);
634	if (RT_SUCCESS(rc))
635	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
636	}
637
638	rtR0MemBlockFree(pBlock);
639	return NULL;
640	}
641
642
643	/**
644	* Internal free.
645	*/
646	static void rtR0MemFree(const char pszOp, RTMEMTYPE enmType, void pv, void *pvCaller, RT_SRC_POS_DECL)
647	{
648	NOREF(enmType); RT_SRC_POS_NOREF();
649
650	/*
651	* Simple case.
652	*/
653	if (!pv)
654	return;
655
656	/*
657	* Check watch points.
658	*/
659	for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++)
660	if (gapvRTMemFreeWatch[i] == pv)
661	RTAssertDoPanic();
662
663	/*
664	* Find the block.
665	*/
666	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockRemove(pv);
667	if (pBlock)
668	{
669	if (gfRTMemFreeLog)
670	RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
671
672	#ifdef RTR0MEM_EF_NOMAN_FILLER
673	/*
674	* Check whether the no man's land is untouched.
675	*/
676	# ifdef RTR0MEM_EF_IN_FRONT
677	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
678	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned,
679	RTR0MEM_EF_NOMAN_FILLER);
680	# else
681	/* Alignment must match allocation alignment in rtMemAlloc(). */
682	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
683	pBlock->cbAligned - pBlock->cbUnaligned,
684	RTR0MEM_EF_NOMAN_FILLER);
685	if (pvWrong)
686	RTAssertDoPanic();
687	pvWrong = ASMMemFirstMismatchingU8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK),
688	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned,
689	RTR0MEM_EF_NOMAN_FILLER);
690	# endif
691	if (pvWrong)
692	RTAssertDoPanic();
693	#endif
694
695	#ifdef RTR0MEM_EF_FREE_FILL
696	/*
697	* Fill the user part of the block.
698	*/
699	memset(pv, RTR0MEM_EF_FREE_FILL, pBlock->cbUnaligned);
700	#endif
701
702	#if defined(RTR0MEM_EF_FREE_DELAYED) && RTR0MEM_EF_FREE_DELAYED > 0
703	/*
704	* We're doing delayed freeing.
705	* That means we'll expand the E-fence to cover the entire block.
706	*/
707	int rc = RTR0MemObjProtect(pBlock->hMemObj,
708	# ifdef RTR0MEM_EF_IN_FRONT
709	RTR0MEM_EF_SIZE,
710	# else
711	0 /offSub/,
712	# endif
713	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE),
714	RTMEM_PROT_NONE);
715	if (RT_SUCCESS(rc))
716	{
717	/*
718	* Insert it into the free list and process pending frees.
719	*/
720	rtR0MemBlockDelayInsert(pBlock);
721	while ((pBlock = rtR0MemBlockDelayRemove()) != NULL)
722	rtR0MemFreeBlock(pBlock, pszOp);
723	}
724	else
725	rtR0MemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
726
727	#else /* !RTR0MEM_EF_FREE_DELAYED */
728	rtR0MemFreeBlock(pBlock, pszOp);
729	#endif /* !RTR0MEM_EF_FREE_DELAYED */
730	}
731	else
732	rtR0MemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
733	}
734
735
736	/**
737	* Internal realloc.
738	*/
739	static void rtR0MemRealloc(const char pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
740	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
741	{
742	/*
743	* Allocate new and copy.
744	*/
745	if (!pvOld)
746	return rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
747	if (!cbNew)
748	{
749	rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
750	return NULL;
751	}
752
753	/*
754	* Get the block, allocate the new, copy the data, free the old one.
755	*/
756	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockGet(pvOld);
757	if (pBlock)
758	{
759	void *pvRet = rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
760	if (pvRet)
761	{
762	memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
763	rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
764	}
765	return pvRet;
766	}
767	rtR0MemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
768	return NULL;
769	}
770
771
772
773
774	RTDECL(void ) RTMemEfTmpAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
775	{
776	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
777	}
778
779
780	RTDECL(void ) RTMemEfTmpAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
781	{
782	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
783	}
784
785
786	RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
787	{
788	if (pv)
789	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
790	}
791
792
793	RTDECL(void ) RTMemEfAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
794	{
795	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
796	}
797
798
799	RTDECL(void ) RTMemEfAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
800	{
801	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
802	}
803
804
805	RTDECL(void ) RTMemEfAllocVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
806	{
807	size_t cbAligned;
808	if (cbUnaligned >= 16)
809	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
810	else
811	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
812	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
813	}
814
815
816	RTDECL(void ) RTMemEfAllocZVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
817	{
818	size_t cbAligned;
819	if (cbUnaligned >= 16)
820	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
821	else
822	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
823	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
824	}
825
826
827	RTDECL(void ) RTMemEfRealloc(void pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
828	{
829	return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
830	}
831
832
833	RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
834	{
835	if (pv)
836	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
837	}
838
839
840	RTDECL(void ) RTMemEfDup(const void pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
841	{
842	void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS);
843	if (pvDst)
844	memcpy(pvDst, pvSrc, cb);
845	return pvDst;
846	}
847
848
849	RTDECL(void ) RTMemEfDupEx(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
850	{
851	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS);
852	if (pvDst)
853	{
854	memcpy(pvDst, pvSrc, cbSrc);
855	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
856	}
857	return pvDst;
858	}
859
860
861
862
863	/*
864	*
865	* The NP (no position) versions.
866	*
867	*/
868
869
870
871	RTDECL(void ) RTMemEfTmpAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
872	{
873	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
874	}
875
876
877	RTDECL(void ) RTMemEfTmpAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
878	{
879	return rtR0MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
880	}
881
882
883	RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF
884	{
885	if (pv)
886	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
887	}
888
889
890	RTDECL(void ) RTMemEfAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
891	{
892	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
893	}
894
895
896	RTDECL(void ) RTMemEfAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
897	{
898	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
899	}
900
901
902	RTDECL(void ) RTMemEfAllocVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
903	{
904	size_t cbAligned;
905	if (cbUnaligned >= 16)
906	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
907	else
908	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
909	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
910	}
911
912
913	RTDECL(void ) RTMemEfAllocZVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
914	{
915	size_t cbAligned;
916	if (cbUnaligned >= 16)
917	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
918	else
919	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
920	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
921	}
922
923
924	RTDECL(void ) RTMemEfReallocNP(void pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
925	{
926	return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
927	}
928
929
930	RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF
931	{
932	if (pv)
933	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
934	}
935
936
937	RTDECL(void ) RTMemEfDupNP(const void pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF
938	{
939	void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL);
940	if (pvDst)
941	memcpy(pvDst, pvSrc, cb);
942	return pvDst;
943	}
944
945
946	RTDECL(void ) RTMemEfDupExNP(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF
947	{
948	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL);
949	if (pvDst)
950	{
951	memcpy(pvDst, pvSrc, cbSrc);
952	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
953	}
954	return pvDst;
955	}
956

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp@ 61572

Download in other formats: