alloc-ef.cpp@ 86186

Last change on this file since 86186 was 83548, checked in by vboxsync, 5 years ago
IPRT: Added RTMemFreeZ, RTMemTmpFreeZ, and RTMemEf* variants. [build fix] bugref:9698
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 33.1 KB

Line
1	/* $Id: alloc-ef.cpp 83548 2020-04-04 11:08:06Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Allocation, electric fence.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "alloc-ef.h"
32	#include <iprt/mem.h>
33	#include <iprt/log.h>
34	#include <iprt/asm.h>
35	#include <iprt/thread.h>
36	#include <VBox/sup.h>
37	#include <iprt/errcore.h>
38	#include <errno.h>
39	#include <stdio.h>
40	#include <stdlib.h>
41
42	#include <iprt/alloc.h>
43	#include <iprt/assert.h>
44	#include <iprt/param.h>
45	#include <iprt/string.h>
46
47	#ifdef RTALLOC_REPLACE_MALLOC
48	# include <VBox/dis.h>
49	# include <VBox/disopcode.h>
50	# include <dlfcn.h>
51	# ifdef RT_OS_DARWIN
52	# include <malloc/malloc.h>
53	# endif
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	#ifdef RTALLOC_REPLACE_MALLOC
61	# define RTMEM_REPLACMENT_ALIGN(a_cb) ((a_cb) >= 16 ? RT_ALIGN_Z(a_cb, 16) \
62	: (a_cb) >= sizeof(uintptr_t) ? RT_ALIGN_Z(a_cb, sizeof(uintptr_t)) : (a_cb))
63	#endif
64
65
66	/*********************************************************************************************************************************
67	* Global Variables *
68	*********************************************************************************************************************************/
69	#ifdef RTALLOC_EFENCE_TRACE
70	/** Spinlock protecting the all the block's globals. */
71	static volatile uint32_t g_BlocksLock;
72	/** Tree tracking the allocations. */
73	static AVLPVTREE g_BlocksTree;
74	# ifdef RTALLOC_EFENCE_FREE_DELAYED
75	/** Tail of the delayed blocks. */
76	static volatile PRTMEMBLOCK g_pBlocksDelayHead;
77	/** Tail of the delayed blocks. */
78	static volatile PRTMEMBLOCK g_pBlocksDelayTail;
79	/** Number of bytes in the delay list (includes fences). */
80	static volatile size_t g_cbBlocksDelay;
81	# endif /* RTALLOC_EFENCE_FREE_DELAYED */
82	# ifdef RTALLOC_REPLACE_MALLOC
83	/** @name For calling the real allocation API we've replaced.
84	* @{ */
85	void * (*g_pfnOrgMalloc)(size_t);
86	void * (*g_pfnOrgCalloc)(size_t, size_t);
87	void * (g_pfnOrgRealloc)(void , size_t);
88	void (g_pfnOrgFree)(void );
89	size_t (g_pfnOrgMallocSize)(void );
90	/** @} */
91	# endif
92	#endif /* RTALLOC_EFENCE_TRACE */
93	/** Array of pointers free watches for. */
94	void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
95	/** Enable logging of all freed memory. */
96	bool gfRTMemFreeLog = false;
97
98
99	/*********************************************************************************************************************************
100	* Internal Functions *
101	*********************************************************************************************************************************/
102	#ifdef RTALLOC_REPLACE_MALLOC
103	static void rtMemReplaceMallocAndFriends(void);
104	#endif
105
106
107	/**
108	* Complains about something.
109	*/
110	static void rtmemComplain(const char pszOp, const char pszFormat, ...)
111	{
112	va_list args;
113	fprintf(stderr, "RTMem error: %s: ", pszOp);
114	va_start(args, pszFormat);
115	vfprintf(stderr, pszFormat, args);
116	va_end(args);
117	RTAssertDoPanic();
118	}
119
120	/**
121	* Log an event.
122	*/
123	DECLINLINE(void) rtmemLog(const char pszOp, const char pszFormat, ...)
124	{
125	#if 0
126	va_list args;
127	fprintf(stderr, "RTMem info: %s: ", pszOp);
128	va_start(args, pszFormat);
129	vfprintf(stderr, pszFormat, args);
130	va_end(args);
131	#else
132	NOREF(pszOp); NOREF(pszFormat);
133	#endif
134	}
135
136
137	#ifdef RTALLOC_EFENCE_TRACE
138
139	/**
140	* Acquires the lock.
141	*/
142	DECLINLINE(void) rtmemBlockLock(void)
143	{
144	unsigned c = 0;
145	while (!ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
146	RTThreadSleepNoLog(((++c) >> 2) & 31);
147	}
148
149
150	/**
151	* Releases the lock.
152	*/
153	DECLINLINE(void) rtmemBlockUnlock(void)
154	{
155	Assert(g_BlocksLock == 1);
156	ASMAtomicXchgU32(&g_BlocksLock, 0);
157	}
158
159
160	/**
161	* Creates a block.
162	*/
163	DECLINLINE(PRTMEMBLOCK) rtmemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
164	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
165	{
166	# ifdef RTALLOC_REPLACE_MALLOC
167	if (!g_pfnOrgMalloc)
168	rtMemReplaceMallocAndFriends();
169	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)g_pfnOrgMalloc(sizeof(*pBlock));
170	# else
171	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)malloc(sizeof(*pBlock));
172	# endif
173	if (pBlock)
174	{
175	pBlock->enmType = enmType;
176	pBlock->cbUnaligned = cbUnaligned;
177	pBlock->cbAligned = cbAligned;
178	pBlock->pszTag = pszTag;
179	pBlock->pvCaller = pvCaller;
180	pBlock->iLine = iLine;
181	pBlock->pszFile = pszFile;
182	pBlock->pszFunction = pszFunction;
183	}
184	return pBlock;
185	}
186
187
188	/**
189	* Frees a block.
190	*/
191	DECLINLINE(void) rtmemBlockFree(PRTMEMBLOCK pBlock)
192	{
193	# ifdef RTALLOC_REPLACE_MALLOC
194	g_pfnOrgFree(pBlock);
195	# else
196	free(pBlock);
197	# endif
198	}
199
200
201	/**
202	* Insert a block from the tree.
203	*/
204	DECLINLINE(void) rtmemBlockInsert(PRTMEMBLOCK pBlock, void *pv)
205	{
206	pBlock->Core.Key = pv;
207	rtmemBlockLock();
208	bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
209	rtmemBlockUnlock();
210	AssertRelease(fRc);
211	}
212
213
214	/**
215	* Remove a block from the tree and returns it to the caller.
216	*/
217	DECLINLINE(PRTMEMBLOCK) rtmemBlockRemove(void *pv)
218	{
219	rtmemBlockLock();
220	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
221	rtmemBlockUnlock();
222	return pBlock;
223	}
224
225	/**
226	* Gets a block.
227	*/
228	DECLINLINE(PRTMEMBLOCK) rtmemBlockGet(void *pv)
229	{
230	rtmemBlockLock();
231	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
232	rtmemBlockUnlock();
233	return pBlock;
234	}
235
236	/**
237	* Dumps one allocation.
238	*/
239	static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
240	{
241	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)pNode;
242	fprintf(stderr, "%p %08lx(+%02lx) %p\n",
243	pBlock->Core.Key,
244	(unsigned long)pBlock->cbUnaligned,
245	(unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
246	pBlock->pvCaller);
247	NOREF(pvUser);
248	return 0;
249	}
250
251	/**
252	* Dumps the allocated blocks.
253	* This is something which you should call from gdb.
254	*/
255	extern "C" void RTMemDump(void);
256	void RTMemDump(void)
257	{
258	fprintf(stderr, "address size(alg) caller\n");
259	RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
260	}
261
262	# ifdef RTALLOC_EFENCE_FREE_DELAYED
263
264	/**
265	* Insert a delayed block.
266	*/
267	DECLINLINE(void) rtmemBlockDelayInsert(PRTMEMBLOCK pBlock)
268	{
269	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
270	pBlock->Core.pRight = NULL;
271	pBlock->Core.pLeft = NULL;
272	rtmemBlockLock();
273	if (g_pBlocksDelayHead)
274	{
275	g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock;
276	pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead;
277	g_pBlocksDelayHead = pBlock;
278	}
279	else
280	{
281	g_pBlocksDelayTail = pBlock;
282	g_pBlocksDelayHead = pBlock;
283	}
284	g_cbBlocksDelay += cbBlock;
285	rtmemBlockUnlock();
286	}
287
288	/**
289	* Removes a delayed block.
290	*/
291	DECLINLINE(PRTMEMBLOCK) rtmemBlockDelayRemove(void)
292	{
293	PRTMEMBLOCK pBlock = NULL;
294	rtmemBlockLock();
295	if (g_cbBlocksDelay > RTALLOC_EFENCE_FREE_DELAYED)
296	{
297	pBlock = g_pBlocksDelayTail;
298	if (pBlock)
299	{
300	g_pBlocksDelayTail = (PRTMEMBLOCK)pBlock->Core.pLeft;
301	if (pBlock->Core.pLeft)
302	pBlock->Core.pLeft->pRight = NULL;
303	else
304	g_pBlocksDelayHead = NULL;
305	g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
306	}
307	}
308	rtmemBlockUnlock();
309	return pBlock;
310	}
311
312	# endif /* RTALLOC_EFENCE_FREE_DELAYED */
313
314	#endif /* RTALLOC_EFENCE_TRACE */
315
316
317	#if defined(RTALLOC_REPLACE_MALLOC) && defined(RTALLOC_EFENCE_TRACE)
318	/*
319	*
320	* Replacing malloc, calloc, realloc, & free.
321	*
322	*/
323
324	/** Replacement for malloc. */
325	static void *rtMemReplacementMalloc(size_t cb)
326	{
327	size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
328	void *pv = rtR3MemAlloc("r-malloc", RTMEMTYPE_RTMEMALLOC, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
329	if (!pv)
330	pv = g_pfnOrgMalloc(cb);
331	return pv;
332	}
333
334	/** Replacement for calloc. */
335	static void *rtMemReplacementCalloc(size_t cbItem, size_t cItems)
336	{
337	size_t cb = cbItem * cItems;
338	size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
339	void *pv = rtR3MemAlloc("r-calloc", RTMEMTYPE_RTMEMALLOCZ, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
340	if (!pv)
341	pv = g_pfnOrgCalloc(cbItem, cItems);
342	return pv;
343	}
344
345	/** Replacement for realloc. */
346	static void rtMemReplacementRealloc(void pvOld, size_t cbNew)
347	{
348	if (pvOld)
349	{
350	/* We're not strict about where the memory was allocated. */
351	PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
352	if (pBlock)
353	{
354	size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cbNew);
355	return rtR3MemRealloc("r-realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
356	}
357	return g_pfnOrgRealloc(pvOld, cbNew);
358	}
359	return rtMemReplacementMalloc(cbNew);
360	}
361
362	/** Replacement for free(). */
363	static void rtMemReplacementFree(void *pv)
364	{
365	if (pv)
366	{
367	/* We're not strict about where the memory was allocated. */
368	PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
369	if (pBlock)
370	rtR3MemFree("r-free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS);
371	else
372	g_pfnOrgFree(pv);
373	}
374	}
375
376	# ifdef RT_OS_DARWIN
377	/** Replacement for malloc. */
378	static size_t rtMemReplacementMallocSize(void *pv)
379	{
380	size_t cb;
381	if (pv)
382	{
383	/* We're not strict about where the memory was allocated. */
384	PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
385	if (pBlock)
386	cb = pBlock->cbUnaligned;
387	else
388	cb = g_pfnOrgMallocSize(pv);
389	}
390	else
391	cb = 0;
392	return cb;
393	}
394	# endif
395
396
397	static void rtMemReplaceMallocAndFriends(void)
398	{
399	struct
400	{
401	const char *pszName;
402	PFNRT pfnReplacement;
403	PFNRT pfnOrg;
404	PFNRT *ppfnJumpBack;
405	} aApis[] =
406	{
407	{ "free", (PFNRT)rtMemReplacementFree, (PFNRT)free, (PFNRT *)&g_pfnOrgFree },
408	{ "realloc", (PFNRT)rtMemReplacementRealloc, (PFNRT)realloc, (PFNRT *)&g_pfnOrgRealloc },
409	{ "calloc", (PFNRT)rtMemReplacementCalloc, (PFNRT)calloc, (PFNRT *)&g_pfnOrgCalloc },
410	{ "malloc", (PFNRT)rtMemReplacementMalloc, (PFNRT)malloc, (PFNRT *)&g_pfnOrgMalloc },
411	#ifdef RT_OS_DARWIN
412	{ "malloc_size", (PFNRT)rtMemReplacementMallocSize, (PFNRT)malloc_size, (PFNRT *)&g_pfnOrgMallocSize },
413	#endif
414	};
415
416	/*
417	* Initialize the jump backs to avoid recursivly entering this function.
418	*/
419	for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
420	*aApis[i].ppfnJumpBack = aApis[i].pfnOrg;
421
422	/*
423	* Give the user an option to skip replacing malloc.
424	*/
425	if (getenv("IPRT_DONT_REPLACE_MALLOC"))
426	return;
427
428	/*
429	* Allocate a page for jump back code (we leak it).
430	*/
431	uint8_t pbExecPage = (uint8_t )RTMemPageAlloc(PAGE_SIZE); AssertFatal(pbExecPage);
432	int rc = RTMemProtect(pbExecPage, PAGE_SIZE, RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC); AssertFatalRC(rc);
433
434	/*
435	* Do the ground work.
436	*/
437	uint8_t *pb = pbExecPage;
438	for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
439	{
440	/* Resolve it. */
441	PFNRT pfnOrg = (PFNRT)(uintptr_t)dlsym(RTLD_DEFAULT, aApis[i].pszName);
442	if (pfnOrg)
443	aApis[i].pfnOrg = pfnOrg;
444	else
445	pfnOrg = aApis[i].pfnOrg;
446
447	/* Figure what we can replace and how much to duplicate in the jump back code. */
448	# ifdef RT_ARCH_AMD64
449	uint32_t cbNeeded = 12;
450	DISCPUMODE const enmCpuMode = DISCPUMODE_64BIT;
451	# elif defined(RT_ARCH_X86)
452	uint32_t const cbNeeded = 5;
453	DISCPUMODE const enmCpuMode = DISCPUMODE_32BIT;
454	# else
455	# error "Port me"
456	# endif
457	uint32_t offJmpBack = 0;
458	uint32_t cbCopy = 0;
459	while (offJmpBack < cbNeeded)
460	{
461	DISCPUSTATE Dis;
462	uint32_t cbInstr = 1;
463	rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
464	AssertFatal(!(Dis.pCurInstr->fOpType & (DISOPTYPE_CONTROLFLOW)));
465	# ifdef RT_ARCH_AMD64
466	# ifdef RT_OS_DARWIN
467	/* Kludge for: cmp [malloc_def_zone_state], 1; jg 2; call _malloc_initialize; 2: */
468	DISQPVPARAMVAL Parm;
469	if ( Dis.ModRM.Bits.Mod == 0
470	&& Dis.ModRM.Bits.Rm == 5 /* wrt RIP */
471	&& (Dis.Param2.fUse & (DISUSE_IMMEDIATE16_SX8 \| DISUSE_IMMEDIATE32_SX8 \| DISUSE_IMMEDIATE64_SX8))
472	&& Dis.Param2.uValue == 1
473	&& Dis.pCurInstr->uOpcode == OP_CMP)
474	{
475	cbCopy = offJmpBack;
476
477	offJmpBack += cbInstr;
478	rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
479	if ( Dis.pCurInstr->uOpcode == OP_JNBE
480	&& Dis.Param1.uDisp.i8 == 5)
481	{
482	offJmpBack += cbInstr + 5;
483	AssertFatal(offJmpBack >= cbNeeded);
484	break;
485	}
486	}
487	# endif
488	AssertFatal(!(Dis.ModRM.Bits.Mod == 0 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */));
489	# endif
490	offJmpBack += cbInstr;
491	}
492	if (!cbCopy)
493	cbCopy = offJmpBack;
494
495	/* Assemble the jump back. */
496	memcpy(pb, (void *)(uintptr_t)pfnOrg, cbCopy);
497	uint32_t off = cbCopy;
498	# ifdef RT_ARCH_AMD64
499	pb[off++] = 0xff; /* jmp qword [$+8 wrt RIP] */
500	pb[off++] = 0x25;
501	(uint32_t )&pb[off] = 0;
502	off += 4;
503	(uint64_t )&pb[off] = (uintptr_t)pfnOrg + offJmpBack;
504	off += 8;
505	off = RT_ALIGN_32(off, 16);
506	# elif defined(RT_ARCH_X86)
507	pb[off++] = 0xe9; /* jmp rel32 */
508	(uint32_t )&pb[off] = (uintptr_t)pfnOrg + offJmpBack - (uintptr_t)&pb[4];
509	off += 4;
510	off = RT_ALIGN_32(off, 8);
511	# else
512	# error "Port me"
513	# endif
514	*aApis[i].ppfnJumpBack = (PFNRT)(uintptr_t)pb;
515	pb += off;
516	}
517
518	/*
519	* Modify the APIs.
520	*/
521	for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
522	{
523	pb = (uint8_t *)(uintptr_t)aApis[i].pfnOrg;
524	rc = RTMemProtect(pb, 16, RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC); AssertFatalRC(rc);
525
526	# ifdef RT_ARCH_AMD64
527	/* Assemble the LdrLoadDll patch. */
528	pb++ = 0x48; / mov rax, qword */
529	*pb++ = 0xb8;
530	(uint64_t )pb = (uintptr_t)aApis[i].pfnReplacement;
531	pb += 8;
532	pb++ = 0xff; / jmp rax */
533	*pb++ = 0xe0;
534	# elif defined(RT_ARCH_X86)
535	pb++ = 0xe9; / jmp rel32 */
536	(uint32_t )pb = (uintptr_t)aApis[i].pfnReplacement - (uintptr_t)&pb[4];
537	# else
538	# error "Port me"
539	# endif
540	}
541	}
542
543	#endif /* RTALLOC_REPLACE_MALLOC && RTALLOC_EFENCE_TRACE */
544
545
546	/**
547	* Internal allocator.
548	*/
549	RTDECL(void ) rtR3MemAlloc(const char pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
550	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
551	{
552	/*
553	* Sanity.
554	*/
555	if ( RT_ALIGN_Z(RTALLOC_EFENCE_SIZE, PAGE_SIZE) != RTALLOC_EFENCE_SIZE
556	&& RTALLOC_EFENCE_SIZE <= 0)
557	{
558	rtmemComplain(pszOp, "Invalid E-fence size! %#x\n", RTALLOC_EFENCE_SIZE);
559	return NULL;
560	}
561	if (!cbUnaligned)
562	{
563	#if 0
564	rtmemComplain(pszOp, "Request of ZERO bytes allocation!\n");
565	return NULL;
566	#else
567	cbAligned = cbUnaligned = 1;
568	#endif
569	}
570
571	#ifndef RTALLOC_EFENCE_IN_FRONT
572	/* Alignment decreases fence accuracy, but this is at least partially
573	* counteracted by filling and checking the alignment padding. When the
574	* fence is in front then then no extra alignment is needed. */
575	cbAligned = RT_ALIGN_Z(cbAligned, RTALLOC_EFENCE_ALIGNMENT);
576	#endif
577
578	#ifdef RTALLOC_EFENCE_TRACE
579	/*
580	* Allocate the trace block.
581	*/
582	PRTMEMBLOCK pBlock = rtmemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
583	if (!pBlock)
584	{
585	rtmemComplain(pszOp, "Failed to allocate trace block!\n");
586	return NULL;
587	}
588	#endif
589
590	/*
591	* Allocate a block with page alignment space + the size of the E-fence.
592	*/
593	size_t cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
594	void *pvBlock = RTMemPageAlloc(cbBlock);
595	if (pvBlock)
596	{
597	/*
598	* Calc the start of the fence and the user block
599	* and then change the page protection of the fence.
600	*/
601	#ifdef RTALLOC_EFENCE_IN_FRONT
602	void *pvEFence = pvBlock;
603	void pv = (char )pvEFence + RTALLOC_EFENCE_SIZE;
604	# ifdef RTALLOC_EFENCE_NOMAN_FILLER
605	memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbUnaligned);
606	# endif
607	#else
608	void pvEFence = (char )pvBlock + (cbBlock - RTALLOC_EFENCE_SIZE);
609	void pv = (char )pvEFence - cbAligned;
610	# ifdef RTALLOC_EFENCE_NOMAN_FILLER
611	memset(pvBlock, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbAligned);
612	memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbAligned - cbUnaligned);
613	# endif
614	#endif
615
616	#ifdef RTALLOC_EFENCE_FENCE_FILLER
617	memset(pvEFence, RTALLOC_EFENCE_FENCE_FILLER, RTALLOC_EFENCE_SIZE);
618	#endif
619	int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_NONE);
620	if (!rc)
621	{
622	#ifdef RTALLOC_EFENCE_TRACE
623	rtmemBlockInsert(pBlock, pv);
624	#endif
625	if (enmType == RTMEMTYPE_RTMEMALLOCZ)
626	memset(pv, 0, cbUnaligned);
627	#ifdef RTALLOC_EFENCE_FILLER
628	else
629	memset(pv, RTALLOC_EFENCE_FILLER, cbUnaligned);
630	#endif
631
632	rtmemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
633	return pv;
634	}
635	rtmemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
636	RTMemPageFree(pvBlock, cbBlock);
637	}
638	else
639	rtmemComplain(pszOp, "Failed to allocated %lu (%lu) bytes.\n", (unsigned long)cbBlock, (unsigned long)cbUnaligned);
640
641	#ifdef RTALLOC_EFENCE_TRACE
642	rtmemBlockFree(pBlock);
643	#endif
644	return NULL;
645	}
646
647
648	/**
649	* Internal free.
650	*/
651	RTDECL(void) rtR3MemFree(const char pszOp, RTMEMTYPE enmType, void pv, size_t cbUser, void *pvCaller, RT_SRC_POS_DECL)
652	{
653	NOREF(enmType); RT_SRC_POS_NOREF();
654
655	/*
656	* Simple case.
657	*/
658	if (!pv)
659	return;
660
661	/*
662	* Check watch points.
663	*/
664	for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++)
665	if (gapvRTMemFreeWatch[i] == pv)
666	RTAssertDoPanic();
667
668	#ifdef RTALLOC_EFENCE_TRACE
669	/*
670	* Find the block.
671	*/
672	PRTMEMBLOCK pBlock = rtmemBlockRemove(pv);
673	if (pBlock)
674	{
675	if (gfRTMemFreeLog)
676	RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
677
678	# ifdef RTALLOC_EFENCE_NOMAN_FILLER
679	/*
680	* Check whether the no man's land is untouched.
681	*/
682	# ifdef RTALLOC_EFENCE_IN_FRONT
683	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
684	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned,
685	RTALLOC_EFENCE_NOMAN_FILLER);
686	# else
687	/* Alignment must match allocation alignment in rtMemAlloc(). */
688	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
689	pBlock->cbAligned - pBlock->cbUnaligned,
690	RTALLOC_EFENCE_NOMAN_FILLER);
691	if (pvWrong)
692	RTAssertDoPanic();
693	pvWrong = ASMMemFirstMismatchingU8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK),
694	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned,
695	RTALLOC_EFENCE_NOMAN_FILLER);
696	# endif
697	if (pvWrong)
698	RTAssertDoPanic();
699	# endif
700
701	/*
702	* Fill the user part of the block.
703	*/
704	AssertMsg(enmType != RTMEMTYPE_RTMEMFREEZ \|\| cbUser == pBlock->cbUnaligned,
705	("cbUser=%#zx cbUnaligned=%#zx\n", cbUser, pBlock->cbUnaligned));
706	RT_NOREF(cbUser);
707	if (enmType == RTMEMTYPE_RTMEMFREEZ)
708	RT_BZERO(pv, pBlock->cbUnaligned);
709	# ifdef RTALLOC_EFENCE_FREE_FILL
710	else
711	memset(pv, RTALLOC_EFENCE_FREE_FILL, pBlock->cbUnaligned);
712	# endif
713
714	# if defined(RTALLOC_EFENCE_FREE_DELAYED) && RTALLOC_EFENCE_FREE_DELAYED > 0
715	/*
716	* We're doing delayed freeing.
717	* That means we'll expand the E-fence to cover the entire block.
718	*/
719	int rc = RTMemProtect(pv, pBlock->cbAligned, RTMEM_PROT_NONE);
720	if (RT_SUCCESS(rc))
721	{
722	/*
723	* Insert it into the free list and process pending frees.
724	*/
725	rtmemBlockDelayInsert(pBlock);
726	while ((pBlock = rtmemBlockDelayRemove()) != NULL)
727	{
728	pv = pBlock->Core.Key;
729	# ifdef RTALLOC_EFENCE_IN_FRONT
730	void pvBlock = (char )pv - RTALLOC_EFENCE_SIZE;
731	# else
732	void pvBlock = (void )((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
733	# endif
734	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
735	rc = RTMemProtect(pvBlock, cbBlock, RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
736	if (RT_SUCCESS(rc))
737	RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
738	else
739	rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ \| RTMEM_PROT_WRITE) -> %d\n", pvBlock, cbBlock, rc);
740	rtmemBlockFree(pBlock);
741	}
742	}
743	else
744	rtmemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
745
746	# else /* !RTALLOC_EFENCE_FREE_DELAYED */
747
748	/*
749	* Turn of the E-fence and free it.
750	*/
751	# ifdef RTALLOC_EFENCE_IN_FRONT
752	void pvBlock = (char )pv - RTALLOC_EFENCE_SIZE;
753	void *pvEFence = pvBlock;
754	# else
755	void pvBlock = (void )((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
756	void pvEFence = (char )pv + pBlock->cb;
757	# endif
758	int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
759	if (RT_SUCCESS(rc))
760	RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
761	else
762	rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ \| RTMEM_PROT_WRITE) -> %d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
763	rtmemBlockFree(pBlock);
764
765	# endif /* !RTALLOC_EFENCE_FREE_DELAYED */
766	}
767	else
768	rtmemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
769
770	#else /* !RTALLOC_EFENCE_TRACE */
771
772	/*
773	* We have no size tracking, so we're not doing any freeing because
774	* we cannot if the E-fence is after the block.
775	* Let's just expand the E-fence to the first page of the user bit
776	* since we know that it's around.
777	*/
778	if (enmType == RTMEMTYPE_RTMEMFREEZ)
779	RT_BZERO(pv, cbUser);
780	int rc = RTMemProtect((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), PAGE_SIZE, RTMEM_PROT_NONE);
781	if (RT_FAILURE(rc))
782	rtmemComplain(pszOp, "RTMemProtect(%p, PAGE_SIZE, RTMEM_PROT_NONE) -> %d\n", (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), rc);
783	#endif /* !RTALLOC_EFENCE_TRACE */
784	}
785
786
787	/**
788	* Internal realloc.
789	*/
790	RTDECL(void ) rtR3MemRealloc(const char pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
791	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
792	{
793	/*
794	* Allocate new and copy.
795	*/
796	if (!pvOld)
797	return rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
798	if (!cbNew)
799	{
800	rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS);
801	return NULL;
802	}
803
804	#ifdef RTALLOC_EFENCE_TRACE
805
806	/*
807	* Get the block, allocate the new, copy the data, free the old one.
808	*/
809	PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
810	if (pBlock)
811	{
812	void *pvRet = rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
813	if (pvRet)
814	{
815	memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
816	rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS);
817	}
818	return pvRet;
819	}
820	else
821	rtmemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
822	return NULL;
823
824	#else /* !RTALLOC_EFENCE_TRACE */
825
826	rtmemComplain(pszOp, "Not supported if RTALLOC_EFENCE_TRACE isn't defined!\n");
827	return NULL;
828
829	#endif /* !RTALLOC_EFENCE_TRACE */
830	}
831
832
833
834
835	RTDECL(void ) RTMemEfTmpAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
836	{
837	return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
838	}
839
840
841	RTDECL(void ) RTMemEfTmpAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
842	{
843	return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
844	}
845
846
847	RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
848	{
849	if (pv)
850	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS);
851	}
852
853
854	RTDECL(void) RTMemEfTmpFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF
855	{
856	if (pv)
857	rtR3MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS);
858	}
859
860
861	RTDECL(void ) RTMemEfAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
862	{
863	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
864	}
865
866
867	RTDECL(void ) RTMemEfAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
868	{
869	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
870	}
871
872
873	RTDECL(void ) RTMemEfAllocVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
874	{
875	size_t cbAligned;
876	if (cbUnaligned >= 16)
877	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
878	else
879	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
880	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
881	}
882
883
884	RTDECL(void ) RTMemEfAllocZVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
885	{
886	size_t cbAligned;
887	if (cbUnaligned >= 16)
888	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
889	else
890	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
891	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
892	}
893
894
895	RTDECL(void ) RTMemEfRealloc(void pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
896	{
897	return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
898	}
899
900
901	RTDECL(void ) RTMemEfReallocZ(void pvOld, size_t cbOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
902	{
903	void *pvDst = rtR3MemRealloc("ReallocZ", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
904	if (pvDst && cbNew > cbOld)
905	memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld);
906	return pvDst;
907	}
908
909
910	RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
911	{
912	if (pv)
913	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS);
914	}
915
916
917	RTDECL(void) RTMemEfFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF
918	{
919	if (pv)
920	rtR3MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS);
921	}
922
923
924	RTDECL(void ) RTMemEfDup(const void pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
925	{
926	void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS);
927	if (pvDst)
928	memcpy(pvDst, pvSrc, cb);
929	return pvDst;
930	}
931
932
933	RTDECL(void ) RTMemEfDupEx(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
934	{
935	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS);
936	if (pvDst)
937	{
938	memcpy(pvDst, pvSrc, cbSrc);
939	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
940	}
941	return pvDst;
942	}
943
944
945
946
947	/*
948	*
949	* The NP (no position) versions.
950	*
951	*/
952
953
954
955	RTDECL(void ) RTMemEfTmpAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
956	{
957	return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
958	}
959
960
961	RTDECL(void ) RTMemEfTmpAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
962	{
963	return rtR3MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
964	}
965
966
967	RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF
968	{
969	if (pv)
970	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL);
971	}
972
973
974	RTDECL(void) RTMemEfTmpFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF
975	{
976	if (pv)
977	rtR3MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL);
978	}
979
980
981	RTDECL(void ) RTMemEfAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
982	{
983	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
984	}
985
986
987	RTDECL(void ) RTMemEfAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
988	{
989	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
990	}
991
992
993	RTDECL(void ) RTMemEfAllocVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
994	{
995	size_t cbAligned;
996	if (cbUnaligned >= 16)
997	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
998	else
999	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
1000	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1001	}
1002
1003
1004	RTDECL(void ) RTMemEfAllocZVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
1005	{
1006	size_t cbAligned;
1007	if (cbUnaligned >= 16)
1008	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
1009	else
1010	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
1011	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1012	}
1013
1014
1015	RTDECL(void ) RTMemEfReallocNP(void pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
1016	{
1017	return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1018	}
1019
1020
1021	RTDECL(void ) RTMemEfReallocZNP(void pvOld, size_t cbOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
1022	{
1023	void *pvDst = rtR3MemRealloc("ReallocZ", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1024	if (pvDst && cbNew > cbOld)
1025	memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld);
1026	return pvDst;
1027	}
1028
1029
1030	RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF
1031	{
1032	if (pv)
1033	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL);
1034	}
1035
1036
1037	RTDECL(void) RTMemEfFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF
1038	{
1039	if (pv)
1040	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL);
1041	}
1042
1043
1044	RTDECL(void ) RTMemEfDupNP(const void pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF
1045	{
1046	void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL);
1047	if (pvDst)
1048	memcpy(pvDst, pvSrc, cb);
1049	return pvDst;
1050	}
1051
1052
1053	RTDECL(void ) RTMemEfDupExNP(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF
1054	{
1055	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL);
1056	if (pvDst)
1057	{
1058	memcpy(pvDst, pvSrc, cbSrc);
1059	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
1060	}
1061	return pvDst;
1062	}
1063

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r3/alloc-ef.cpp@ 86186

Download in other formats: