memcache.cpp@ 42072

Last change on this file since 42072 was 41783, checked in by vboxsync, 13 years ago
Doxygen, comment typos.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 18.9 KB

Line
1	/* $Id: memcache.cpp 41783 2012-06-16 19:24:15Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Object Allocation Cache.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include <iprt/memcache.h>
32	#include "internal/iprt.h"
33
34	#include <iprt/assert.h>
35	#include <iprt/asm.h>
36	#include <iprt/critsect.h>
37	#include <iprt/err.h>
38	#include <iprt/mem.h>
39	#include <iprt/param.h>
40
41	#include "internal/magics.h"
42
43
44	/*******************************************************************************
45	* Structures and Typedefs *
46	*******************************************************************************/
47	/** Pointer to a cache instance. */
48	typedef struct RTMEMCACHEINT *PRTMEMCACHEINT;
49	/** Pointer to a cache page. */
50	typedef struct RTMEMCACHEPAGE *PRTMEMCACHEPAGE;
51
52
53
54	/**
55	* A free object.
56	*
57	* @remarks This only works if the objects don't have a constructor or
58	* destructor and are big enough.
59	*/
60	typedef struct RTMEMCACHEFREEOBJ
61	{
62	/** Pointer to the next free object */
63	struct RTMEMCACHEFREEOBJ * volatile pNext;
64	} RTMEMCACHEFREEOBJ;
65	/** Pointer to a free object. */
66	typedef RTMEMCACHEFREEOBJ *PRTMEMCACHEFREEOBJ;
67
68
69	/**
70	* A cache page.
71	*
72	* This is a page of memory that we split up in to a bunch object sized chunks
73	* and hand out to the cache users. The bitmap is updated in an atomic fashion
74	* so that we don't have to take any locks when freeing or allocating memory.
75	*/
76	typedef struct RTMEMCACHEPAGE
77	{
78	/** Pointer to the cache owning this page.
79	* This is used for validation purposes only. */
80	PRTMEMCACHEINT pCache;
81	/** Pointer to the next page.
82	* This is marked as volatile since we'll be adding new entries to the list
83	* without taking any locks. */
84	PRTMEMCACHEPAGE volatile pNext;
85	/** Bitmap tracking allocated blocks. */
86	void volatile *pbmAlloc;
87	/** Bitmap tracking which blocks that has been thru the constructor. */
88	void volatile *pbmCtor;
89	/** Pointer to the object array.. */
90	uint8_t *pbObjects;
91	/** The number of objects on this page. */
92	uint32_t cObjects;
93
94	/** Padding to force cFree into the next cache line. (ASSUMES CL = 64) */
95	uint8_t abPadding[ARCH_BITS == 32 ? 64 - 64 : 64 - 58 - 4];
96	/** The number of free objects. */
97	int32_t volatile cFree;
98	} RTMEMCACHEPAGE;
99	AssertCompileMemberOffset(RTMEMCACHEPAGE, cFree, 64);
100
101
102	/**
103	* Memory object cache instance.
104	*/
105	typedef struct RTMEMCACHEINT
106	{
107	/** Magic value (RTMEMCACHE_MAGIC). */
108	uint32_t u32Magic;
109	/** The object size. */
110	uint32_t cbObject;
111	/** Object alignment. */
112	uint32_t cbAlignment;
113	/** The per page object count. */
114	uint32_t cPerPage;
115	/** Number of bits in the bitmap.
116	* @remarks This is higher or equal to cPerPage and it is aligned such that
117	* the search operation will be most efficient on x86/AMD64. */
118	uint32_t cBits;
119	/** The maximum number of objects. */
120	uint32_t cMax;
121	/** Whether to the use the free list or not. */
122	bool fUseFreeList;
123	/** Head of the page list. */
124	PRTMEMCACHEPAGE pPageHead;
125	/** Constructor callback. */
126	PFNMEMCACHECTOR pfnCtor;
127	/** Destructor callback. */
128	PFNMEMCACHEDTOR pfnDtor;
129	/** Callback argument. */
130	void *pvUser;
131	/** Critical section serializing page allocation and similar. */
132	RTCRITSECT CritSect;
133
134	/** The total object count. */
135	uint32_t volatile cTotal;
136	/** The number of free objects. */
137	int32_t volatile cFree;
138	/** This may point to a page with free entries. */
139	PRTMEMCACHEPAGE volatile pPageHint;
140	/** Stack of free items.
141	* These are marked as used in the allocation bitmaps.
142	*
143	* @todo This doesn't scale well when several threads are beating on the
144	* cache. Also, it totally doesn't work when we've got a
145	* constructor/destructor around or the objects are too small. */
146	PRTMEMCACHEFREEOBJ volatile pFreeTop;
147	} RTMEMCACHEINT;
148
149
150
151	RTDECL(int) RTMemCacheCreate(PRTMEMCACHE phMemCache, size_t cbObject, size_t cbAlignment, uint32_t cMaxObjects,
152	PFNMEMCACHECTOR pfnCtor, PFNMEMCACHEDTOR pfnDtor, void *pvUser, uint32_t fFlags)
153
154	{
155	AssertPtr(phMemCache);
156	AssertPtrNull(pfnCtor);
157	AssertPtrNull(pfnDtor);
158	AssertReturn(!pfnDtor \|\| pfnCtor, VERR_INVALID_PARAMETER);
159	AssertReturn(cbObject > 0, VERR_INVALID_PARAMETER);
160	AssertReturn(cbObject <= PAGE_SIZE / 8, VERR_INVALID_PARAMETER);
161	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
162
163	if (cbAlignment == 0)
164	{
165	if (cbObject <= 2)
166	cbAlignment = cbObject;
167	else if (cbObject <= 4)
168	cbAlignment = 4;
169	else if (cbObject <= 8)
170	cbAlignment = 8;
171	else if (cbObject <= 16)
172	cbAlignment = 16;
173	else if (cbObject <= 32)
174	cbAlignment = 32;
175	else
176	cbAlignment = 64;
177	}
178	else
179	{
180	AssertReturn(!((cbAlignment - 1) & cbAlignment), VERR_NOT_POWER_OF_TWO);
181	AssertReturn(cbAlignment <= 64, VERR_OUT_OF_RANGE);
182	}
183
184	/*
185	* Allocate and initialize the instance memory.
186	*/
187	RTMEMCACHEINT pThis = (RTMEMCACHEINT )RTMemAlloc(sizeof(*pThis));
188	if (!pThis)
189	return VERR_NO_MEMORY;
190	int rc = RTCritSectInit(&pThis->CritSect);
191	if (RT_FAILURE(rc))
192	{
193	RTMemFree(pThis);
194	return rc;
195	}
196
197	pThis->u32Magic = RTMEMCACHE_MAGIC;
198	pThis->cbObject = (uint32_t)RT_ALIGN_Z(cbObject, cbAlignment);
199	pThis->cbAlignment = (uint32_t)cbAlignment;
200	pThis->cPerPage = (uint32_t)((PAGE_SIZE - RT_ALIGN_Z(sizeof(RTMEMCACHEPAGE), cbAlignment)) / pThis->cbObject);
201	while ( RT_ALIGN_Z(sizeof(RTMEMCACHEPAGE), 8)
202	+ pThis->cPerPage * pThis->cbObject
203	+ RT_ALIGN(pThis->cPerPage, 64) / 8 * 2
204	> PAGE_SIZE)
205	pThis->cPerPage--;
206	pThis->cBits = RT_ALIGN(pThis->cPerPage, 64);
207	pThis->cMax = cMaxObjects;
208	pThis->fUseFreeList = cbObject >= sizeof(RTMEMCACHEFREEOBJ)
209	&& !pfnCtor
210	&& !pfnDtor;
211	pThis->pPageHead = NULL;
212	pThis->pfnCtor = pfnCtor;
213	pThis->pfnDtor = pfnDtor;
214	pThis->pvUser = pvUser;
215	pThis->cTotal = 0;
216	pThis->cFree = 0;
217	pThis->pPageHint = NULL;
218	pThis->pFreeTop = NULL;
219
220	/** @todo
221	* Here is a puzzler (or maybe I'm just blind), the free list code breaks
222	* badly on my macbook pro (i7) (32-bit).
223	*
224	* I tried changing the reads from unordered to ordered to no avail. Then I
225	* tried optimizing the code with the ASMAtomicCmpXchgExPtr function to
226	* avoid some reads - no change. Inserting pause instructions did nothing
227	* (as expected). The only thing which seems to make a difference is
228	* reading the pFreeTop pointer twice in the free code... This is weird or I'm
229	* overlooking something..
230	*
231	* No time to figure it out, so I'm disabling the broken code paths for
232	* now. */
233	pThis->fUseFreeList = false;
234
235	*phMemCache = pThis;
236	return VINF_SUCCESS;
237	}
238
239
240	RTDECL(int) RTMemCacheDestroy(RTMEMCACHE hMemCache)
241	{
242	RTMEMCACHEINT *pThis = hMemCache;
243	if (!pThis)
244	return VINF_SUCCESS;
245	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
246	AssertReturn(pThis->u32Magic == RTMEMCACHE_MAGIC, VERR_INVALID_HANDLE);
247	#ifdef RT_STRICT
248	uint32_t cFree = pThis->cFree;
249	for (PRTMEMCACHEFREEOBJ pFree = pThis->pFreeTop; pFree && cFree < pThis->cTotal + 5; pFree = pFree->pNext)
250	cFree++;
251	AssertMsg(cFree == pThis->cTotal, ("cFree=%u cTotal=%u\n", cFree, pThis->cTotal));
252	#endif
253
254	/*
255	* Destroy it.
256	*/
257	AssertReturn(ASMAtomicCmpXchgU32(&pThis->u32Magic, RTMEMCACHE_MAGIC_DEAD, RTMEMCACHE_MAGIC), VERR_INVALID_HANDLE);
258	RTCritSectDelete(&pThis->CritSect);
259
260	while (pThis->pPageHead)
261	{
262	PRTMEMCACHEPAGE pPage = pThis->pPageHead;
263	pThis->pPageHead = pPage->pNext;
264	pPage->cFree = 0;
265
266	if (pThis->pfnDtor)
267	{
268	uint32_t iObj = pPage->cObjects;
269	while (iObj-- > 0)
270	if (ASMBitTestAndClear(pPage->pbmCtor, iObj))
271	pThis->pfnDtor(hMemCache, pPage->pbObjects + iObj * pThis->cbObject, pThis->pvUser);
272	}
273
274	RTMemPageFree(pPage, PAGE_SIZE);
275	}
276
277	RTMemFree(pThis);
278	return VINF_SUCCESS;
279	}
280
281
282	/**
283	* Grows the cache.
284	*
285	* @returns IPRT status code.
286	* @param pThis The memory cache instance.
287	*/
288	static int rtMemCacheGrow(RTMEMCACHEINT *pThis)
289	{
290	/*
291	* Enter the critical section here to avoid allocation races leading to
292	* wasted memory (++) and make it easier to link in the new page.
293	*/
294	RTCritSectEnter(&pThis->CritSect);
295	int rc = VINF_SUCCESS;
296	if (pThis->cFree < 0)
297	{
298	/*
299	* Allocate and initialize the new page.
300	*
301	* We put the constructor bitmap at the lower end right after cFree.
302	* We then push the object array to the end of the page and place the
303	* allocation bitmap below it. The hope is to increase the chance that
304	* the allocation bitmap is in a different cache line than cFree since
305	* this increases performance markably when lots of threads are beating
306	* on the cache.
307	*/
308	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)RTMemPageAlloc(PAGE_SIZE);
309	if (pPage)
310	{
311	uint32_t const cObjects = RT_MIN(pThis->cPerPage, pThis->cMax - pThis->cTotal);
312
313	ASMMemZeroPage(pPage);
314	pPage->pCache = pThis;
315	pPage->pNext = NULL;
316	pPage->cFree = cObjects;
317	pPage->cObjects = cObjects;
318	uint8_t pb = (uint8_t )(pPage + 1);
319	pb = RT_ALIGN_PT(pb, 8, uint8_t *);
320	pPage->pbmCtor = pb;
321	pb = (uint8_t )pPage + PAGE_SIZE - pThis->cbObject cObjects;
322	pPage->pbObjects = pb; Assert(RT_ALIGN_P(pb, pThis->cbAlignment) == pb);
323	pb -= pThis->cBits / 8;
324	pb = (uint8_t *)((uintptr_t)pb & ~(uintptr_t)7);
325	pPage->pbmAlloc = pb;
326	Assert((uintptr_t)pPage->pbmCtor + pThis->cBits / 8 <= (uintptr_t)pPage->pbmAlloc);
327
328	/* Mark the bitmap padding and any unused objects as allocated. */
329	for (uint32_t iBit = cObjects; iBit < pThis->cBits; iBit++)
330	ASMBitSet(pPage->pbmAlloc, iBit);
331
332	/* Make it the hint. */
333	ASMAtomicWritePtr(&pThis->pPageHint, pPage);
334
335	/* Link the page. */
336	PRTMEMCACHEPAGE pPrevPage = pThis->pPageHead;
337	if (!pPrevPage)
338	ASMAtomicWritePtr(&pThis->pPageHead, pPage);
339	else
340	{
341	while (pPrevPage->pNext)
342	pPrevPage = pPrevPage->pNext;
343	ASMAtomicWritePtr(&pPrevPage->pNext, pPage);
344	}
345
346	/* Add it to the page counts. */
347	ASMAtomicAddS32(&pThis->cFree, cObjects);
348	ASMAtomicAddU32(&pThis->cTotal, cObjects);
349	}
350	else
351	rc = VERR_NO_MEMORY;
352	}
353	RTCritSectLeave(&pThis->CritSect);
354	return rc;
355	}
356
357
358	/**
359	* Grabs a an object in a page.
360	* @returns New cFree value on success (0 or higher), -1 on failure.
361	* @param pPage Pointer to the page.
362	*/
363	DECL_FORCE_INLINE(int32_t) rtMemCacheGrabObj(PRTMEMCACHEPAGE pPage)
364	{
365	int32_t cFreeNew = ASMAtomicDecS32(&pPage->cFree);
366	if (cFreeNew < 0)
367	{
368	ASMAtomicIncS32(&pPage->cFree);
369	return -1;
370	}
371	return cFreeNew;
372	}
373
374
375	RTDECL(int) RTMemCacheAllocEx(RTMEMCACHE hMemCache, void **ppvObj)
376	{
377	RTMEMCACHEINT *pThis = hMemCache;
378	AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
379	AssertReturn(pThis->u32Magic == RTMEMCACHE_MAGIC, VERR_INVALID_PARAMETER);
380
381	/*
382	* Try grab a free object from the stack.
383	*/
384	PRTMEMCACHEFREEOBJ pObj = ASMAtomicUoReadPtrT(&pThis->pFreeTop, PRTMEMCACHEFREEOBJ);
385	if (pObj)
386	{
387	do
388	{
389	PRTMEMCACHEFREEOBJ pNext = ASMAtomicUoReadPtrT(&pObj->pNext, PRTMEMCACHEFREEOBJ);
390	PRTMEMCACHEFREEOBJ pObjOld;
391	if (ASMAtomicCmpXchgExPtr(&pThis->pFreeTop, pNext, pObj, &pObjOld))
392	{
393	Assert(pObjOld == pObj);
394	Assert(pNext != pObjOld);
395	pObj->pNext = NULL;
396	*ppvObj = pObj;
397	return VINF_SUCCESS;
398	}
399	pObj = pObjOld;
400	ASMNopPause();
401	} while (pObj);
402	}
403
404	/*
405	* Try grab a free object at the cache level.
406	*/
407	int32_t cNewFree = ASMAtomicDecS32(&pThis->cFree);
408	if (RT_LIKELY(cNewFree < 0))
409	{
410	uint32_t cTotal = ASMAtomicUoReadU32(&pThis->cTotal);
411	if ( (uint32_t)(cTotal + -cNewFree) > pThis->cMax
412	\|\| (uint32_t)(cTotal + -cNewFree) <= cTotal)
413	{
414	ASMAtomicIncS32(&pThis->cFree);
415	return VERR_MEM_CACHE_MAX_SIZE;
416	}
417
418	int rc = rtMemCacheGrow(pThis);
419	if (RT_FAILURE(rc))
420	{
421	ASMAtomicIncS32(&pThis->cFree);
422	return rc;
423	}
424	}
425
426	/*
427	* Grab a free object at the page level.
428	*/
429	PRTMEMCACHEPAGE pPage = ASMAtomicReadPtrT(&pThis->pPageHint, PRTMEMCACHEPAGE);
430	int32_t iObj = pPage ? rtMemCacheGrabObj(pPage) : -1;
431	if (iObj < 0)
432	{
433	for (unsigned cLoops = 0; ; cLoops++)
434	{
435	for (pPage = pThis->pPageHead; pPage; pPage = pPage->pNext)
436	{
437	iObj = rtMemCacheGrabObj(pPage);
438	if (iObj >= 0)
439	{
440	if (iObj > 0)
441	ASMAtomicWritePtr(&pThis->pPageHint, pPage);
442	break;
443	}
444	}
445	if (iObj >= 0)
446	break;
447	Assert(cLoops != 2);
448	Assert(cLoops < 10);
449	}
450	}
451	Assert(iObj >= 0);
452	Assert((uint32_t)iObj < pThis->cMax);
453
454	/*
455	* Find a free object in the allocation bitmap. Use the new cFree count
456	* as a hint.
457	*/
458	if (ASMAtomicBitTestAndSet(pPage->pbmAlloc, iObj))
459	{
460	for (unsigned cLoops2 = 0;; cLoops2++)
461	{
462	iObj = ASMBitFirstClear(pPage->pbmAlloc, pThis->cBits);
463	if (RT_LIKELY(iObj >= 0))
464	{
465	if (!ASMAtomicBitTestAndSet(pPage->pbmAlloc, iObj))
466	break;
467	}
468	else
469	ASMMemoryFence();
470	Assert(cLoops2 != 40);
471	}
472	Assert(iObj >= 0);
473	}
474	void pvObj = &pPage->pbObjects[iObj pThis->cbObject];
475	Assert((uintptr_t)pvObj - (uintptr_t)pPage < PAGE_SIZE);
476
477	/*
478	* Call the constructor?
479	*/
480	if ( pThis->pfnCtor
481	&& !ASMAtomicBitTestAndSet(pPage->pbmCtor, iObj))
482	{
483	int rc = pThis->pfnCtor(hMemCache, pvObj, pThis->pvUser);
484	if (RT_FAILURE(rc))
485	{
486	ASMAtomicBitClear(pPage->pbmCtor, iObj);
487	RTMemCacheFree(pThis, pvObj);
488	return rc;
489	}
490	}
491
492	*ppvObj = pvObj;
493	return VINF_SUCCESS;
494	}
495
496
497	RTDECL(void *) RTMemCacheAlloc(RTMEMCACHE hMemCache)
498	{
499	void *pvObj;
500	int rc = RTMemCacheAllocEx(hMemCache, &pvObj);
501	if (RT_SUCCESS(rc))
502	return pvObj;
503	return NULL;
504	}
505
506
507	RTDECL(void) RTMemCacheFree(RTMEMCACHE hMemCache, void *pvObj)
508	{
509	if (!pvObj)
510	return;
511
512	RTMEMCACHEINT *pThis = hMemCache;
513	AssertPtrReturnVoid(pThis);
514	AssertReturnVoid(pThis->u32Magic == RTMEMCACHE_MAGIC);
515
516	AssertPtr(pvObj);
517	Assert(RT_ALIGN_P(pvObj, pThis->cbAlignment) == pvObj);
518
519	if (pThis->fUseFreeList)
520	{
521	# ifdef RT_STRICT
522	/* This is the same as the other branch, except it's not actually freed. */
523	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)(((uintptr_t)pvObj) & ~(uintptr_t)PAGE_OFFSET_MASK);
524	Assert(pPage->pCache == pThis);
525	Assert(ASMAtomicUoReadS32(&pPage->cFree) < (int32_t)pThis->cPerPage);
526	uintptr_t offObj = (uintptr_t)pvObj - (uintptr_t)pPage->pbObjects;
527	uintptr_t iObj = offObj / pThis->cbObject;
528	Assert(iObj * pThis->cbObject == offObj);
529	Assert(iObj < pThis->cPerPage);
530	AssertReturnVoid(ASMBitTest(pPage->pbmAlloc, (int32_t)iObj));
531	# endif
532
533	/*
534	* Push it onto the free stack.
535	*/
536	PRTMEMCACHEFREEOBJ pObj = (PRTMEMCACHEFREEOBJ)pvObj;
537	PRTMEMCACHEFREEOBJ pNext = ASMAtomicUoReadPtrT(&pThis->pFreeTop, PRTMEMCACHEFREEOBJ);
538	PRTMEMCACHEFREEOBJ pFreeTopOld;
539	pObj->pNext = pNext;
540	while (!ASMAtomicCmpXchgExPtr(&pThis->pFreeTop, pObj, pNext, &pFreeTopOld))
541	{
542	pNext = pFreeTopOld;
543	Assert(pNext != pObj);
544	pObj->pNext = pNext;
545	ASMNopPause();
546	}
547	}
548	else
549	{
550	/* Note: Do NOT attempt to poison the object if we have a constructor
551	or/and destructor! */
552
553	/*
554	* Find the cache page. The page structure is at the start of the page.
555	*/
556	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)(((uintptr_t)pvObj) & ~(uintptr_t)PAGE_OFFSET_MASK);
557	Assert(pPage->pCache == pThis);
558	Assert(ASMAtomicUoReadS32(&pPage->cFree) < (int32_t)pThis->cPerPage);
559
560	/*
561	* Clear the bitmap bit and update the two object counter. Order matters!
562	*/
563	uintptr_t offObj = (uintptr_t)pvObj - (uintptr_t)pPage->pbObjects;
564	uintptr_t iObj = offObj / pThis->cbObject;
565	Assert(iObj * pThis->cbObject == offObj);
566	Assert(iObj < pThis->cPerPage);
567	AssertReturnVoid(ASMAtomicBitTestAndClear(pPage->pbmAlloc, iObj));
568
569	ASMAtomicIncS32(&pPage->cFree);
570	ASMAtomicIncS32(&pThis->cFree);
571	}
572	}
573

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/alloc/memcache.cpp@ 42072

Download in other formats: