memcache.cpp@ 47428

Last change on this file since 47428 was 46257, checked in by vboxsync, 12 years ago
memcache.cpp: Optimization in the growth code. Drop strictness regarding no in-use entries on destruction.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 18.8 KB

Line
1	/* $Id: memcache.cpp 46257 2013-05-24 19:18:27Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Object Allocation Cache.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include <iprt/memcache.h>
32	#include "internal/iprt.h"
33
34	#include <iprt/assert.h>
35	#include <iprt/asm.h>
36	#include <iprt/critsect.h>
37	#include <iprt/err.h>
38	#include <iprt/mem.h>
39	#include <iprt/param.h>
40
41	#include "internal/magics.h"
42
43
44	/*******************************************************************************
45	* Structures and Typedefs *
46	*******************************************************************************/
47	/** Pointer to a cache instance. */
48	typedef struct RTMEMCACHEINT *PRTMEMCACHEINT;
49	/** Pointer to a cache page. */
50	typedef struct RTMEMCACHEPAGE *PRTMEMCACHEPAGE;
51
52
53
54	/**
55	* A free object.
56	*
57	* @remarks This only works if the objects don't have a constructor or
58	* destructor and are big enough.
59	*/
60	typedef struct RTMEMCACHEFREEOBJ
61	{
62	/** Pointer to the next free object */
63	struct RTMEMCACHEFREEOBJ * volatile pNext;
64	} RTMEMCACHEFREEOBJ;
65	/** Pointer to a free object. */
66	typedef RTMEMCACHEFREEOBJ *PRTMEMCACHEFREEOBJ;
67
68
69	/**
70	* A cache page.
71	*
72	* This is a page of memory that we split up in to a bunch object sized chunks
73	* and hand out to the cache users. The bitmap is updated in an atomic fashion
74	* so that we don't have to take any locks when freeing or allocating memory.
75	*/
76	typedef struct RTMEMCACHEPAGE
77	{
78	/** Pointer to the cache owning this page.
79	* This is used for validation purposes only. */
80	PRTMEMCACHEINT pCache;
81	/** Pointer to the next page.
82	* This is marked as volatile since we'll be adding new entries to the list
83	* without taking any locks. */
84	PRTMEMCACHEPAGE volatile pNext;
85	/** Bitmap tracking allocated blocks. */
86	void volatile *pbmAlloc;
87	/** Bitmap tracking which blocks that has been thru the constructor. */
88	void volatile *pbmCtor;
89	/** Pointer to the object array.. */
90	uint8_t *pbObjects;
91	/** The number of objects on this page. */
92	uint32_t cObjects;
93
94	/** Padding to force cFree into the next cache line. (ASSUMES CL = 64) */
95	uint8_t abPadding[ARCH_BITS == 32 ? 64 - 64 : 64 - 58 - 4];
96	/** The number of free objects. */
97	int32_t volatile cFree;
98	} RTMEMCACHEPAGE;
99	AssertCompileMemberOffset(RTMEMCACHEPAGE, cFree, 64);
100
101
102	/**
103	* Memory object cache instance.
104	*/
105	typedef struct RTMEMCACHEINT
106	{
107	/** Magic value (RTMEMCACHE_MAGIC). */
108	uint32_t u32Magic;
109	/** The object size. */
110	uint32_t cbObject;
111	/** Object alignment. */
112	uint32_t cbAlignment;
113	/** The per page object count. */
114	uint32_t cPerPage;
115	/** Number of bits in the bitmap.
116	* @remarks This is higher or equal to cPerPage and it is aligned such that
117	* the search operation will be most efficient on x86/AMD64. */
118	uint32_t cBits;
119	/** The maximum number of objects. */
120	uint32_t cMax;
121	/** Whether to the use the free list or not. */
122	bool fUseFreeList;
123	/** Head of the page list. */
124	PRTMEMCACHEPAGE pPageHead;
125	/** Poiner to the insertion point in the page list. */
126	PRTMEMCACHEPAGE volatile *ppPageNext;
127	/** Constructor callback. */
128	PFNMEMCACHECTOR pfnCtor;
129	/** Destructor callback. */
130	PFNMEMCACHEDTOR pfnDtor;
131	/** Callback argument. */
132	void *pvUser;
133	/** Critical section serializing page allocation and similar. */
134	RTCRITSECT CritSect;
135
136	/** The total object count. */
137	uint32_t volatile cTotal;
138	/** The number of free objects. */
139	int32_t volatile cFree;
140	/** This may point to a page with free entries. */
141	PRTMEMCACHEPAGE volatile pPageHint;
142	/** Stack of free items.
143	* These are marked as used in the allocation bitmaps.
144	*
145	* @todo This doesn't scale well when several threads are beating on the
146	* cache. Also, it totally doesn't work when the objects are too
147	* small. */
148	PRTMEMCACHEFREEOBJ volatile pFreeTop;
149	} RTMEMCACHEINT;
150
151
152
153	RTDECL(int) RTMemCacheCreate(PRTMEMCACHE phMemCache, size_t cbObject, size_t cbAlignment, uint32_t cMaxObjects,
154	PFNMEMCACHECTOR pfnCtor, PFNMEMCACHEDTOR pfnDtor, void *pvUser, uint32_t fFlags)
155
156	{
157	AssertPtr(phMemCache);
158	AssertPtrNull(pfnCtor);
159	AssertPtrNull(pfnDtor);
160	AssertReturn(!pfnDtor \|\| pfnCtor, VERR_INVALID_PARAMETER);
161	AssertReturn(cbObject > 0, VERR_INVALID_PARAMETER);
162	AssertReturn(cbObject <= PAGE_SIZE / 8, VERR_INVALID_PARAMETER);
163	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
164
165	if (cbAlignment == 0)
166	{
167	if (cbObject <= 2)
168	cbAlignment = cbObject;
169	else if (cbObject <= 4)
170	cbAlignment = 4;
171	else if (cbObject <= 8)
172	cbAlignment = 8;
173	else if (cbObject <= 16)
174	cbAlignment = 16;
175	else if (cbObject <= 32)
176	cbAlignment = 32;
177	else
178	cbAlignment = 64;
179	}
180	else
181	{
182	AssertReturn(!((cbAlignment - 1) & cbAlignment), VERR_NOT_POWER_OF_TWO);
183	AssertReturn(cbAlignment <= 64, VERR_OUT_OF_RANGE);
184	}
185
186	/*
187	* Allocate and initialize the instance memory.
188	*/
189	RTMEMCACHEINT pThis = (RTMEMCACHEINT )RTMemAlloc(sizeof(*pThis));
190	if (!pThis)
191	return VERR_NO_MEMORY;
192	int rc = RTCritSectInit(&pThis->CritSect);
193	if (RT_FAILURE(rc))
194	{
195	RTMemFree(pThis);
196	return rc;
197	}
198
199	pThis->u32Magic = RTMEMCACHE_MAGIC;
200	pThis->cbObject = (uint32_t)RT_ALIGN_Z(cbObject, cbAlignment);
201	pThis->cbAlignment = (uint32_t)cbAlignment;
202	pThis->cPerPage = (uint32_t)((PAGE_SIZE - RT_ALIGN_Z(sizeof(RTMEMCACHEPAGE), cbAlignment)) / pThis->cbObject);
203	while ( RT_ALIGN_Z(sizeof(RTMEMCACHEPAGE), 8)
204	+ pThis->cPerPage * pThis->cbObject
205	+ RT_ALIGN(pThis->cPerPage, 64) / 8 * 2
206	> PAGE_SIZE)
207	pThis->cPerPage--;
208	pThis->cBits = RT_ALIGN(pThis->cPerPage, 64);
209	pThis->cMax = cMaxObjects;
210	pThis->fUseFreeList = cbObject >= sizeof(RTMEMCACHEFREEOBJ)
211	&& !pfnCtor
212	&& !pfnDtor;
213	pThis->pPageHead = NULL;
214	pThis->ppPageNext = &pThis->pPageHead;
215	pThis->pfnCtor = pfnCtor;
216	pThis->pfnDtor = pfnDtor;
217	pThis->pvUser = pvUser;
218	pThis->cTotal = 0;
219	pThis->cFree = 0;
220	pThis->pPageHint = NULL;
221	pThis->pFreeTop = NULL;
222
223	/** @todo
224	* Here is a puzzler (or maybe I'm just blind), the free list code breaks
225	* badly on my macbook pro (i7) (32-bit).
226	*
227	* I tried changing the reads from unordered to ordered to no avail. Then I
228	* tried optimizing the code with the ASMAtomicCmpXchgExPtr function to
229	* avoid some reads - no change. Inserting pause instructions did nothing
230	* (as expected). The only thing which seems to make a difference is
231	* reading the pFreeTop pointer twice in the free code... This is weird or I'm
232	* overlooking something..
233	*
234	* No time to figure it out, so I'm disabling the broken code paths for
235	* now. */
236	pThis->fUseFreeList = false;
237
238	*phMemCache = pThis;
239	return VINF_SUCCESS;
240	}
241
242
243	RTDECL(int) RTMemCacheDestroy(RTMEMCACHE hMemCache)
244	{
245	RTMEMCACHEINT *pThis = hMemCache;
246	if (!pThis)
247	return VINF_SUCCESS;
248	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
249	AssertReturn(pThis->u32Magic == RTMEMCACHE_MAGIC, VERR_INVALID_HANDLE);
250
251	#if 0 /def RT_STRICT - don't require eveything to be freed. Caches are very convenient for lazy cleanup. /
252	uint32_t cFree = pThis->cFree;
253	for (PRTMEMCACHEFREEOBJ pFree = pThis->pFreeTop; pFree && cFree < pThis->cTotal + 5; pFree = pFree->pNext)
254	cFree++;
255	AssertMsg(cFree == pThis->cTotal, ("cFree=%u cTotal=%u\n", cFree, pThis->cTotal));
256	#endif
257
258	/*
259	* Destroy it.
260	*/
261	AssertReturn(ASMAtomicCmpXchgU32(&pThis->u32Magic, RTMEMCACHE_MAGIC_DEAD, RTMEMCACHE_MAGIC), VERR_INVALID_HANDLE);
262	RTCritSectDelete(&pThis->CritSect);
263
264	while (pThis->pPageHead)
265	{
266	PRTMEMCACHEPAGE pPage = pThis->pPageHead;
267	pThis->pPageHead = pPage->pNext;
268	pPage->cFree = 0;
269
270	if (pThis->pfnDtor)
271	{
272	uint32_t iObj = pPage->cObjects;
273	while (iObj-- > 0)
274	if (ASMBitTestAndClear(pPage->pbmCtor, iObj))
275	pThis->pfnDtor(hMemCache, pPage->pbObjects + iObj * pThis->cbObject, pThis->pvUser);
276	}
277
278	RTMemPageFree(pPage, PAGE_SIZE);
279	}
280
281	RTMemFree(pThis);
282	return VINF_SUCCESS;
283	}
284
285
286	/**
287	* Grows the cache.
288	*
289	* @returns IPRT status code.
290	* @param pThis The memory cache instance.
291	*/
292	static int rtMemCacheGrow(RTMEMCACHEINT *pThis)
293	{
294	/*
295	* Enter the critical section here to avoid allocation races leading to
296	* wasted memory (++) and make it easier to link in the new page.
297	*/
298	RTCritSectEnter(&pThis->CritSect);
299	int rc = VINF_SUCCESS;
300	if (pThis->cFree < 0)
301	{
302	/*
303	* Allocate and initialize the new page.
304	*
305	* We put the constructor bitmap at the lower end right after cFree.
306	* We then push the object array to the end of the page and place the
307	* allocation bitmap below it. The hope is to increase the chance that
308	* the allocation bitmap is in a different cache line than cFree since
309	* this increases performance markably when lots of threads are beating
310	* on the cache.
311	*/
312	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)RTMemPageAlloc(PAGE_SIZE);
313	if (pPage)
314	{
315	uint32_t const cObjects = RT_MIN(pThis->cPerPage, pThis->cMax - pThis->cTotal);
316
317	ASMMemZeroPage(pPage);
318	pPage->pCache = pThis;
319	pPage->pNext = NULL;
320	pPage->cFree = cObjects;
321	pPage->cObjects = cObjects;
322	uint8_t pb = (uint8_t )(pPage + 1);
323	pb = RT_ALIGN_PT(pb, 8, uint8_t *);
324	pPage->pbmCtor = pb;
325	pb = (uint8_t )pPage + PAGE_SIZE - pThis->cbObject cObjects;
326	pPage->pbObjects = pb; Assert(RT_ALIGN_P(pb, pThis->cbAlignment) == pb);
327	pb -= pThis->cBits / 8;
328	pb = (uint8_t *)((uintptr_t)pb & ~(uintptr_t)7);
329	pPage->pbmAlloc = pb;
330	Assert((uintptr_t)pPage->pbmCtor + pThis->cBits / 8 <= (uintptr_t)pPage->pbmAlloc);
331
332	/* Mark the bitmap padding and any unused objects as allocated. */
333	for (uint32_t iBit = cObjects; iBit < pThis->cBits; iBit++)
334	ASMBitSet(pPage->pbmAlloc, iBit);
335
336	/* Make it the hint. */
337	ASMAtomicWritePtr(&pThis->pPageHint, pPage);
338
339	/* Link the page in at the end of the list. */
340	ASMAtomicWritePtr(pThis->ppPageNext, pPage);
341	pThis->ppPageNext = &pPage->pNext;
342
343	/* Add it to the page counts. */
344	ASMAtomicAddS32(&pThis->cFree, cObjects);
345	ASMAtomicAddU32(&pThis->cTotal, cObjects);
346	}
347	else
348	rc = VERR_NO_MEMORY;
349	}
350	RTCritSectLeave(&pThis->CritSect);
351	return rc;
352	}
353
354
355	/**
356	* Grabs a an object in a page.
357	* @returns New cFree value on success (0 or higher), -1 on failure.
358	* @param pPage Pointer to the page.
359	*/
360	DECL_FORCE_INLINE(int32_t) rtMemCacheGrabObj(PRTMEMCACHEPAGE pPage)
361	{
362	int32_t cFreeNew = ASMAtomicDecS32(&pPage->cFree);
363	if (cFreeNew < 0)
364	{
365	ASMAtomicIncS32(&pPage->cFree);
366	return -1;
367	}
368	return cFreeNew;
369	}
370
371
372	RTDECL(int) RTMemCacheAllocEx(RTMEMCACHE hMemCache, void **ppvObj)
373	{
374	RTMEMCACHEINT *pThis = hMemCache;
375	AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
376	AssertReturn(pThis->u32Magic == RTMEMCACHE_MAGIC, VERR_INVALID_PARAMETER);
377
378	/*
379	* Try grab a free object from the stack.
380	*/
381	PRTMEMCACHEFREEOBJ pObj = ASMAtomicUoReadPtrT(&pThis->pFreeTop, PRTMEMCACHEFREEOBJ);
382	if (pObj)
383	{
384	do
385	{
386	PRTMEMCACHEFREEOBJ pNext = ASMAtomicUoReadPtrT(&pObj->pNext, PRTMEMCACHEFREEOBJ);
387	PRTMEMCACHEFREEOBJ pObjOld;
388	if (ASMAtomicCmpXchgExPtr(&pThis->pFreeTop, pNext, pObj, &pObjOld))
389	{
390	Assert(pObjOld == pObj);
391	Assert(pNext != pObjOld);
392	pObj->pNext = NULL;
393	*ppvObj = pObj;
394	return VINF_SUCCESS;
395	}
396	pObj = pObjOld;
397	ASMNopPause();
398	} while (pObj);
399	}
400
401	/*
402	* Try grab a free object at the cache level.
403	*/
404	int32_t cNewFree = ASMAtomicDecS32(&pThis->cFree);
405	if (RT_LIKELY(cNewFree < 0))
406	{
407	uint32_t cTotal = ASMAtomicUoReadU32(&pThis->cTotal);
408	if ( (uint32_t)(cTotal + -cNewFree) > pThis->cMax
409	\|\| (uint32_t)(cTotal + -cNewFree) <= cTotal)
410	{
411	ASMAtomicIncS32(&pThis->cFree);
412	return VERR_MEM_CACHE_MAX_SIZE;
413	}
414
415	int rc = rtMemCacheGrow(pThis);
416	if (RT_FAILURE(rc))
417	{
418	ASMAtomicIncS32(&pThis->cFree);
419	return rc;
420	}
421	}
422
423	/*
424	* Grab a free object at the page level.
425	*/
426	PRTMEMCACHEPAGE pPage = ASMAtomicReadPtrT(&pThis->pPageHint, PRTMEMCACHEPAGE);
427	int32_t iObj = pPage ? rtMemCacheGrabObj(pPage) : -1;
428	if (iObj < 0)
429	{
430	for (unsigned cLoops = 0; ; cLoops++)
431	{
432	for (pPage = pThis->pPageHead; pPage; pPage = pPage->pNext)
433	{
434	iObj = rtMemCacheGrabObj(pPage);
435	if (iObj >= 0)
436	{
437	if (iObj > 0)
438	ASMAtomicWritePtr(&pThis->pPageHint, pPage);
439	break;
440	}
441	}
442	if (iObj >= 0)
443	break;
444	Assert(cLoops != 2);
445	Assert(cLoops < 10);
446	}
447	}
448	Assert(iObj >= 0);
449	Assert((uint32_t)iObj < pThis->cMax);
450
451	/*
452	* Find a free object in the allocation bitmap. Use the new cFree count
453	* as a hint.
454	*/
455	if (ASMAtomicBitTestAndSet(pPage->pbmAlloc, iObj))
456	{
457	for (unsigned cLoops2 = 0;; cLoops2++)
458	{
459	iObj = ASMBitFirstClear(pPage->pbmAlloc, pThis->cBits);
460	if (RT_LIKELY(iObj >= 0))
461	{
462	if (!ASMAtomicBitTestAndSet(pPage->pbmAlloc, iObj))
463	break;
464	}
465	else
466	ASMMemoryFence();
467	Assert(cLoops2 != 40);
468	}
469	Assert(iObj >= 0);
470	}
471	void pvObj = &pPage->pbObjects[iObj pThis->cbObject];
472	Assert((uintptr_t)pvObj - (uintptr_t)pPage < PAGE_SIZE);
473
474	/*
475	* Call the constructor?
476	*/
477	if ( pThis->pfnCtor
478	&& !ASMAtomicBitTestAndSet(pPage->pbmCtor, iObj))
479	{
480	int rc = pThis->pfnCtor(hMemCache, pvObj, pThis->pvUser);
481	if (RT_FAILURE(rc))
482	{
483	ASMAtomicBitClear(pPage->pbmCtor, iObj);
484	RTMemCacheFree(pThis, pvObj);
485	return rc;
486	}
487	}
488
489	*ppvObj = pvObj;
490	return VINF_SUCCESS;
491	}
492
493
494	RTDECL(void *) RTMemCacheAlloc(RTMEMCACHE hMemCache)
495	{
496	void *pvObj;
497	int rc = RTMemCacheAllocEx(hMemCache, &pvObj);
498	if (RT_SUCCESS(rc))
499	return pvObj;
500	return NULL;
501	}
502
503
504	RTDECL(void) RTMemCacheFree(RTMEMCACHE hMemCache, void *pvObj)
505	{
506	if (!pvObj)
507	return;
508
509	RTMEMCACHEINT *pThis = hMemCache;
510	AssertPtrReturnVoid(pThis);
511	AssertReturnVoid(pThis->u32Magic == RTMEMCACHE_MAGIC);
512
513	AssertPtr(pvObj);
514	Assert(RT_ALIGN_P(pvObj, pThis->cbAlignment) == pvObj);
515
516	if (pThis->fUseFreeList)
517	{
518	# ifdef RT_STRICT
519	/* This is the same as the other branch, except it's not actually freed. */
520	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)(((uintptr_t)pvObj) & ~(uintptr_t)PAGE_OFFSET_MASK);
521	Assert(pPage->pCache == pThis);
522	Assert(ASMAtomicUoReadS32(&pPage->cFree) < (int32_t)pThis->cPerPage);
523	uintptr_t offObj = (uintptr_t)pvObj - (uintptr_t)pPage->pbObjects;
524	uintptr_t iObj = offObj / pThis->cbObject;
525	Assert(iObj * pThis->cbObject == offObj);
526	Assert(iObj < pThis->cPerPage);
527	AssertReturnVoid(ASMBitTest(pPage->pbmAlloc, (int32_t)iObj));
528	# endif
529
530	/*
531	* Push it onto the free stack.
532	*/
533	PRTMEMCACHEFREEOBJ pObj = (PRTMEMCACHEFREEOBJ)pvObj;
534	PRTMEMCACHEFREEOBJ pNext = ASMAtomicUoReadPtrT(&pThis->pFreeTop, PRTMEMCACHEFREEOBJ);
535	PRTMEMCACHEFREEOBJ pFreeTopOld;
536	pObj->pNext = pNext;
537	while (!ASMAtomicCmpXchgExPtr(&pThis->pFreeTop, pObj, pNext, &pFreeTopOld))
538	{
539	pNext = pFreeTopOld;
540	Assert(pNext != pObj);
541	pObj->pNext = pNext;
542	ASMNopPause();
543	}
544	}
545	else
546	{
547	/* Note: Do NOT attempt to poison the object! */
548
549	/*
550	* Find the cache page. The page structure is at the start of the page.
551	*/
552	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)(((uintptr_t)pvObj) & ~(uintptr_t)PAGE_OFFSET_MASK);
553	Assert(pPage->pCache == pThis);
554	Assert(ASMAtomicUoReadS32(&pPage->cFree) < (int32_t)pThis->cPerPage);
555
556	/*
557	* Clear the bitmap bit and update the two object counter. Order matters!
558	*/
559	uintptr_t offObj = (uintptr_t)pvObj - (uintptr_t)pPage->pbObjects;
560	uintptr_t iObj = offObj / pThis->cbObject;
561	Assert(iObj * pThis->cbObject == offObj);
562	Assert(iObj < pThis->cPerPage);
563	AssertReturnVoid(ASMAtomicBitTestAndClear(pPage->pbmAlloc, iObj));
564
565	ASMAtomicIncS32(&pPage->cFree);
566	ASMAtomicIncS32(&pThis->cFree);
567	}
568	}
569

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/alloc/memcache.cpp@ 47428

Download in other formats: