memobj-r0drv.cpp@ 91449

Last change on this file since 91449 was 91446, checked in by vboxsync, 3 years ago
IPRT/memobj: Adding RTR0MemObjAllocLarge for speeding up large page allocations. bugref:5324
Property svn:eol-style set to `native` Property svn:keywords set to `Id Rev Revision`
File size: 32.1 KB

Line
1	/* $Id: memobj-r0drv.cpp 91446 2021-09-28 19:53:25Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Common Code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#define LOG_GROUP RTLOGGROUP_DEFAULT /// @todo RTLOGGROUP_MEM
32	#define RTMEM_NO_WRAP_TO_EF_APIS /* circular dependency otherwise. */
33	#include <iprt/memobj.h>
34	#include "internal/iprt.h"
35
36	#include <iprt/alloc.h>
37	#include <iprt/asm.h>
38	#include <iprt/assert.h>
39	#include <iprt/err.h>
40	#include <iprt/log.h>
41	#include <iprt/mp.h>
42	#include <iprt/param.h>
43	#include <iprt/process.h>
44	#include <iprt/thread.h>
45
46	#include "internal/memobj.h"
47
48
49	/**
50	* Internal function for allocating a new memory object.
51	*
52	* @returns The allocated and initialized handle.
53	* @param cbSelf The size of the memory object handle. 0 mean default size.
54	* @param enmType The memory object type.
55	* @param pv The memory object mapping.
56	* @param cb The size of the memory object.
57	*/
58	DECLHIDDEN(PRTR0MEMOBJINTERNAL) rtR0MemObjNew(size_t cbSelf, RTR0MEMOBJTYPE enmType, void *pv, size_t cb)
59	{
60	PRTR0MEMOBJINTERNAL pNew;
61
62	/* validate the size */
63	if (!cbSelf)
64	cbSelf = sizeof(*pNew);
65	Assert(cbSelf >= sizeof(*pNew));
66	Assert(cbSelf == (uint32_t)cbSelf);
67	AssertMsg(RT_ALIGN_Z(cb, PAGE_SIZE) == cb, ("%#zx\n", cb));
68
69	/*
70	* Allocate and initialize the object.
71	*/
72	pNew = (PRTR0MEMOBJINTERNAL)RTMemAllocZ(cbSelf);
73	if (pNew)
74	{
75	pNew->u32Magic = RTR0MEMOBJ_MAGIC;
76	pNew->cbSelf = (uint32_t)cbSelf;
77	pNew->enmType = enmType;
78	pNew->fFlags = 0;
79	pNew->cb = cb;
80	pNew->pv = pv;
81	}
82	return pNew;
83	}
84
85
86	/**
87	* Deletes an incomplete memory object.
88	*
89	* This is for cleaning up after failures during object creation.
90	*
91	* @param pMem The incomplete memory object to delete.
92	*/
93	DECLHIDDEN(void) rtR0MemObjDelete(PRTR0MEMOBJINTERNAL pMem)
94	{
95	if (pMem)
96	{
97	ASMAtomicUoWriteU32(&pMem->u32Magic, ~RTR0MEMOBJ_MAGIC);
98	pMem->enmType = RTR0MEMOBJTYPE_END;
99	RTMemFree(pMem);
100	}
101	}
102
103
104	/**
105	* Links a mapping object to a primary object.
106	*
107	* @returns IPRT status code.
108	* @retval VINF_SUCCESS on success.
109	* @retval VINF_NO_MEMORY if we couldn't expand the mapping array of the parent.
110	* @param pParent The parent (primary) memory object.
111	* @param pChild The child (mapping) memory object.
112	*/
113	static int rtR0MemObjLink(PRTR0MEMOBJINTERNAL pParent, PRTR0MEMOBJINTERNAL pChild)
114	{
115	uint32_t i;
116
117	/* sanity */
118	Assert(rtR0MemObjIsMapping(pChild));
119	Assert(!rtR0MemObjIsMapping(pParent));
120
121	/* expand the array? */
122	i = pParent->uRel.Parent.cMappings;
123	if (i >= pParent->uRel.Parent.cMappingsAllocated)
124	{
125	void *pv = RTMemRealloc(pParent->uRel.Parent.papMappings,
126	(i + 32) * sizeof(pParent->uRel.Parent.papMappings[0]));
127	if (!pv)
128	return VERR_NO_MEMORY;
129	pParent->uRel.Parent.papMappings = (PPRTR0MEMOBJINTERNAL)pv;
130	pParent->uRel.Parent.cMappingsAllocated = i + 32;
131	Assert(i == pParent->uRel.Parent.cMappings);
132	}
133
134	/* do the linking. */
135	pParent->uRel.Parent.papMappings[i] = pChild;
136	pParent->uRel.Parent.cMappings++;
137	pChild->uRel.Child.pParent = pParent;
138
139	return VINF_SUCCESS;
140	}
141
142
143	/**
144	* Checks if this is mapping or not.
145	*
146	* @returns true if it's a mapping, otherwise false.
147	* @param MemObj The ring-0 memory object handle.
148	*/
149	RTR0DECL(bool) RTR0MemObjIsMapping(RTR0MEMOBJ MemObj)
150	{
151	/* Validate the object handle. */
152	PRTR0MEMOBJINTERNAL pMem;
153	AssertPtrReturn(MemObj, false);
154	pMem = (PRTR0MEMOBJINTERNAL)MemObj;
155	AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), false);
156	AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), false);
157
158	/* hand it on to the inlined worker. */
159	return rtR0MemObjIsMapping(pMem);
160	}
161	RT_EXPORT_SYMBOL(RTR0MemObjIsMapping);
162
163
164	/**
165	* Gets the address of a ring-0 memory object.
166	*
167	* @returns The address of the memory object.
168	* @returns NULL if the handle is invalid (asserts in strict builds) or if there isn't any mapping.
169	* @param MemObj The ring-0 memory object handle.
170	*/
171	RTR0DECL(void *) RTR0MemObjAddress(RTR0MEMOBJ MemObj)
172	{
173	/* Validate the object handle. */
174	PRTR0MEMOBJINTERNAL pMem;
175	if (RT_UNLIKELY(MemObj == NIL_RTR0MEMOBJ))
176	return NULL;
177	AssertPtrReturn(MemObj, NULL);
178	pMem = (PRTR0MEMOBJINTERNAL)MemObj;
179	AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), NULL);
180	AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), NULL);
181
182	/* return the mapping address. */
183	return pMem->pv;
184	}
185	RT_EXPORT_SYMBOL(RTR0MemObjAddress);
186
187
188	/**
189	* Gets the ring-3 address of a ring-0 memory object.
190	*
191	* This only applies to ring-0 memory object with ring-3 mappings of some kind, i.e.
192	* locked user memory, reserved user address space and user mappings. This API should
193	* not be used on any other objects.
194	*
195	* @returns The address of the memory object.
196	* @returns NIL_RTR3PTR if the handle is invalid or if it's not an object with a ring-3 mapping.
197	* Strict builds will assert in both cases.
198	* @param MemObj The ring-0 memory object handle.
199	*/
200	RTR0DECL(RTR3PTR) RTR0MemObjAddressR3(RTR0MEMOBJ MemObj)
201	{
202	PRTR0MEMOBJINTERNAL pMem;
203
204	/* Validate the object handle. */
205	if (RT_UNLIKELY(MemObj == NIL_RTR0MEMOBJ))
206	return NIL_RTR3PTR;
207	AssertPtrReturn(MemObj, NIL_RTR3PTR);
208	pMem = (PRTR0MEMOBJINTERNAL)MemObj;
209	AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), NIL_RTR3PTR);
210	AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), NIL_RTR3PTR);
211	if (RT_UNLIKELY( ( pMem->enmType != RTR0MEMOBJTYPE_MAPPING
212	\|\| pMem->u.Mapping.R0Process == NIL_RTR0PROCESS)
213	&& ( pMem->enmType != RTR0MEMOBJTYPE_LOCK
214	\|\| pMem->u.Lock.R0Process == NIL_RTR0PROCESS)
215	&& ( pMem->enmType != RTR0MEMOBJTYPE_PHYS_NC
216	\|\| pMem->u.Lock.R0Process == NIL_RTR0PROCESS)
217	&& ( pMem->enmType != RTR0MEMOBJTYPE_RES_VIRT
218	\|\| pMem->u.ResVirt.R0Process == NIL_RTR0PROCESS)))
219	return NIL_RTR3PTR;
220
221	/* return the mapping address. */
222	return (RTR3PTR)pMem->pv;
223	}
224	RT_EXPORT_SYMBOL(RTR0MemObjAddressR3);
225
226
227	/**
228	* Gets the size of a ring-0 memory object.
229	*
230	* The returned value may differ from the one specified to the API creating the
231	* object because of alignment adjustments. The minimal alignment currently
232	* employed by any API is PAGE_SIZE, so the result can safely be shifted by
233	* PAGE_SHIFT to calculate a page count.
234	*
235	* @returns The object size.
236	* @returns 0 if the handle is invalid (asserts in strict builds) or if there isn't any mapping.
237	* @param MemObj The ring-0 memory object handle.
238	*/
239	RTR0DECL(size_t) RTR0MemObjSize(RTR0MEMOBJ MemObj)
240	{
241	PRTR0MEMOBJINTERNAL pMem;
242
243	/* Validate the object handle. */
244	if (RT_UNLIKELY(MemObj == NIL_RTR0MEMOBJ))
245	return 0;
246	AssertPtrReturn(MemObj, 0);
247	pMem = (PRTR0MEMOBJINTERNAL)MemObj;
248	AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), 0);
249	AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), 0);
250	AssertMsg(RT_ALIGN_Z(pMem->cb, PAGE_SIZE) == pMem->cb, ("%#zx\n", pMem->cb));
251
252	/* return the size. */
253	return pMem->cb;
254	}
255	RT_EXPORT_SYMBOL(RTR0MemObjSize);
256
257
258	/**
259	* Get the physical address of an page in the memory object.
260	*
261	* @returns The physical address.
262	* @returns NIL_RTHCPHYS if the object doesn't contain fixed physical pages.
263	* @returns NIL_RTHCPHYS if the iPage is out of range.
264	* @returns NIL_RTHCPHYS if the object handle isn't valid.
265	* @param MemObj The ring-0 memory object handle.
266	* @param iPage The page number within the object.
267	*/
268	/* Work around gcc bug 55940 */
269	#if defined(__GNUC__) && defined(RT_ARCH_X86) && (__GNUC__ * 100 + __GNUC_MINOR__) == 407
270	__attribute__((__optimize__ ("no-shrink-wrap")))
271	#endif
272	RTR0DECL(RTHCPHYS) RTR0MemObjGetPagePhysAddr(RTR0MEMOBJ MemObj, size_t iPage)
273	{
274	/* Validate the object handle. */
275	PRTR0MEMOBJINTERNAL pMem;
276	size_t cPages;
277	AssertPtrReturn(MemObj, NIL_RTHCPHYS);
278	pMem = (PRTR0MEMOBJINTERNAL)MemObj;
279	AssertReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, NIL_RTHCPHYS);
280	AssertReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, NIL_RTHCPHYS);
281	AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), NIL_RTHCPHYS);
282	AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), NIL_RTHCPHYS);
283	cPages = (pMem->cb >> PAGE_SHIFT);
284	if (iPage >= cPages)
285	{
286	/* permit: while (RTR0MemObjGetPagePhysAddr(pMem, iPage++) != NIL_RTHCPHYS) {} */
287	if (iPage == cPages)
288	return NIL_RTHCPHYS;
289	AssertReturn(iPage < (pMem->cb >> PAGE_SHIFT), NIL_RTHCPHYS);
290	}
291
292	/*
293	* We know the address of physically contiguous allocations and mappings.
294	*/
295	if (pMem->enmType == RTR0MEMOBJTYPE_CONT)
296	return pMem->u.Cont.Phys + iPage * PAGE_SIZE;
297	if (pMem->enmType == RTR0MEMOBJTYPE_PHYS)
298	return pMem->u.Phys.PhysBase + iPage * PAGE_SIZE;
299
300	/*
301	* Do the job.
302	*/
303	return rtR0MemObjNativeGetPagePhysAddr(pMem, iPage);
304	}
305	RT_EXPORT_SYMBOL(RTR0MemObjGetPagePhysAddr);
306
307
308	/**
309	* Frees a ring-0 memory object.
310	*
311	* @returns IPRT status code.
312	* @retval VERR_INVALID_HANDLE if
313	* @param MemObj The ring-0 memory object to be freed. NULL is accepted.
314	* @param fFreeMappings Whether or not to free mappings of the object.
315	*/
316	RTR0DECL(int) RTR0MemObjFree(RTR0MEMOBJ MemObj, bool fFreeMappings)
317	{
318	/*
319	* Validate the object handle.
320	*/
321	PRTR0MEMOBJINTERNAL pMem;
322	int rc;
323
324	if (MemObj == NIL_RTR0MEMOBJ)
325	return VINF_SUCCESS;
326	AssertPtrReturn(MemObj, VERR_INVALID_HANDLE);
327	pMem = (PRTR0MEMOBJINTERNAL)MemObj;
328	AssertReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE);
329	AssertReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE);
330	RT_ASSERT_PREEMPTIBLE();
331
332	/*
333	* Deal with mappings according to fFreeMappings.
334	*/
335	if ( !rtR0MemObjIsMapping(pMem)
336	&& pMem->uRel.Parent.cMappings > 0)
337	{
338	/* fail if not requested to free mappings. */
339	if (!fFreeMappings)
340	return VERR_MEMORY_BUSY;
341
342	while (pMem->uRel.Parent.cMappings > 0)
343	{
344	PRTR0MEMOBJINTERNAL pChild = pMem->uRel.Parent.papMappings[--pMem->uRel.Parent.cMappings];
345	pMem->uRel.Parent.papMappings[pMem->uRel.Parent.cMappings] = NULL;
346
347	/* sanity checks. */
348	AssertPtr(pChild);
349	AssertFatal(pChild->u32Magic == RTR0MEMOBJ_MAGIC);
350	AssertFatal(pChild->enmType > RTR0MEMOBJTYPE_INVALID && pChild->enmType < RTR0MEMOBJTYPE_END);
351	AssertFatal(rtR0MemObjIsMapping(pChild));
352
353	/* free the mapping. */
354	rc = rtR0MemObjNativeFree(pChild);
355	if (RT_FAILURE(rc))
356	{
357	Log(("RTR0MemObjFree: failed to free mapping %p: %p %#zx; rc=%Rrc\n", pChild, pChild->pv, pChild->cb, rc));
358	pMem->uRel.Parent.papMappings[pMem->uRel.Parent.cMappings++] = pChild;
359	return rc;
360	}
361
362	pChild->u32Magic++;
363	pChild->enmType = RTR0MEMOBJTYPE_END;
364	RTMemFree(pChild);
365	}
366	}
367
368	/*
369	* Free this object.
370	*/
371	rc = rtR0MemObjNativeFree(pMem);
372	if (RT_SUCCESS(rc))
373	{
374	/*
375	* Ok, it was freed just fine. Now, if it's a mapping we'll have to remove it from the parent.
376	*/
377	if (rtR0MemObjIsMapping(pMem))
378	{
379	PRTR0MEMOBJINTERNAL pParent = pMem->uRel.Child.pParent;
380	uint32_t i;
381
382	/* sanity checks */
383	AssertPtr(pParent);
384	AssertFatal(pParent->u32Magic == RTR0MEMOBJ_MAGIC);
385	AssertFatal(pParent->enmType > RTR0MEMOBJTYPE_INVALID && pParent->enmType < RTR0MEMOBJTYPE_END);
386	AssertFatal(!rtR0MemObjIsMapping(pParent));
387	AssertFatal(pParent->uRel.Parent.cMappings > 0);
388	AssertPtr(pParent->uRel.Parent.papMappings);
389
390	/* locate and remove from the array of mappings. */
391	i = pParent->uRel.Parent.cMappings;
392	while (i-- > 0)
393	{
394	if (pParent->uRel.Parent.papMappings[i] == pMem)
395	{
396	pParent->uRel.Parent.papMappings[i] = pParent->uRel.Parent.papMappings[--pParent->uRel.Parent.cMappings];
397	break;
398	}
399	}
400	Assert(i != UINT32_MAX);
401	}
402	else
403	Assert(pMem->uRel.Parent.cMappings == 0);
404
405	/*
406	* Finally, destroy the handle.
407	*/
408	pMem->u32Magic++;
409	pMem->enmType = RTR0MEMOBJTYPE_END;
410	if (!rtR0MemObjIsMapping(pMem))
411	RTMemFree(pMem->uRel.Parent.papMappings);
412	RTMemFree(pMem);
413	}
414	else
415	Log(("RTR0MemObjFree: failed to free %p: %d %p %#zx; rc=%Rrc\n",
416	pMem, pMem->enmType, pMem->pv, pMem->cb, rc));
417	return rc;
418	}
419	RT_EXPORT_SYMBOL(RTR0MemObjFree);
420
421
422
423	RTR0DECL(int) RTR0MemObjAllocPageTag(PRTR0MEMOBJ pMemObj, size_t cb, bool fExecutable, const char *pszTag)
424	{
425	/* sanity checks. */
426	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
427	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
428	*pMemObj = NIL_RTR0MEMOBJ;
429	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
430	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
431	RT_ASSERT_PREEMPTIBLE();
432
433	RT_NOREF_PV(pszTag);
434
435	/* do the allocation. */
436	return rtR0MemObjNativeAllocPage(pMemObj, cbAligned, fExecutable);
437	}
438	RT_EXPORT_SYMBOL(RTR0MemObjAllocPageTag);
439
440
441	RTR0DECL(int) RTR0MemObjAllocLargeTag(PRTR0MEMOBJ pMemObj, size_t cb, size_t cbLargePage, uint32_t fFlags, const char *pszTag)
442	{
443	/* sanity checks. */
444	const size_t cbAligned = RT_ALIGN_Z(cb, cbLargePage);
445	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
446	*pMemObj = NIL_RTR0MEMOBJ;
447	#ifdef RT_ARCH_AMD64
448	AssertReturn(cbLargePage == _2M \|\| cbLargePage == _1G, VERR_OUT_OF_RANGE);
449	#elif defined(RT_ARCH_X86)
450	AssertReturn(cbLargePage == _2M \|\| cbLargePage == _4M, VERR_OUT_OF_RANGE);
451	#else
452	AssertReturn(RT_IS_POWER_OF_TWO(cbLargePage), VERR_NOT_POWER_OF_TWO);
453	AssertReturn(cbLargePage > PAGE_SIZE, VERR_OUT_OF_RANGE);
454	#endif
455	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
456	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
457	AssertReturn(!(fFlags & ~RTMEMOBJ_ALLOC_LARGE_F_VALID_MASK), VERR_INVALID_PARAMETER);
458	RT_ASSERT_PREEMPTIBLE();
459
460	/* do the allocation. */
461	return rtR0MemObjNativeAllocLarge(pMemObj, cbAligned, cbLargePage, fFlags, pszTag);
462	}
463	RT_EXPORT_SYMBOL(RTR0MemObjAllocLargeTag);
464
465
466	/**
467	* Fallback implementation of rtR0MemObjNativeAllocLarge and implements single
468	* page allocation using rtR0MemObjNativeAllocPhys.
469	*/
470	DECLHIDDEN(int) rtR0MemObjFallbackAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags,
471	const char *pszTag)
472	{
473	RT_NOREF(pszTag, fFlags);
474	if (cb == cbLargePage)
475	return rtR0MemObjNativeAllocPhys(ppMem, cb, NIL_RTHCPHYS, cbLargePage);
476	return VERR_NOT_SUPPORTED;
477	}
478
479
480	RTR0DECL(int) RTR0MemObjAllocLowTag(PRTR0MEMOBJ pMemObj, size_t cb, bool fExecutable, const char *pszTag)
481	{
482	/* sanity checks. */
483	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
484	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
485	*pMemObj = NIL_RTR0MEMOBJ;
486	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
487	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
488	RT_ASSERT_PREEMPTIBLE();
489
490	RT_NOREF_PV(pszTag);
491
492	/* do the allocation. */
493	return rtR0MemObjNativeAllocLow(pMemObj, cbAligned, fExecutable);
494	}
495	RT_EXPORT_SYMBOL(RTR0MemObjAllocLowTag);
496
497
498	RTR0DECL(int) RTR0MemObjAllocContTag(PRTR0MEMOBJ pMemObj, size_t cb, bool fExecutable, const char *pszTag)
499	{
500	/* sanity checks. */
501	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
502	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
503	*pMemObj = NIL_RTR0MEMOBJ;
504	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
505	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
506	RT_ASSERT_PREEMPTIBLE();
507
508	RT_NOREF_PV(pszTag);
509
510	/* do the allocation. */
511	return rtR0MemObjNativeAllocCont(pMemObj, cbAligned, fExecutable);
512	}
513	RT_EXPORT_SYMBOL(RTR0MemObjAllocContTag);
514
515
516	RTR0DECL(int) RTR0MemObjLockUserTag(PRTR0MEMOBJ pMemObj, RTR3PTR R3Ptr, size_t cb,
517	uint32_t fAccess, RTR0PROCESS R0Process, const char *pszTag)
518	{
519	/* sanity checks. */
520	const size_t cbAligned = RT_ALIGN_Z(cb + (R3Ptr & PAGE_OFFSET_MASK), PAGE_SIZE);
521	RTR3PTR const R3PtrAligned = (R3Ptr & ~(RTR3PTR)PAGE_OFFSET_MASK);
522	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
523	*pMemObj = NIL_RTR0MEMOBJ;
524	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
525	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
526	if (R0Process == NIL_RTR0PROCESS)
527	R0Process = RTR0ProcHandleSelf();
528	AssertReturn(!(fAccess & ~(RTMEM_PROT_READ \| RTMEM_PROT_WRITE)), VERR_INVALID_PARAMETER);
529	AssertReturn(fAccess, VERR_INVALID_PARAMETER);
530	RT_ASSERT_PREEMPTIBLE();
531
532	RT_NOREF_PV(pszTag);
533
534	/* do the locking. */
535	return rtR0MemObjNativeLockUser(pMemObj, R3PtrAligned, cbAligned, fAccess, R0Process);
536	}
537	RT_EXPORT_SYMBOL(RTR0MemObjLockUserTag);
538
539
540	RTR0DECL(int) RTR0MemObjLockKernelTag(PRTR0MEMOBJ pMemObj, void pv, size_t cb, uint32_t fAccess, const char pszTag)
541	{
542	/* sanity checks. */
543	const size_t cbAligned = RT_ALIGN_Z(cb + ((uintptr_t)pv & PAGE_OFFSET_MASK), PAGE_SIZE);
544	void * const pvAligned = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
545	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
546	*pMemObj = NIL_RTR0MEMOBJ;
547	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
548	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
549	AssertPtrReturn(pvAligned, VERR_INVALID_POINTER);
550	AssertReturn(!(fAccess & ~(RTMEM_PROT_READ \| RTMEM_PROT_WRITE)), VERR_INVALID_PARAMETER);
551	AssertReturn(fAccess, VERR_INVALID_PARAMETER);
552	RT_ASSERT_PREEMPTIBLE();
553
554	RT_NOREF_PV(pszTag);
555
556	/* do the allocation. */
557	return rtR0MemObjNativeLockKernel(pMemObj, pvAligned, cbAligned, fAccess);
558	}
559	RT_EXPORT_SYMBOL(RTR0MemObjLockKernelTag);
560
561
562	RTR0DECL(int) RTR0MemObjAllocPhysTag(PRTR0MEMOBJ pMemObj, size_t cb, RTHCPHYS PhysHighest, const char *pszTag)
563	{
564	/* sanity checks. */
565	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
566	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
567	*pMemObj = NIL_RTR0MEMOBJ;
568	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
569	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
570	AssertReturn(PhysHighest >= cb, VERR_INVALID_PARAMETER);
571	RT_ASSERT_PREEMPTIBLE();
572
573	RT_NOREF_PV(pszTag);
574
575	/* do the allocation. */
576	return rtR0MemObjNativeAllocPhys(pMemObj, cbAligned, PhysHighest, PAGE_SIZE /* page aligned */);
577	}
578	RT_EXPORT_SYMBOL(RTR0MemObjAllocPhysTag);
579
580
581	RTR0DECL(int) RTR0MemObjAllocPhysExTag(PRTR0MEMOBJ pMemObj, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment, const char *pszTag)
582	{
583	/* sanity checks. */
584	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
585	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
586	*pMemObj = NIL_RTR0MEMOBJ;
587	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
588	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
589	AssertReturn(PhysHighest >= cb, VERR_INVALID_PARAMETER);
590	if (uAlignment == 0)
591	uAlignment = PAGE_SIZE;
592	AssertReturn( uAlignment == PAGE_SIZE
593	\|\| uAlignment == _2M
594	\|\| uAlignment == _4M
595	\|\| uAlignment == _1G,
596	VERR_INVALID_PARAMETER);
597	#if HC_ARCH_BITS == 32
598	/* Memory allocated in this way is typically mapped into kernel space as well; simply
599	don't allow this on 32 bits hosts as the kernel space is too crowded already. */
600	if (uAlignment != PAGE_SIZE)
601	return VERR_NOT_SUPPORTED;
602	#endif
603	RT_ASSERT_PREEMPTIBLE();
604
605	RT_NOREF_PV(pszTag);
606
607	/* do the allocation. */
608	return rtR0MemObjNativeAllocPhys(pMemObj, cbAligned, PhysHighest, uAlignment);
609	}
610	RT_EXPORT_SYMBOL(RTR0MemObjAllocPhysExTag);
611
612
613	RTR0DECL(int) RTR0MemObjAllocPhysNCTag(PRTR0MEMOBJ pMemObj, size_t cb, RTHCPHYS PhysHighest, const char *pszTag)
614	{
615	/* sanity checks. */
616	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
617	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
618	*pMemObj = NIL_RTR0MEMOBJ;
619	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
620	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
621	AssertReturn(PhysHighest >= cb, VERR_INVALID_PARAMETER);
622	RT_ASSERT_PREEMPTIBLE();
623
624	RT_NOREF_PV(pszTag);
625
626	/* do the allocation. */
627	return rtR0MemObjNativeAllocPhysNC(pMemObj, cbAligned, PhysHighest);
628	}
629	RT_EXPORT_SYMBOL(RTR0MemObjAllocPhysNCTag);
630
631
632	RTR0DECL(int) RTR0MemObjEnterPhysTag(PRTR0MEMOBJ pMemObj, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy, const char *pszTag)
633	{
634	/* sanity checks. */
635	const size_t cbAligned = RT_ALIGN_Z(cb + (Phys & PAGE_OFFSET_MASK), PAGE_SIZE);
636	const RTHCPHYS PhysAligned = Phys & ~(RTHCPHYS)PAGE_OFFSET_MASK;
637	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
638	*pMemObj = NIL_RTR0MEMOBJ;
639	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
640	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
641	AssertReturn(Phys != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
642	AssertReturn( uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE
643	\|\| uCachePolicy == RTMEM_CACHE_POLICY_MMIO,
644	VERR_INVALID_PARAMETER);
645	RT_ASSERT_PREEMPTIBLE();
646
647	RT_NOREF_PV(pszTag);
648
649	/* do the allocation. */
650	return rtR0MemObjNativeEnterPhys(pMemObj, PhysAligned, cbAligned, uCachePolicy);
651	}
652	RT_EXPORT_SYMBOL(RTR0MemObjEnterPhysTag);
653
654
655	RTR0DECL(int) RTR0MemObjReserveKernelTag(PRTR0MEMOBJ pMemObj, void pvFixed, size_t cb, size_t uAlignment, const char pszTag)
656	{
657	/* sanity checks. */
658	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
659	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
660	*pMemObj = NIL_RTR0MEMOBJ;
661	if (uAlignment == 0)
662	uAlignment = PAGE_SIZE;
663	AssertReturn(uAlignment == PAGE_SIZE \|\| uAlignment == _2M \|\| uAlignment == _4M, VERR_INVALID_PARAMETER);
664	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
665	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
666	if (pvFixed != (void *)-1)
667	AssertReturn(!((uintptr_t)pvFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER);
668	RT_ASSERT_PREEMPTIBLE();
669
670	RT_NOREF_PV(pszTag);
671
672	/* do the reservation. */
673	return rtR0MemObjNativeReserveKernel(pMemObj, pvFixed, cbAligned, uAlignment);
674	}
675	RT_EXPORT_SYMBOL(RTR0MemObjReserveKernelTag);
676
677
678	RTR0DECL(int) RTR0MemObjReserveUserTag(PRTR0MEMOBJ pMemObj, RTR3PTR R3PtrFixed, size_t cb,
679	size_t uAlignment, RTR0PROCESS R0Process, const char *pszTag)
680	{
681	/* sanity checks. */
682	const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE);
683	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
684	*pMemObj = NIL_RTR0MEMOBJ;
685	if (uAlignment == 0)
686	uAlignment = PAGE_SIZE;
687	AssertReturn(uAlignment == PAGE_SIZE \|\| uAlignment == _2M \|\| uAlignment == _4M, VERR_INVALID_PARAMETER);
688	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
689	AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER);
690	if (R3PtrFixed != (RTR3PTR)-1)
691	AssertReturn(!(R3PtrFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER);
692	if (R0Process == NIL_RTR0PROCESS)
693	R0Process = RTR0ProcHandleSelf();
694	RT_ASSERT_PREEMPTIBLE();
695
696	RT_NOREF_PV(pszTag);
697
698	/* do the reservation. */
699	return rtR0MemObjNativeReserveUser(pMemObj, R3PtrFixed, cbAligned, uAlignment, R0Process);
700	}
701	RT_EXPORT_SYMBOL(RTR0MemObjReserveUserTag);
702
703
704	RTR0DECL(int) RTR0MemObjMapKernelTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, void *pvFixed,
705	size_t uAlignment, unsigned fProt, const char *pszTag)
706	{
707	return RTR0MemObjMapKernelExTag(pMemObj, MemObjToMap, pvFixed, uAlignment, fProt, 0, 0, pszTag);
708	}
709	RT_EXPORT_SYMBOL(RTR0MemObjMapKernelTag);
710
711
712	RTR0DECL(int) RTR0MemObjMapKernelExTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, void *pvFixed, size_t uAlignment,
713	unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag)
714	{
715	PRTR0MEMOBJINTERNAL pMemToMap;
716	PRTR0MEMOBJINTERNAL pNew;
717	int rc;
718
719	/* sanity checks. */
720	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
721	*pMemObj = NIL_RTR0MEMOBJ;
722	AssertPtrReturn(MemObjToMap, VERR_INVALID_HANDLE);
723	pMemToMap = (PRTR0MEMOBJINTERNAL)MemObjToMap;
724	AssertReturn(pMemToMap->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE);
725	AssertReturn(pMemToMap->enmType > RTR0MEMOBJTYPE_INVALID && pMemToMap->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE);
726	AssertReturn(!rtR0MemObjIsMapping(pMemToMap), VERR_INVALID_PARAMETER);
727	AssertReturn(pMemToMap->enmType != RTR0MEMOBJTYPE_RES_VIRT, VERR_INVALID_PARAMETER);
728	if (uAlignment == 0)
729	uAlignment = PAGE_SIZE;
730	AssertReturn(uAlignment == PAGE_SIZE \|\| uAlignment == _2M \|\| uAlignment == _4M, VERR_INVALID_PARAMETER);
731	if (pvFixed != (void *)-1)
732	AssertReturn(!((uintptr_t)pvFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER);
733	AssertReturn(fProt != RTMEM_PROT_NONE, VERR_INVALID_PARAMETER);
734	AssertReturn(!(fProt & ~(RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC)), VERR_INVALID_PARAMETER);
735	AssertReturn(!(offSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
736	AssertReturn(offSub < pMemToMap->cb, VERR_INVALID_PARAMETER);
737	AssertReturn(!(cbSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
738	AssertReturn(cbSub <= pMemToMap->cb, VERR_INVALID_PARAMETER);
739	AssertReturn((!offSub && !cbSub) \|\| (offSub + cbSub) <= pMemToMap->cb, VERR_INVALID_PARAMETER);
740	RT_ASSERT_PREEMPTIBLE();
741
742	RT_NOREF_PV(pszTag);
743
744	/* adjust the request to simplify the native code. */
745	if (offSub == 0 && cbSub == pMemToMap->cb)
746	cbSub = 0;
747
748	/* do the mapping. */
749	rc = rtR0MemObjNativeMapKernel(&pNew, pMemToMap, pvFixed, uAlignment, fProt, offSub, cbSub);
750	if (RT_SUCCESS(rc))
751	{
752	/* link it. */
753	rc = rtR0MemObjLink(pMemToMap, pNew);
754	if (RT_SUCCESS(rc))
755	*pMemObj = pNew;
756	else
757	{
758	/* damn, out of memory. bail out. */
759	int rc2 = rtR0MemObjNativeFree(pNew);
760	AssertRC(rc2);
761	pNew->u32Magic++;
762	pNew->enmType = RTR0MEMOBJTYPE_END;
763	RTMemFree(pNew);
764	}
765	}
766
767	return rc;
768	}
769	RT_EXPORT_SYMBOL(RTR0MemObjMapKernelExTag);
770
771
772	RTR0DECL(int) RTR0MemObjMapUserTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, RTR3PTR R3PtrFixed,
773	size_t uAlignment, unsigned fProt, RTR0PROCESS R0Process, const char *pszTag)
774	{
775	return RTR0MemObjMapUserExTag(pMemObj, MemObjToMap, R3PtrFixed, uAlignment, fProt, R0Process, 0, 0, pszTag);
776	}
777	RT_EXPORT_SYMBOL(RTR0MemObjMapUserTag);
778
779
780	RTR0DECL(int) RTR0MemObjMapUserExTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
781	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag)
782	{
783	/* sanity checks. */
784	PRTR0MEMOBJINTERNAL pMemToMap;
785	PRTR0MEMOBJINTERNAL pNew;
786	int rc;
787	AssertPtrReturn(pMemObj, VERR_INVALID_POINTER);
788	pMemToMap = (PRTR0MEMOBJINTERNAL)MemObjToMap;
789	*pMemObj = NIL_RTR0MEMOBJ;
790	AssertPtrReturn(MemObjToMap, VERR_INVALID_HANDLE);
791	AssertReturn(pMemToMap->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE);
792	AssertReturn(pMemToMap->enmType > RTR0MEMOBJTYPE_INVALID && pMemToMap->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE);
793	AssertReturn(!rtR0MemObjIsMapping(pMemToMap), VERR_INVALID_PARAMETER);
794	AssertReturn(pMemToMap->enmType != RTR0MEMOBJTYPE_RES_VIRT, VERR_INVALID_PARAMETER);
795	if (uAlignment == 0)
796	uAlignment = PAGE_SIZE;
797	AssertReturn(uAlignment == PAGE_SIZE \|\| uAlignment == _2M \|\| uAlignment == _4M, VERR_INVALID_PARAMETER);
798	if (R3PtrFixed != (RTR3PTR)-1)
799	AssertReturn(!(R3PtrFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER);
800	AssertReturn(fProt != RTMEM_PROT_NONE, VERR_INVALID_PARAMETER);
801	AssertReturn(!(fProt & ~(RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC)), VERR_INVALID_PARAMETER);
802	AssertReturn(!(offSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
803	AssertReturn(offSub < pMemToMap->cb, VERR_INVALID_PARAMETER);
804	AssertReturn(!(cbSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
805	AssertReturn(cbSub <= pMemToMap->cb, VERR_INVALID_PARAMETER);
806	AssertReturn((!offSub && !cbSub) \|\| (offSub + cbSub) <= pMemToMap->cb, VERR_INVALID_PARAMETER);
807	if (R0Process == NIL_RTR0PROCESS)
808	R0Process = RTR0ProcHandleSelf();
809	RT_ASSERT_PREEMPTIBLE();
810
811	RT_NOREF_PV(pszTag);
812
813	/* adjust the request to simplify the native code. */
814	if (offSub == 0 && cbSub == pMemToMap->cb)
815	cbSub = 0;
816
817	/* do the mapping. */
818	rc = rtR0MemObjNativeMapUser(&pNew, pMemToMap, R3PtrFixed, uAlignment, fProt, R0Process, offSub, cbSub);
819	if (RT_SUCCESS(rc))
820	{
821	/* link it. */
822	rc = rtR0MemObjLink(pMemToMap, pNew);
823	if (RT_SUCCESS(rc))
824	*pMemObj = pNew;
825	else
826	{
827	/* damn, out of memory. bail out. */
828	int rc2 = rtR0MemObjNativeFree(pNew);
829	AssertRC(rc2);
830	pNew->u32Magic++;
831	pNew->enmType = RTR0MEMOBJTYPE_END;
832	RTMemFree(pNew);
833	}
834	}
835
836	return rc;
837	}
838	RT_EXPORT_SYMBOL(RTR0MemObjMapUserExTag);
839
840
841	RTR0DECL(int) RTR0MemObjProtect(RTR0MEMOBJ hMemObj, size_t offSub, size_t cbSub, uint32_t fProt)
842	{
843	PRTR0MEMOBJINTERNAL pMemObj;
844	int rc;
845
846	/* sanity checks. */
847	pMemObj = (PRTR0MEMOBJINTERNAL)hMemObj;
848	AssertPtrReturn(pMemObj, VERR_INVALID_HANDLE);
849	AssertReturn(pMemObj->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE);
850	AssertReturn(pMemObj->enmType > RTR0MEMOBJTYPE_INVALID && pMemObj->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE);
851	AssertReturn(rtR0MemObjIsProtectable(pMemObj), VERR_INVALID_PARAMETER);
852	AssertReturn(!(offSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
853	AssertReturn(offSub < pMemObj->cb, VERR_INVALID_PARAMETER);
854	AssertReturn(!(cbSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
855	AssertReturn(cbSub <= pMemObj->cb, VERR_INVALID_PARAMETER);
856	AssertReturn(offSub + cbSub <= pMemObj->cb, VERR_INVALID_PARAMETER);
857	AssertReturn(!(fProt & ~(RTMEM_PROT_NONE \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC)), VERR_INVALID_PARAMETER);
858	RT_ASSERT_PREEMPTIBLE();
859
860	/* do the job */
861	rc = rtR0MemObjNativeProtect(pMemObj, offSub, cbSub, fProt);
862	if (RT_SUCCESS(rc))
863	pMemObj->fFlags \|= RTR0MEMOBJ_FLAGS_PROT_CHANGED; /* record it */
864
865	return rc;
866	}
867	RT_EXPORT_SYMBOL(RTR0MemObjProtect);
868

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/memobj-r0drv.cpp@ 91449

Download in other formats: