memobj-r0drv-solaris.c@ 41627

Last change on this file since 41627 was 41627, checked in by vboxsync, 13 years ago
Runtime/r0drv/solaris: Fix to prevent paging out during freeing of 4K pages, fix unsigned overflow while freeing pages in the failure case and some comment fixes and cleanup.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 38.3 KB

Line
1	/* $Id: memobj-r0drv-solaris.c 41627 2012-06-08 16:16:04Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Solaris.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include "the-solaris-kernel.h"
32	#include "internal/iprt.h"
33	#include <iprt/memobj.h>
34
35	#include <iprt/asm.h>
36	#include <iprt/assert.h>
37	#include <iprt/err.h>
38	#include <iprt/log.h>
39	#include <iprt/mem.h>
40	#include <iprt/param.h>
41	#include <iprt/process.h>
42	#include "internal/memobj.h"
43	#include "memobj-r0drv-solaris.h"
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define SOL_IS_KRNL_ADDR(vx) ((uintptr_t)(vx) >= kernelbase)
49
50
51	/*******************************************************************************
52	* Structures and Typedefs *
53	*******************************************************************************/
54	/**
55	* The Solaris version of the memory object structure.
56	*/
57	typedef struct RTR0MEMOBJSOL
58	{
59	/** The core structure. */
60	RTR0MEMOBJINTERNAL Core;
61	/** Pointer to kernel memory cookie. */
62	ddi_umem_cookie_t Cookie;
63	/** Shadow locked pages. */
64	void *pvHandle;
65	/** Access during locking. */
66	int fAccess;
67	/** Set if large pages are involved in an RTR0MEMOBJTYPE_PHYS
68	* allocation. */
69	bool fLargePage;
70	} RTR0MEMOBJSOL, *PRTR0MEMOBJSOL;
71
72
73	/*******************************************************************************
74	* Global Variables *
75	*******************************************************************************/
76	static vnode_t g_PageVnode;
77	static kmutex_t g_OffsetMtx;
78	static u_offset_t g_offPage;
79
80	static vnode_t g_LargePageVnode;
81	static kmutex_t g_LargePageOffsetMtx;
82	static u_offset_t g_offLargePage;
83
84
85	/**
86	* Returns the physical address for a virtual address.
87	*
88	* @param pv The virtual address.
89	*
90	* @returns The physical address corresponding to @a pv.
91	*/
92	static uint64_t rtR0MemObjSolVirtToPhys(void *pv)
93	{
94	struct hat *pHat = NULL;
95	pfn_t PageFrameNum = 0;
96	uintptr_t uVirtAddr = (uintptr_t)pv;
97
98	if (SOL_IS_KRNL_ADDR(pv))
99	pHat = kas.a_hat;
100	else
101	{
102	proc_t pProcess = (proc_t )RTR0ProcHandleSelf();
103	AssertRelease(pProcess);
104	pHat = pProcess->p_as->a_hat;
105	}
106
107	PageFrameNum = hat_getpfnum(pHat, (caddr_t)(uVirtAddr & PAGEMASK));
108	AssertReleaseMsg(PageFrameNum != PFN_INVALID, ("rtR0MemObjSolVirtToPhys failed. pv=%p\n", pv));
109	return (((uint64_t)PageFrameNum << PAGE_SHIFT) \| (uVirtAddr & PAGE_OFFSET_MASK));
110	}
111
112
113	/**
114	* Returns the physical address for a page.
115	*
116	* @param pPage Pointer to the page.
117	*
118	* @returns The physical address for a page.
119	*/
120	static inline uint64_t rtR0MemObjSolPagePhys(page_t *pPage)
121	{
122	AssertPtr(pPage);
123	pfn_t PageFrameNum = page_pptonum(pPage);
124	AssertReleaseMsg(PageFrameNum != PFN_INVALID, ("rtR0MemObjSolPagePhys failed pPage=%p\n"));
125	return (uint64_t)PageFrameNum << PAGE_SHIFT;
126	}
127
128
129	/**
130	* Allocates one page.
131	*
132	* @param virtAddr The virtual address to which this page maybe mapped in
133	* the future.
134	* @param cbPage The size of the page.
135	*
136	* @returns Pointer to the allocated page, NULL on failure.
137	*/
138	static page_t *rtR0MemObjSolPageAlloc(caddr_t virtAddr, size_t cbPage)
139	{
140	Assert(cbPage == PAGE_SIZE);
141
142	u_offset_t offPage;
143	seg_t KernelSeg;
144
145	mutex_enter(&g_OffsetMtx);
146	AssertCompileSize(u_offset_t, sizeof(uint64_t)); NOREF(RTASSERTVAR);
147	g_offPage = RT_ALIGN_64(g_offPage, cbPage) + cbPage;
148	offPage = g_offPage;
149	mutex_exit(&g_OffsetMtx);
150
151	KernelSeg.s_as = &kas;
152	page_t *pPage = page_create_va(&g_PageVnode, offPage, cbPage, PG_WAIT \| PG_NORELOC, &KernelSeg, virtAddr);
153
154	if (RT_LIKELY(pPage))
155	{
156	/*
157	* Lock this page into memory "long term" to prevent paging out of this page.
158	*/
159	page_pp_lock(pPage, 0 /* COW /, 1 / Kernel */);
160	page_io_unlock(pPage);
161	page_downgrade(pPage);
162	Assert(PAGE_LOCKED_SE(pPage, SE_SHARED));
163	}
164
165	return pPage;
166	}
167
168
169	/**
170	* Allocates physical, non-contiguous memory of pages.
171	*
172	* @param uPhysHi The upper physical address limit (inclusive).
173	* @param puPhys Where to store the physical address of first page. Optional,
174	* can be NULL.
175	* @param cb The size of the allocation.
176	*
177	* @return Array of allocated pages, NULL on failure.
178	*/
179	static page_t *rtR0MemObjSolPagesAlloc(uint64_t uPhysHi, uint64_t puPhys, size_t cb)
180	{
181	/*
182	* VM1:
183	* The page freelist and cachelist both hold pages that are not mapped into any address space.
184	* The cachelist is not really free pages but when memory is exhausted they'll be moved to the
185	* free lists, it's the total of the free+cache list that we see on the 'free' column in vmstat.
186	*
187	* VM2:
188	* @todo Document what happens behind the scenes in VM2 regarding the free and cachelist.
189	*/
190
191	/*
192	* Non-pageable memory reservation request for _4K pages, don't sleep.
193	*/
194	pgcnt_t cPages = (cb + PAGE_SIZE - 1) >> PAGE_SHIFT;
195	int rc = page_resv(cPages, KM_NOSLEEP);
196	if (rc)
197	{
198	size_t cbPages = cPages * sizeof(page_t *);
199	page_t **ppPages = kmem_zalloc(cbPages, KM_SLEEP);
200	if (RT_LIKELY(ppPages))
201	{
202	/*
203	* Get pages from kseg, the 'virtAddr' here is only for colouring but unfortunately
204	* we don't yet have the 'virtAddr' to which this memory may be mapped.
205	*/
206	caddr_t virtAddr = NULL;
207	for (size_t i = 0; i < cPages; i++, virtAddr += PAGE_SIZE)
208	{
209	uint32_t cTries = 3;
210	page_t *pPage = NULL;
211	while (cTries > 0)
212	{
213	/*
214	* Get a page from the free list locked exclusively. The page will be named (hashed in)
215	* and we rely on it during free. Downgrade the page to a shared lock to prevent the page
216	* from being relocated.
217	*/
218	pPage = rtR0MemObjSolPageAlloc(virtAddr, PAGE_SIZE);
219	if (!pPage)
220	break;
221
222	/*
223	* Check if the physical address backing the page is within the requested range if any.
224	* If it isn't, discard the page and try again.
225	*/
226	/** @todo Remove this constraint here, force all high-limit applicable cases
227	* through rtR0SolMemAlloc() */
228	if (uPhysHi != NIL_RTHCPHYS)
229	{
230	uint64_t uPhys = rtR0MemObjSolPagePhys(pPage);
231	if (uPhys > uPhysHi)
232	{
233	page_destroy(pPage, 0 /* move it to the free list */);
234	pPage = NULL;
235	--cTries;
236	continue;
237	}
238	}
239
240	ppPages[i] = pPage;
241	break;
242	}
243
244	if (RT_UNLIKELY(!pPage))
245	{
246	/*
247	* No pages found or found pages didn't meet requirements, release what was grabbed so far.
248	*/
249	for (size_t k = 0; k <= i; k++)
250	page_destroy(ppPages[k], 0 /* move it to the free list */);
251	kmem_free(ppPages, cbPages);
252	page_unresv(cPages);
253	return NULL;
254	}
255	}
256
257	if (puPhys)
258	*puPhys = rtR0MemObjSolPagePhys(ppPages[0]);
259	return ppPages;
260	}
261
262	page_unresv(cPages);
263	}
264
265	return NULL;
266	}
267
268
269	/**
270	* Frees the allocates pages.
271	*
272	* @param ppPages Pointer to the page list.
273	* @param cbPages Size of the allocation.
274	*/
275	static void rtR0MemObjSolPagesFree(page_t **ppPages, size_t cb)
276	{
277	size_t cPages = (cb + PAGE_SIZE - 1) >> PAGE_SHIFT;
278	size_t cbPages = cPages * sizeof(page_t *);
279	for (size_t iPage = 0; iPage < cPages; iPage++)
280	{
281	/*
282	* We need to exclusive lock the pages before freeing them.
283	*/
284	page_t *pPage = ppPages[iPage];
285	u_offset_t offPage = pPage->p_offset;
286
287	int rc = page_tryupgrade(ppPages[iPage]);
288	if (!rc)
289	{
290	page_unlock(pPage);
291	page_t *pFoundPage = page_lookup(&g_PageVnode, offPage, SE_EXCL);
292
293	/*
294	* Since we allocated the pages as PG_NORELOC we should only get back the exact page always.
295	*/
296	AssertReleaseMsg(pFoundPage == pPage, ("Page lookup failed %p:%llx returned %p, expected %p\n",
297	&g_PageVnode, offPage, pFoundPage, pPage));
298	}
299	Assert(PAGE_LOCKED_SE(pPage, SE_EXCL));
300	page_pp_unlock(pPage, 0 /* COW /, 1 / Kernel */);
301	page_destroy(pPage, 0 /* move it to the free list */);
302	}
303	kmem_free(ppPages, cbPages);
304	page_unresv(cPages);
305	}
306
307
308	/**
309	* Allocates one large page.
310	*
311	* @param puPhys Where to store the physical address of the allocated
312	* page. Optional, can be NULL.
313	* @param cbLargePage Size of the large page.
314	*
315	* @returns Pointer to a list of pages that cover the large page, NULL on
316	* failure.
317	*/
318	static page_t *rtR0MemObjSolLargePageAlloc(uint64_t puPhys, size_t cbLargePage)
319	{
320	/*
321	* Non-pageable memory reservation request for _4K pages, don't sleep.
322	*/
323	size_t cPages = (cbLargePage + PAGE_SIZE - 1) >> PAGE_SHIFT;
324	size_t cbPages = cPages * sizeof(page_t *);
325	u_offset_t offPage = 0;
326	int rc = page_resv(cPages, KM_NOSLEEP);
327	if (rc)
328	{
329	page_t **ppPages = kmem_zalloc(cbPages, KM_SLEEP);
330	if (RT_LIKELY(ppPages))
331	{
332	mutex_enter(&g_LargePageOffsetMtx);
333	AssertCompileSize(u_offset_t, sizeof(uint64_t)); NOREF(RTASSERTVAR);
334	g_offLargePage = RT_ALIGN_64(g_offLargePage, cbLargePage) + cbLargePage;
335	offPage = g_offLargePage;
336	mutex_exit(&g_LargePageOffsetMtx);
337
338	seg_t KernelSeg;
339	KernelSeg.s_as = &kas;
340	page_t *pRootPage = page_create_va_large(&g_LargePageVnode, offPage, cbLargePage,
341	PG_EXCL, &KernelSeg, 0 /* vaddr /, NULL / locality group */);
342	if (pRootPage)
343	{
344	/*
345	* Split it into sub-pages, downgrade each page to a shared lock to prevent page relocation.
346	*/
347	page_t *pPageList = pRootPage;
348	for (size_t iPage = 0; iPage < cPages; iPage++)
349	{
350	page_t *pPage = pPageList;
351	AssertPtr(pPage);
352	AssertMsg(page_pptonum(pPage) == iPage + page_pptonum(pRootPage),
353	("%p:%lx %lx+%lx\n", pPage, page_pptonum(pPage), iPage, page_pptonum(pRootPage)));
354	AssertMsg(pPage->p_szc == pRootPage->p_szc, ("Size code mismatch %p %d %d\n", pPage,
355	(int)pPage->p_szc, (int)pRootPage->p_szc));
356
357	/*
358	* Lock the page into memory "long term". This prevents callers of page_try_demote_pages() (such as the
359	* pageout scanner) from demoting the large page into smaller pages while we temporarily release the
360	* exclusive lock (during free). We pass "0, 1" since we've already accounted for availrmem during
361	* page_resv().
362	*/
363	page_pp_lock(pPage, 0 /* COW /, 1 / Kernel */);
364
365	page_sub(&pPageList, pPage);
366	page_io_unlock(pPage);
367	page_downgrade(pPage);
368	Assert(PAGE_LOCKED_SE(pPage, SE_SHARED));
369
370	ppPages[iPage] = pPage;
371	}
372	Assert(pPageList == NULL);
373	Assert(ppPages[0] == pRootPage);
374
375	uint64_t uPhys = rtR0MemObjSolPagePhys(pRootPage);
376	AssertMsg(!(uPhys & (cbLargePage - 1)), ("%llx %zx\n", uPhys, cbLargePage));
377	if (puPhys)
378	*puPhys = uPhys;
379	return ppPages;
380	}
381
382	/*
383	* Don't restore offPrev in case of failure (race condition), we have plenty of offset space.
384	* The offset must be unique (for the same vnode) or we'll encounter panics on page_create_va_large().
385	*/
386	kmem_free(ppPages, cbPages);
387	}
388
389	page_unresv(cPages);
390	}
391	return NULL;
392	}
393
394
395	/**
396	* Frees the large page.
397	*
398	* @param ppPages Pointer to the list of small pages that cover the
399	* large page.
400	* @param cbLargePage Size of the allocation (i.e. size of the large
401	* page).
402	*/
403	static void rtR0MemObjSolLargePageFree(page_t **ppPages, size_t cbLargePage)
404	{
405	Assert(ppPages);
406	Assert(cbLargePage > PAGE_SIZE);
407
408	bool fDemoted = false;
409	size_t cPages = (cbLargePage + PAGE_SIZE - 1) >> PAGE_SHIFT;
410	size_t cbPages = cPages * sizeof(page_t *);
411	page_t *pPageList = ppPages[0];
412
413	for (size_t iPage = 0; iPage < cPages; iPage++)
414	{
415	/*
416	* We need the pages exclusively locked, try upgrading the shared lock.
417	* If it fails, drop the shared page lock (cannot access any page_t members once this is done)
418	* and lookup the page from the page hash locking it exclusively.
419	*/
420	page_t *pPage = ppPages[iPage];
421	u_offset_t offPage = pPage->p_offset;
422	int rc = page_tryupgrade(pPage);
423	if (!rc)
424	{
425	page_unlock(pPage);
426	page_t *pFoundPage = page_lookup(&g_LargePageVnode, offPage, SE_EXCL);
427	AssertRelease(pFoundPage);
428
429	#if 0
430	/*
431	* This can only be guaranteed if PG_NORELOC is used while allocating the pages.
432	*/
433	AssertReleaseMsg(pFoundPage == pPage,
434	("lookup failed %p:%llu returned %p, expected %p\n", &g_LargePageVnode, offPage,
435	pFoundPage, pPage));
436	#endif
437
438	/*
439	* Check for page demotion (regardless of relocation). Some places in Solaris (e.g. VM1 page_retire())
440	* could possibly demote the large page to _4K pages between our call to page_unlock() and page_lookup().
441	*/
442	if (page_get_pagecnt(pFoundPage->p_szc) == 1) /* Base size of only _4K associated with this page. */
443	fDemoted = true;
444	pPage = pFoundPage;
445	ppPages[iPage] = pFoundPage;
446	}
447	Assert(PAGE_LOCKED_SE(pPage, SE_EXCL));
448	page_pp_unlock(pPage, 0 /* COW /, 1 / Kernel */);
449	}
450
451	if (fDemoted)
452	{
453	for (size_t iPage = 0; iPage < cPages; iPage++)
454	{
455	Assert(page_get_pagecnt(ppPages[iPage]->p_szc) == 1);
456	page_destroy(ppPages[iPage], 0 /* move it to the free list */);
457	}
458	}
459	else
460	{
461	/*
462	* Although we shred the adjacent pages in the linked list, page_destroy_pages works on
463	* adjacent pages via array increments. So this does indeed free all the pages.
464	*/
465	AssertPtr(pPageList);
466	page_destroy_pages(pPageList);
467	}
468	kmem_free(ppPages, cbPages);
469	page_unresv(cPages);
470	}
471
472
473	/**
474	* Unmaps kernel/user-space mapped memory.
475	*
476	* @param pv Pointer to the mapped memory block.
477	* @param cb Size of the memory block.
478	*/
479	static void rtR0MemObjSolUnmap(void *pv, size_t cb)
480	{
481	if (SOL_IS_KRNL_ADDR(pv))
482	{
483	hat_unload(kas.a_hat, pv, cb, HAT_UNLOAD \| HAT_UNLOAD_UNLOCK);
484	vmem_free(heap_arena, pv, cb);
485	}
486	else
487	{
488	struct as pAddrSpace = ((proc_t )RTR0ProcHandleSelf())->p_as;
489	AssertPtr(pAddrSpace);
490	as_rangelock(pAddrSpace);
491	as_unmap(pAddrSpace, pv, cb);
492	as_rangeunlock(pAddrSpace);
493	}
494	}
495
496
497	/**
498	* Lock down memory mappings for a virtual address.
499	*
500	* @param pv Pointer to the memory to lock down.
501	* @param cb Size of the memory block.
502	* @param fAccess Page access rights (S_READ, S_WRITE, S_EXEC)
503	*
504	* @returns IPRT status code.
505	*/
506	static int rtR0MemObjSolLock(void *pv, size_t cb, int fPageAccess)
507	{
508	/*
509	* Kernel memory mappings on x86/amd64 are always locked, only handle user-space memory.
510	*/
511	if (!SOL_IS_KRNL_ADDR(pv))
512	{
513	proc_t pProc = (proc_t )RTR0ProcHandleSelf();
514	AssertPtr(pProc);
515	faultcode_t rc = as_fault(pProc->p_as->a_hat, pProc->p_as, (caddr_t)pv, cb, F_SOFTLOCK, fPageAccess);
516	if (rc)
517	{
518	LogRel(("rtR0MemObjSolLock failed for pv=%pv cb=%lx fPageAccess=%d rc=%d\n", pv, cb, fPageAccess, rc));
519	return VERR_LOCK_FAILED;
520	}
521	}
522	return VINF_SUCCESS;
523	}
524
525
526	/**
527	* Unlock memory mappings for a virtual address.
528	*
529	* @param pv Pointer to the locked memory.
530	* @param cb Size of the memory block.
531	* @param fPageAccess Page access rights (S_READ, S_WRITE, S_EXEC).
532	*/
533	static void rtR0MemObjSolUnlock(void *pv, size_t cb, int fPageAccess)
534	{
535	if (!SOL_IS_KRNL_ADDR(pv))
536	{
537	proc_t pProcess = (proc_t )RTR0ProcHandleSelf();
538	AssertPtr(pProcess);
539	as_fault(pProcess->p_as->a_hat, pProcess->p_as, (caddr_t)pv, cb, F_SOFTUNLOCK, fPageAccess);
540	}
541	}
542
543
544	/**
545	* Maps a list of physical pages into user address space.
546	*
547	* @param pVirtAddr Where to store the virtual address of the mapping.
548	* @param fPageAccess Page access rights (PROT_READ, PROT_WRITE,
549	* PROT_EXEC)
550	* @param paPhysAddrs Array of physical addresses to pages.
551	* @param cb Size of memory being mapped.
552	*
553	* @returns IPRT status code.
554	*/
555	static int rtR0MemObjSolUserMap(caddr_t pVirtAddr, unsigned fPageAccess, uint64_t paPhysAddrs, size_t cb, size_t cbPageSize)
556	{
557	struct as pAddrSpace = ((proc_t )RTR0ProcHandleSelf())->p_as;
558	int rc = VERR_INTERNAL_ERROR;
559	SEGVBOX_CRARGS Args;
560
561	Args.paPhysAddrs = paPhysAddrs;
562	Args.fPageAccess = fPageAccess;
563	Args.cbPageSize = cbPageSize;
564
565	as_rangelock(pAddrSpace);
566	map_addr(pVirtAddr, cb, 0 /* offset /, 0 / vacalign */, MAP_SHARED);
567	if (*pVirtAddr != NULL)
568	rc = as_map(pAddrSpace, *pVirtAddr, cb, rtR0SegVBoxSolCreate, &Args);
569	else
570	rc = ENOMEM;
571	as_rangeunlock(pAddrSpace);
572
573	return RTErrConvertFromErrno(rc);
574	}
575
576
577	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
578	{
579	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)pMem;
580
581	switch (pMemSolaris->Core.enmType)
582	{
583	case RTR0MEMOBJTYPE_LOW:
584	rtR0SolMemFree(pMemSolaris->Core.pv, pMemSolaris->Core.cb);
585	break;
586
587	case RTR0MEMOBJTYPE_PHYS:
588	if (pMemSolaris->Core.u.Phys.fAllocated)
589	{
590	if (pMemSolaris->fLargePage)
591	rtR0MemObjSolLargePageFree(pMemSolaris->pvHandle, pMemSolaris->Core.cb);
592	else
593	rtR0SolMemFree(pMemSolaris->Core.pv, pMemSolaris->Core.cb);
594	}
595	break;
596
597	case RTR0MEMOBJTYPE_PHYS_NC:
598	rtR0MemObjSolPagesFree(pMemSolaris->pvHandle, pMemSolaris->Core.cb);
599	break;
600
601	case RTR0MEMOBJTYPE_PAGE:
602	ddi_umem_free(pMemSolaris->Cookie);
603	break;
604
605	case RTR0MEMOBJTYPE_LOCK:
606	rtR0MemObjSolUnlock(pMemSolaris->Core.pv, pMemSolaris->Core.cb, pMemSolaris->fAccess);
607	break;
608
609	case RTR0MEMOBJTYPE_MAPPING:
610	rtR0MemObjSolUnmap(pMemSolaris->Core.pv, pMemSolaris->Core.cb);
611	break;
612
613	case RTR0MEMOBJTYPE_RES_VIRT:
614	{
615	if (pMemSolaris->Core.u.ResVirt.R0Process == NIL_RTR0PROCESS)
616	vmem_xfree(heap_arena, pMemSolaris->Core.pv, pMemSolaris->Core.cb);
617	else
618	AssertFailed();
619	break;
620	}
621
622	case RTR0MEMOBJTYPE_CONT: /* we don't use this type here. */
623	default:
624	AssertMsgFailed(("enmType=%d\n", pMemSolaris->Core.enmType));
625	return VERR_INTERNAL_ERROR;
626	}
627
628	return VINF_SUCCESS;
629	}
630
631
632	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
633	{
634	/* Create the object. */
635	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PAGE, NULL, cb);
636	if (RT_UNLIKELY(!pMemSolaris))
637	return VERR_NO_MEMORY;
638
639	void *pvMem = ddi_umem_alloc(cb, DDI_UMEM_SLEEP, &pMemSolaris->Cookie);
640	if (RT_UNLIKELY(!pvMem))
641	{
642	rtR0MemObjDelete(&pMemSolaris->Core);
643	return VERR_NO_PAGE_MEMORY;
644	}
645
646	pMemSolaris->Core.pv = pvMem;
647	pMemSolaris->pvHandle = NULL;
648	*ppMem = &pMemSolaris->Core;
649	return VINF_SUCCESS;
650	}
651
652
653	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
654	{
655	NOREF(fExecutable);
656
657	/* Create the object */
658	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOW, NULL, cb);
659	if (!pMemSolaris)
660	return VERR_NO_MEMORY;
661
662	/* Allocate physically low page-aligned memory. */
663	uint64_t uPhysHi = _4G - 1;
664	void pvMem = rtR0SolMemAlloc(uPhysHi, NULL / puPhys /, cb, PAGE_SIZE, false / fContig */);
665	if (RT_UNLIKELY(!pvMem))
666	{
667	rtR0MemObjDelete(&pMemSolaris->Core);
668	return VERR_NO_LOW_MEMORY;
669	}
670	pMemSolaris->Core.pv = pvMem;
671	pMemSolaris->pvHandle = NULL;
672	*ppMem = &pMemSolaris->Core;
673	return VINF_SUCCESS;
674	}
675
676
677	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
678	{
679	NOREF(fExecutable);
680	return rtR0MemObjNativeAllocPhys(ppMem, cb, _4G - 1, PAGE_SIZE /* alignment */);
681	}
682
683
684	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
685	{
686	#if HC_ARCH_BITS == 64
687	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS_NC, NULL, cb);
688	if (RT_UNLIKELY(!pMemSolaris))
689	return VERR_NO_MEMORY;
690
691	uint64_t PhysAddr = UINT64_MAX;
692	void *pvPages = rtR0MemObjSolPagesAlloc((uint64_t)PhysHighest, &PhysAddr, cb);
693	if (!pvPages)
694	{
695	LogRel(("rtR0MemObjNativeAllocPhysNC: rtR0MemObjSolPagesAlloc failed for cb=%u.\n", cb));
696	rtR0MemObjDelete(&pMemSolaris->Core);
697	return VERR_NO_MEMORY;
698	}
699	pMemSolaris->Core.pv = NULL;
700	pMemSolaris->pvHandle = pvPages;
701
702	Assert(PhysAddr != UINT64_MAX);
703	Assert(!(PhysAddr & PAGE_OFFSET_MASK));
704	*ppMem = &pMemSolaris->Core;
705	return VINF_SUCCESS;
706
707	#else /* 32 bit: */
708	return VERR_NOT_SUPPORTED; /* see the RTR0MemObjAllocPhysNC specs */
709	#endif
710	}
711
712
713	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
714	{
715	AssertMsgReturn(PhysHighest >= 16 *_1M, ("PhysHigest=%RHp\n", PhysHighest), VERR_NOT_SUPPORTED);
716
717	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS, NULL, cb);
718	if (RT_UNLIKELY(!pMemSolaris))
719	return VERR_NO_MEMORY;
720
721	/*
722	* Allocating one large page gets special treatment.
723	*/
724	static uint32_t s_cbLargePage = UINT32_MAX;
725	if (s_cbLargePage == UINT32_MAX)
726	{
727	if (page_num_pagesizes() > 1)
728	ASMAtomicWriteU32(&s_cbLargePage, page_get_pagesize(1)); /* Page-size code 1 maps to _2M on Solaris x86/amd64. */
729	else
730	ASMAtomicWriteU32(&s_cbLargePage, 0);
731	}
732	uint64_t PhysAddr;
733	if ( cb == s_cbLargePage
734	&& cb == uAlignment
735	&& PhysHighest == NIL_RTHCPHYS)
736	{
737	/*
738	* Allocate one large page (backed by physically contiguous memory).
739	*/
740	void *pvPages = rtR0MemObjSolLargePageAlloc(&PhysAddr, cb);
741	if (RT_LIKELY(pvPages))
742	{
743	AssertMsg(!(PhysAddr & (cb - 1)), ("%RHp\n", PhysAddr));
744	pMemSolaris->Core.pv = NULL;
745	pMemSolaris->Core.u.Phys.PhysBase = PhysAddr;
746	pMemSolaris->Core.u.Phys.fAllocated = true;
747	pMemSolaris->pvHandle = pvPages;
748	pMemSolaris->fLargePage = true;
749
750	*ppMem = &pMemSolaris->Core;
751	return VINF_SUCCESS;
752	}
753	}
754	else
755	{
756	/*
757	* Allocate physically contiguous memory aligned as specified.
758	* Note: contig_alloc() can be agonizingly slow for large (e.g. >= _2M) contiguous allocations.
759	* So we shouldn't ideally be in this path for large-page allocations. .
760	*/
761	AssertCompile(NIL_RTHCPHYS == UINT64_MAX); NOREF(RTASSERTVAR);
762	PhysAddr = PhysHighest;
763	void pvMem = rtR0SolMemAlloc(PhysHighest, &PhysAddr, cb, uAlignment, true / fContig */);
764	if (RT_LIKELY(pvMem))
765	{
766	Assert(!(PhysAddr & PAGE_OFFSET_MASK));
767	Assert(PhysAddr < PhysHighest);
768	Assert(PhysAddr + cb <= PhysHighest);
769
770	pMemSolaris->Core.pv = pvMem;
771	pMemSolaris->Core.u.Phys.PhysBase = PhysAddr;
772	pMemSolaris->Core.u.Phys.fAllocated = true;
773	pMemSolaris->pvHandle = NULL;
774	pMemSolaris->fLargePage = false;
775
776	*ppMem = &pMemSolaris->Core;
777	return VINF_SUCCESS;
778	}
779	}
780	rtR0MemObjDelete(&pMemSolaris->Core);
781	return VERR_NO_CONT_MEMORY;
782	}
783
784
785	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
786	{
787	AssertReturn(uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE, VERR_NOT_SUPPORTED);
788
789	/* Create the object. */
790	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS, NULL, cb);
791	if (!pMemSolaris)
792	return VERR_NO_MEMORY;
793
794	/* There is no allocation here, it needs to be mapped somewhere first. */
795	pMemSolaris->Core.u.Phys.fAllocated = false;
796	pMemSolaris->Core.u.Phys.PhysBase = Phys;
797	pMemSolaris->Core.u.Phys.uCachePolicy = uCachePolicy;
798	*ppMem = &pMemSolaris->Core;
799	return VINF_SUCCESS;
800	}
801
802
803	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess,
804	RTR0PROCESS R0Process)
805	{
806	AssertReturn(R0Process == RTR0ProcHandleSelf(), VERR_INVALID_PARAMETER);
807	NOREF(fAccess);
808
809	/* Create the locking object */
810	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(pMemSolaris), RTR0MEMOBJTYPE_LOCK, (void )R3Ptr, cb);
811	if (!pMemSolaris)
812	return VERR_NO_MEMORY;
813
814	/* Lock down user pages. */
815	int fPageAccess = S_READ;
816	if (fAccess & RTMEM_PROT_WRITE)
817	fPageAccess = S_WRITE;
818	if (fAccess & RTMEM_PROT_EXEC)
819	fPageAccess = S_EXEC;
820	int rc = rtR0MemObjSolLock((void *)R3Ptr, cb, fPageAccess);
821	if (RT_FAILURE(rc))
822	{
823	LogRel(("rtR0MemObjNativeLockUser: rtR0MemObjSolLock failed rc=%d\n", rc));
824	rtR0MemObjDelete(&pMemSolaris->Core);
825	return rc;
826	}
827
828	/* Fill in the object attributes and return successfully. */
829	pMemSolaris->Core.u.Lock.R0Process = R0Process;
830	pMemSolaris->pvHandle = NULL;
831	pMemSolaris->fAccess = fPageAccess;
832	*ppMem = &pMemSolaris->Core;
833	return VINF_SUCCESS;
834	}
835
836
837	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
838	{
839	NOREF(fAccess);
840
841	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOCK, pv, cb);
842	if (!pMemSolaris)
843	return VERR_NO_MEMORY;
844
845	/* Lock down kernel pages. */
846	int fPageAccess = S_READ;
847	if (fAccess & RTMEM_PROT_WRITE)
848	fPageAccess = S_WRITE;
849	if (fAccess & RTMEM_PROT_EXEC)
850	fPageAccess = S_EXEC;
851	int rc = rtR0MemObjSolLock(pv, cb, fPageAccess);
852	if (RT_FAILURE(rc))
853	{
854	LogRel(("rtR0MemObjNativeLockKernel: rtR0MemObjSolLock failed rc=%d\n", rc));
855	rtR0MemObjDelete(&pMemSolaris->Core);
856	return rc;
857	}
858
859	/* Fill in the object attributes and return successfully. */
860	pMemSolaris->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
861	pMemSolaris->pvHandle = NULL;
862	pMemSolaris->fAccess = fPageAccess;
863	*ppMem = &pMemSolaris->Core;
864	return VINF_SUCCESS;
865	}
866
867
868	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
869	{
870	PRTR0MEMOBJSOL pMemSolaris;
871
872	/*
873	* Use xalloc.
874	*/
875	void pv = vmem_xalloc(heap_arena, cb, uAlignment, 0 / phase /, 0 / nocross */,
876	NULL /* minaddr /, NULL / maxaddr */, VM_SLEEP);
877	if (RT_UNLIKELY(!pv))
878	return VERR_NO_MEMORY;
879
880	/* Create the object. */
881	pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
882	if (!pMemSolaris)
883	{
884	LogRel(("rtR0MemObjNativeReserveKernel failed to alloc memory object.\n"));
885	vmem_xfree(heap_arena, pv, cb);
886	return VERR_NO_MEMORY;
887	}
888
889	pMemSolaris->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
890	*ppMem = &pMemSolaris->Core;
891	return VINF_SUCCESS;
892	}
893
894
895	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment,
896	RTR0PROCESS R0Process)
897	{
898	return VERR_NOT_SUPPORTED;
899	}
900
901
902	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
903	unsigned fProt, size_t offSub, size_t cbSub)
904	{
905	/* Fail if requested to do something we can't. */
906	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
907	if (uAlignment > PAGE_SIZE)
908	return VERR_NOT_SUPPORTED;
909
910	/*
911	* Use xalloc to get address space.
912	*/
913	if (!cbSub)
914	cbSub = pMemToMap->cb;
915	void pv = vmem_xalloc(heap_arena, cbSub, uAlignment, 0 / phase /, 0 / nocross */,
916	NULL /* minaddr /, NULL / maxaddr */, VM_SLEEP);
917	if (RT_UNLIKELY(!pv))
918	return VERR_MAP_FAILED;
919
920	/*
921	* Load the pages from the other object into it.
922	*/
923	uint32_t fAttr = HAT_UNORDERED_OK \| HAT_MERGING_OK \| HAT_LOADCACHING_OK \| HAT_STORECACHING_OK;
924	if (fProt & RTMEM_PROT_READ)
925	fAttr \|= PROT_READ;
926	if (fProt & RTMEM_PROT_EXEC)
927	fAttr \|= PROT_EXEC;
928	if (fProt & RTMEM_PROT_WRITE)
929	fAttr \|= PROT_WRITE;
930	fAttr \|= HAT_NOSYNC;
931
932	int rc = VINF_SUCCESS;
933	size_t off = 0;
934	while (off < cbSub)
935	{
936	RTHCPHYS HCPhys = rtR0MemObjNativeGetPagePhysAddr(pMemToMap, (offSub + offSub) >> PAGE_SHIFT);
937	AssertBreakStmt(HCPhys != NIL_RTHCPHYS, rc = VERR_INTERNAL_ERROR_2);
938	pfn_t pfn = HCPhys >> PAGE_SHIFT;
939	AssertBreakStmt(((RTHCPHYS)pfn << PAGE_SHIFT) == HCPhys, rc = VERR_INTERNAL_ERROR_3);
940
941	hat_devload(kas.a_hat, (uint8_t *)pv + off, PAGE_SIZE, pfn, fAttr, HAT_LOAD_LOCK);
942
943	/* Advance. */
944	off += PAGE_SIZE;
945	}
946	if (RT_SUCCESS(rc))
947	{
948	/*
949	* Create a memory object for the mapping.
950	*/
951	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_MAPPING, pv, cbSub);
952	if (pMemSolaris)
953	{
954	pMemSolaris->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
955	*ppMem = &pMemSolaris->Core;
956	return VINF_SUCCESS;
957	}
958
959	LogRel(("rtR0MemObjNativeMapKernel failed to alloc memory object.\n"));
960	rc = VERR_NO_MEMORY;
961	}
962
963	if (off)
964	hat_unload(kas.a_hat, pv, off, HAT_UNLOAD \| HAT_UNLOAD_UNLOCK);
965	vmem_xfree(heap_arena, pv, cbSub);
966	return rc;
967	}
968
969
970	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, PRTR0MEMOBJINTERNAL pMemToMap, RTR3PTR R3PtrFixed,
971	size_t uAlignment, unsigned fProt, RTR0PROCESS R0Process)
972	{
973	/*
974	* Fend off things we cannot do.
975	*/
976	AssertMsgReturn(R3PtrFixed == (RTR3PTR)-1, ("%p\n", R3PtrFixed), VERR_NOT_SUPPORTED);
977	AssertMsgReturn(R0Process == RTR0ProcHandleSelf(), ("%p != %p\n", R0Process, RTR0ProcHandleSelf()), VERR_NOT_SUPPORTED);
978	if (uAlignment != PAGE_SIZE)
979	return VERR_NOT_SUPPORTED;
980
981	/*
982	* Get parameters from the source object.
983	*/
984	PRTR0MEMOBJSOL pMemToMapSolaris = (PRTR0MEMOBJSOL)pMemToMap;
985	void *pv = pMemToMapSolaris->Core.pv;
986	size_t cb = pMemToMapSolaris->Core.cb;
987	size_t cPages = (cb + PAGE_SIZE - 1) >> PAGE_SHIFT;
988
989	/*
990	* Create the mapping object
991	*/
992	PRTR0MEMOBJSOL pMemSolaris;
993	pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_MAPPING, pv, cb);
994	if (RT_UNLIKELY(!pMemSolaris))
995	return VERR_NO_MEMORY;
996
997	int rc = VINF_SUCCESS;
998	uint64_t paPhysAddrs = kmem_zalloc(sizeof(uint64_t) cPages, KM_SLEEP);
999	if (RT_LIKELY(paPhysAddrs))
1000	{
1001	/*
1002	* Prepare the pages for mapping according to type.
1003	*/
1004	if (pMemToMapSolaris->Core.enmType == RTR0MEMOBJTYPE_PHYS_NC)
1005	{
1006	page_t **ppPages = pMemToMapSolaris->pvHandle;
1007	for (size_t iPage = 0; iPage < cPages; iPage++)
1008	paPhysAddrs[iPage] = rtR0MemObjSolPagePhys(ppPages[iPage]);
1009	}
1010	else if ( pMemToMapSolaris->Core.enmType == RTR0MEMOBJTYPE_PHYS
1011	&& pMemToMapSolaris->fLargePage)
1012	{
1013	RTHCPHYS Phys = pMemToMapSolaris->Core.u.Phys.PhysBase;
1014	for (size_t iPage = 0; iPage < cPages; iPage++, Phys += PAGE_SIZE)
1015	paPhysAddrs[iPage] = Phys;
1016	}
1017	else
1018	{
1019	/*
1020	* Have kernel mapping, just translate virtual to physical.
1021	*/
1022	AssertPtr(pv);
1023	rc = VINF_SUCCESS;
1024	for (size_t iPage = 0; iPage < cPages; iPage++)
1025	{
1026	paPhysAddrs[iPage] = rtR0MemObjSolVirtToPhys(pv);
1027	if (RT_UNLIKELY(paPhysAddrs[iPage] == -(uint64_t)1))
1028	{
1029	LogRel(("rtR0MemObjNativeMapUser: no page to map.\n"));
1030	rc = VERR_MAP_FAILED;
1031	break;
1032	}
1033	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
1034	}
1035	}
1036	if (RT_SUCCESS(rc))
1037	{
1038	unsigned fPageAccess = PROT_READ;
1039	if (fProt & RTMEM_PROT_WRITE)
1040	fPageAccess \|= PROT_WRITE;
1041	if (fProt & RTMEM_PROT_EXEC)
1042	fPageAccess \|= PROT_EXEC;
1043
1044	/*
1045	* Perform the actual mapping.
1046	*/
1047	caddr_t UserAddr = NULL;
1048	rc = rtR0MemObjSolUserMap(&UserAddr, fPageAccess, paPhysAddrs, cb, PAGE_SIZE);
1049	if (RT_SUCCESS(rc))
1050	{
1051	pMemSolaris->Core.u.Mapping.R0Process = R0Process;
1052	pMemSolaris->Core.pv = UserAddr;
1053
1054	*ppMem = &pMemSolaris->Core;
1055	kmem_free(paPhysAddrs, sizeof(uint64_t) * cPages);
1056	return VINF_SUCCESS;
1057	}
1058
1059	LogRel(("rtR0MemObjNativeMapUser: rtR0MemObjSolUserMap failed rc=%d.\n", rc));
1060	}
1061
1062	rc = VERR_MAP_FAILED;
1063	kmem_free(paPhysAddrs, sizeof(uint64_t) * cPages);
1064	}
1065	else
1066	rc = VERR_NO_MEMORY;
1067	rtR0MemObjDelete(&pMemSolaris->Core);
1068	return rc;
1069	}
1070
1071
1072	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1073	{
1074	NOREF(pMem);
1075	NOREF(offSub);
1076	NOREF(cbSub);
1077	NOREF(fProt);
1078	return VERR_NOT_SUPPORTED;
1079	}
1080
1081
1082	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1083	{
1084	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)pMem;
1085
1086	switch (pMemSolaris->Core.enmType)
1087	{
1088	case RTR0MEMOBJTYPE_PHYS_NC:
1089	if (pMemSolaris->Core.u.Phys.fAllocated)
1090	{
1091	uint8_t pb = (uint8_t )pMemSolaris->Core.pv + ((size_t)iPage << PAGE_SHIFT);
1092	return rtR0MemObjSolVirtToPhys(pb);
1093	}
1094	page_t **ppPages = pMemSolaris->pvHandle;
1095	return rtR0MemObjSolPagePhys(ppPages[iPage]);
1096
1097	case RTR0MEMOBJTYPE_PAGE:
1098	case RTR0MEMOBJTYPE_LOW:
1099	case RTR0MEMOBJTYPE_LOCK:
1100	{
1101	uint8_t pb = (uint8_t )pMemSolaris->Core.pv + ((size_t)iPage << PAGE_SHIFT);
1102	return rtR0MemObjSolVirtToPhys(pb);
1103	}
1104
1105	/*
1106	* Although mapping can be handled by rtR0MemObjSolVirtToPhys(offset) like the above case,
1107	* request it from the parent so that we have a clear distinction between CONT/PHYS_NC.
1108	*/
1109	case RTR0MEMOBJTYPE_MAPPING:
1110	return rtR0MemObjNativeGetPagePhysAddr(pMemSolaris->Core.uRel.Child.pParent, iPage);
1111
1112	case RTR0MEMOBJTYPE_CONT:
1113	case RTR0MEMOBJTYPE_PHYS:
1114	AssertFailed(); /* handled by the caller */
1115	case RTR0MEMOBJTYPE_RES_VIRT:
1116	default:
1117	return NIL_RTHCPHYS;
1118	}
1119	}
1120

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/solaris/memobj-r0drv-solaris.c@ 41627

Download in other formats: