memobj-r0drv-solaris.c

Last change on this file was 106061, checked in by vboxsync, 3 months ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 44.2 KB

Line
1	/* $Id: memobj-r0drv-solaris.c 106061 2024-09-16 14:03:52Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Solaris.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* The contents of this file may alternatively be used under the terms
26	* of the Common Development and Distribution License Version 1.0
27	* (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	* in the VirtualBox distribution, in which case the provisions of the
29	* CDDL are applicable instead of those of the GPL.
30	*
31	* You may elect to license modified versions of this file under the
32	* terms and conditions of either the GPL or the CDDL or both.
33	*
34	* SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	*/
36
37
38	/*********************************************************************************************************************************
39	* Header Files *
40	*********************************************************************************************************************************/
41	#include "the-solaris-kernel.h"
42	#include "internal/iprt.h"
43	#include <iprt/memobj.h>
44
45	#include <iprt/asm.h>
46	#include <iprt/assert.h>
47	#include <iprt/err.h>
48	#include <iprt/log.h>
49	#include <iprt/mem.h>
50	#include <iprt/param.h>
51	#include <iprt/process.h>
52	#include <iprt/string.h>
53	#include "internal/memobj.h"
54	#include "memobj-r0drv-solaris.h"
55
56	extern caddr_t hat_kpm_pfn2va(pfn_t); /* Found in vm/hat.h on solaris 11.3, but not on older like 10u7. */
57
58
59	/*********************************************************************************************************************************
60	* Defined Constants And Macros *
61	*********************************************************************************************************************************/
62	#define SOL_IS_KRNL_ADDR(vx) ((uintptr_t)(vx) >= kernelbase)
63
64
65	/*********************************************************************************************************************************
66	* Structures and Typedefs *
67	*********************************************************************************************************************************/
68	/**
69	* The Solaris version of the memory object structure.
70	*/
71	typedef struct RTR0MEMOBJSOL
72	{
73	/** The core structure. */
74	RTR0MEMOBJINTERNAL Core;
75	/** Pointer to kernel memory cookie. */
76	ddi_umem_cookie_t Cookie;
77	/** Shadow locked pages. */
78	void *pvHandle;
79	/** Access during locking. */
80	int fAccess;
81	/** Set if large pages are involved in an RTR0MEMOBJTYPE_PHYS allocation. */
82	bool fLargePage;
83	/** Whether we have individual pages or a kernel-mapped virtual memory
84	* block in an RTR0MEMOBJTYPE_PHYS_NC allocation. */
85	bool fIndivPages;
86	/** Set if executable allocation - only RTR0MEMOBJTYPE_PHYS. */
87	bool fExecutable;
88	} RTR0MEMOBJSOL, *PRTR0MEMOBJSOL;
89
90
91	/*********************************************************************************************************************************
92	* Global Variables *
93	*********************************************************************************************************************************/
94	static vnode_t g_PageVnode;
95	static kmutex_t g_OffsetMtx;
96	static u_offset_t g_offPage;
97
98	static vnode_t g_LargePageVnode;
99	static kmutex_t g_LargePageOffsetMtx;
100	static u_offset_t g_offLargePage;
101	static bool g_fLargePageNoReloc;
102
103
104	/**
105	* Returns the physical address for a virtual address.
106	*
107	* @param pv The virtual address.
108	*
109	* @returns The physical address corresponding to @a pv.
110	*/
111	static uint64_t rtR0MemObjSolVirtToPhys(void *pv)
112	{
113	struct hat *pHat = NULL;
114	pfn_t PageFrameNum = 0;
115	uintptr_t uVirtAddr = (uintptr_t)pv;
116
117	if (SOL_IS_KRNL_ADDR(pv))
118	pHat = kas.a_hat;
119	else
120	{
121	proc_t pProcess = (proc_t )RTR0ProcHandleSelf();
122	AssertRelease(pProcess);
123	pHat = pProcess->p_as->a_hat;
124	}
125
126	PageFrameNum = hat_getpfnum(pHat, (caddr_t)(uVirtAddr & PAGEMASK));
127	AssertReleaseMsg(PageFrameNum != PFN_INVALID, ("rtR0MemObjSolVirtToPhys failed. pv=%p\n", pv));
128	return (((uint64_t)PageFrameNum << PAGE_SHIFT) \| (uVirtAddr & PAGE_OFFSET_MASK));
129	}
130
131
132	/**
133	* Returns the physical address for a page.
134	*
135	* @param pPage Pointer to the page.
136	*
137	* @returns The physical address for a page.
138	*/
139	static inline uint64_t rtR0MemObjSolPagePhys(page_t *pPage)
140	{
141	AssertPtr(pPage);
142	pfn_t PageFrameNum = page_pptonum(pPage);
143	AssertReleaseMsg(PageFrameNum != PFN_INVALID, ("rtR0MemObjSolPagePhys failed pPage=%p\n"));
144	return (uint64_t)PageFrameNum << PAGE_SHIFT;
145	}
146
147
148	/**
149	* Allocates one page.
150	*
151	* @param virtAddr The virtual address to which this page maybe mapped in
152	* the future.
153	*
154	* @returns Pointer to the allocated page, NULL on failure.
155	*/
156	static page_t *rtR0MemObjSolPageAlloc(caddr_t virtAddr)
157	{
158	u_offset_t offPage;
159	seg_t KernelSeg;
160
161	/*
162	* 16777215 terabytes of total memory for all VMs or
163	* restart 8000 1GB VMs 2147483 times until wraparound!
164	*/
165	mutex_enter(&g_OffsetMtx);
166	AssertCompileSize(u_offset_t, sizeof(uint64_t)); NOREF(RTASSERTVAR);
167	g_offPage = RT_ALIGN_64(g_offPage, PAGE_SIZE) + PAGE_SIZE;
168	offPage = g_offPage;
169	mutex_exit(&g_OffsetMtx);
170
171	KernelSeg.s_as = &kas;
172	page_t *pPage = page_create_va(&g_PageVnode, offPage, PAGE_SIZE, PG_WAIT \| PG_NORELOC, &KernelSeg, virtAddr);
173	if (RT_LIKELY(pPage))
174	{
175	/*
176	* Lock this page into memory "long term" to prevent this page from being paged out
177	* when we drop the page lock temporarily (during free). Downgrade to a shared lock
178	* to prevent page relocation.
179	*/
180	page_pp_lock(pPage, 0 /* COW /, 1 / Kernel */);
181	page_io_unlock(pPage);
182	page_downgrade(pPage);
183	Assert(PAGE_LOCKED_SE(pPage, SE_SHARED));
184	}
185
186	return pPage;
187	}
188
189
190	/**
191	* Destroys an allocated page.
192	*
193	* @param pPage Pointer to the page to be destroyed.
194	* @remarks This function expects page in @c pPage to be shared locked.
195	*/
196	static void rtR0MemObjSolPageDestroy(page_t *pPage)
197	{
198	/*
199	* We need to exclusive lock the pages before freeing them, if upgrading the shared lock to exclusive fails,
200	* drop the page lock and look it up from the hash. Record the page offset before we drop the page lock as
201	* we cannot touch any page_t members once the lock is dropped.
202	*/
203	AssertPtr(pPage);
204	Assert(PAGE_LOCKED_SE(pPage, SE_SHARED));
205
206	u_offset_t offPage = pPage->p_offset;
207	int rc = page_tryupgrade(pPage);
208	if (!rc)
209	{
210	page_unlock(pPage);
211	page_t *pFoundPage = page_lookup(&g_PageVnode, offPage, SE_EXCL);
212
213	/*
214	* Since we allocated the pages as PG_NORELOC we should only get back the exact page always.
215	*/
216	AssertReleaseMsg(pFoundPage == pPage, ("Page lookup failed %p:%llx returned %p, expected %p\n",
217	&g_PageVnode, offPage, pFoundPage, pPage));
218	}
219	Assert(PAGE_LOCKED_SE(pPage, SE_EXCL));
220	page_pp_unlock(pPage, 0 /* COW /, 1 / Kernel */);
221	page_destroy(pPage, 0 /* move it to the free list */);
222	}
223
224
225	/* Currently not used on 32-bits, define it to shut up gcc. */
226	#if HC_ARCH_BITS == 64
227	/**
228	* Allocates physical, non-contiguous memory of pages.
229	*
230	* @param puPhys Where to store the physical address of first page. Optional,
231	* can be NULL.
232	* @param cb The size of the allocation.
233	*
234	* @return Array of allocated pages, NULL on failure.
235	*/
236	static page_t *rtR0MemObjSolPagesAlloc(uint64_t puPhys, size_t cb)
237	{
238	/*
239	* VM1:
240	* The page freelist and cachelist both hold pages that are not mapped into any address space.
241	* The cachelist is not really free pages but when memory is exhausted they'll be moved to the
242	* free lists, it's the total of the free+cache list that we see on the 'free' column in vmstat.
243	*
244	* VM2:
245	* @todo Document what happens behind the scenes in VM2 regarding the free and cachelist.
246	*/
247
248	/*
249	* Non-pageable memory reservation request for _4K pages, don't sleep.
250	*/
251	size_t cPages = (cb + PAGE_SIZE - 1) >> PAGE_SHIFT;
252	int rc = page_resv(cPages, KM_NOSLEEP);
253	if (rc)
254	{
255	size_t cbPages = cPages * sizeof(page_t *);
256	page_t **ppPages = kmem_zalloc(cbPages, KM_SLEEP);
257	if (RT_LIKELY(ppPages))
258	{
259	/*
260	* Get pages from kseg, the 'virtAddr' here is only for colouring but unfortunately
261	* we don't yet have the 'virtAddr' to which this memory may be mapped.
262	*/
263	caddr_t virtAddr = 0;
264	for (size_t i = 0; i < cPages; i++, virtAddr += PAGE_SIZE)
265	{
266	/*
267	* Get a page from the free list locked exclusively. The page will be named (hashed in)
268	* and we rely on it during free. The page we get will be shared locked to prevent the page
269	* from being relocated.
270	*/
271	page_t *pPage = rtR0MemObjSolPageAlloc(virtAddr);
272	if (RT_UNLIKELY(!pPage))
273	{
274	/*
275	* No page found, release whatever pages we grabbed so far.
276	*/
277	for (size_t k = 0; k < i; k++)
278	rtR0MemObjSolPageDestroy(ppPages[k]);
279	kmem_free(ppPages, cbPages);
280	page_unresv(cPages);
281	return NULL;
282	}
283
284	ppPages[i] = pPage;
285	}
286
287	if (puPhys)
288	*puPhys = rtR0MemObjSolPagePhys(ppPages[0]);
289	return ppPages;
290	}
291
292	page_unresv(cPages);
293	}
294
295	return NULL;
296	}
297	#endif /* HC_ARCH_BITS == 64 */
298
299
300	/**
301	* Frees the allocates pages.
302	*
303	* @param ppPages Pointer to the page list.
304	* @param cbPages Size of the allocation.
305	*/
306	static void rtR0MemObjSolPagesFree(page_t **ppPages, size_t cb)
307	{
308	size_t cPages = (cb + PAGE_SIZE - 1) >> PAGE_SHIFT;
309	size_t cbPages = cPages * sizeof(page_t *);
310	for (size_t iPage = 0; iPage < cPages; iPage++)
311	rtR0MemObjSolPageDestroy(ppPages[iPage]);
312
313	kmem_free(ppPages, cbPages);
314	page_unresv(cPages);
315	}
316
317
318	/**
319	* Allocates one large page.
320	*
321	* @param puPhys Where to store the physical address of the allocated
322	* page. Optional, can be NULL.
323	* @param cbLargePage Size of the large page.
324	*
325	* @returns Pointer to a list of pages that cover the large page, NULL on
326	* failure.
327	*/
328	static page_t *rtR0MemObjSolLargePageAlloc(uint64_t puPhys, size_t cbLargePage)
329	{
330	/*
331	* Check PG_NORELOC support for large pages. Using this helps prevent _1G page
332	* fragementation on systems that support it.
333	*/
334	static bool fPageNoRelocChecked = false;
335	if (fPageNoRelocChecked == false)
336	{
337	fPageNoRelocChecked = true;
338	g_fLargePageNoReloc = false;
339	if ( g_pfnrtR0Sol_page_noreloc_supported
340	&& g_pfnrtR0Sol_page_noreloc_supported(cbLargePage))
341	{
342	g_fLargePageNoReloc = true;
343	}
344	}
345
346	/*
347	* Non-pageable memory reservation request for _4K pages, don't sleep.
348	*/
349	size_t cPages = (cbLargePage + PAGE_SIZE - 1) >> PAGE_SHIFT;
350	size_t cbPages = cPages * sizeof(page_t *);
351	u_offset_t offPage = 0;
352	int rc = page_resv(cPages, KM_NOSLEEP);
353	if (rc)
354	{
355	page_t **ppPages = kmem_zalloc(cbPages, KM_SLEEP);
356	if (RT_LIKELY(ppPages))
357	{
358	mutex_enter(&g_LargePageOffsetMtx);
359	AssertCompileSize(u_offset_t, sizeof(uint64_t)); NOREF(RTASSERTVAR);
360	g_offLargePage = RT_ALIGN_64(g_offLargePage, cbLargePage) + cbLargePage;
361	offPage = g_offLargePage;
362	mutex_exit(&g_LargePageOffsetMtx);
363
364	seg_t KernelSeg;
365	KernelSeg.s_as = &kas;
366	page_t *pRootPage = page_create_va_large(&g_LargePageVnode, offPage, cbLargePage,
367	PG_EXCL \| (g_fLargePageNoReloc ? PG_NORELOC : 0), &KernelSeg,
368	0 /* vaddr /,NULL / locality group */);
369	if (pRootPage)
370	{
371	/*
372	* Split it into sub-pages, downgrade each page to a shared lock to prevent page relocation.
373	*/
374	page_t *pPageList = pRootPage;
375	for (size_t iPage = 0; iPage < cPages; iPage++)
376	{
377	page_t *pPage = pPageList;
378	AssertPtr(pPage);
379	AssertMsg(page_pptonum(pPage) == iPage + page_pptonum(pRootPage),
380	("%p:%lx %lx+%lx\n", pPage, page_pptonum(pPage), iPage, page_pptonum(pRootPage)));
381	AssertMsg(pPage->p_szc == pRootPage->p_szc, ("Size code mismatch %p %d %d\n", pPage,
382	(int)pPage->p_szc, (int)pRootPage->p_szc));
383
384	/*
385	* Lock the page into memory "long term". This prevents callers of page_try_demote_pages() (such as the
386	* pageout scanner) from demoting the large page into smaller pages while we temporarily release the
387	* exclusive lock (during free). We pass "0, 1" since we've already accounted for availrmem during
388	* page_resv().
389	*/
390	page_pp_lock(pPage, 0 /* COW /, 1 / Kernel */);
391
392	page_sub(&pPageList, pPage);
393	page_io_unlock(pPage);
394	page_downgrade(pPage);
395	Assert(PAGE_LOCKED_SE(pPage, SE_SHARED));
396
397	ppPages[iPage] = pPage;
398	}
399	Assert(pPageList == NULL);
400	Assert(ppPages[0] == pRootPage);
401
402	uint64_t uPhys = rtR0MemObjSolPagePhys(pRootPage);
403	AssertMsg(!(uPhys & (cbLargePage - 1)), ("%llx %zx\n", uPhys, cbLargePage));
404	if (puPhys)
405	*puPhys = uPhys;
406	return ppPages;
407	}
408
409	/*
410	* Don't restore offPrev in case of failure (race condition), we have plenty of offset space.
411	* The offset must be unique (for the same vnode) or we'll encounter panics on page_create_va_large().
412	*/
413	kmem_free(ppPages, cbPages);
414	}
415
416	page_unresv(cPages);
417	}
418	return NULL;
419	}
420
421
422	/**
423	* Frees the large page.
424	*
425	* @param ppPages Pointer to the list of small pages that cover the
426	* large page.
427	* @param cbLargePage Size of the allocation (i.e. size of the large
428	* page).
429	*/
430	static void rtR0MemObjSolLargePageFree(page_t **ppPages, size_t cbLargePage)
431	{
432	Assert(ppPages);
433	Assert(cbLargePage > PAGE_SIZE);
434
435	bool fDemoted = false;
436	size_t cPages = (cbLargePage + PAGE_SIZE - 1) >> PAGE_SHIFT;
437	size_t cbPages = cPages * sizeof(page_t *);
438	page_t *pPageList = ppPages[0];
439
440	for (size_t iPage = 0; iPage < cPages; iPage++)
441	{
442	/*
443	* We need the pages exclusively locked, try upgrading the shared lock.
444	* If it fails, drop the shared page lock (cannot access any page_t members once this is done)
445	* and lookup the page from the page hash locking it exclusively.
446	*/
447	page_t *pPage = ppPages[iPage];
448	u_offset_t offPage = pPage->p_offset;
449	int rc = page_tryupgrade(pPage);
450	if (!rc)
451	{
452	page_unlock(pPage);
453	page_t *pFoundPage = page_lookup(&g_LargePageVnode, offPage, SE_EXCL);
454	AssertRelease(pFoundPage);
455
456	if (g_fLargePageNoReloc)
457	{
458	/*
459	* This can only be guaranteed if PG_NORELOC is used while allocating the pages.
460	*/
461	AssertReleaseMsg(pFoundPage == pPage,
462	("lookup failed %p:%llu returned %p, expected %p\n", &g_LargePageVnode, offPage,
463	pFoundPage, pPage));
464	}
465
466	/*
467	* Check for page demotion (regardless of relocation). Some places in Solaris (e.g. VM1 page_retire())
468	* could possibly demote the large page to _4K pages between our call to page_unlock() and page_lookup().
469	*/
470	if (page_get_pagecnt(pFoundPage->p_szc) == 1) /* Base size of only _4K associated with this page. */
471	fDemoted = true;
472	pPage = pFoundPage;
473	ppPages[iPage] = pFoundPage;
474	}
475	Assert(PAGE_LOCKED_SE(pPage, SE_EXCL));
476	page_pp_unlock(pPage, 0 /* COW /, 1 / Kernel */);
477	}
478
479	if (fDemoted)
480	{
481	for (size_t iPage = 0; iPage < cPages; iPage++)
482	{
483	Assert(page_get_pagecnt(ppPages[iPage]->p_szc) == 1);
484	page_destroy(ppPages[iPage], 0 /* move it to the free list */);
485	}
486	}
487	else
488	{
489	/*
490	* Although we shred the adjacent pages in the linked list, page_destroy_pages works on
491	* adjacent pages via array increments. So this does indeed free all the pages.
492	*/
493	AssertPtr(pPageList);
494	page_destroy_pages(pPageList);
495	}
496	kmem_free(ppPages, cbPages);
497	page_unresv(cPages);
498	}
499
500
501	/**
502	* Unmaps kernel/user-space mapped memory.
503	*
504	* @param pv Pointer to the mapped memory block.
505	* @param cb Size of the memory block.
506	*/
507	static void rtR0MemObjSolUnmap(void *pv, size_t cb)
508	{
509	if (SOL_IS_KRNL_ADDR(pv))
510	{
511	hat_unload(kas.a_hat, pv, cb, HAT_UNLOAD \| HAT_UNLOAD_UNLOCK);
512	vmem_free(heap_arena, pv, cb);
513	}
514	else
515	{
516	struct as pAddrSpace = ((proc_t )RTR0ProcHandleSelf())->p_as;
517	AssertPtr(pAddrSpace);
518	as_rangelock(pAddrSpace);
519	as_unmap(pAddrSpace, pv, cb);
520	as_rangeunlock(pAddrSpace);
521	}
522	}
523
524
525	/**
526	* Lock down memory mappings for a virtual address.
527	*
528	* @param pv Pointer to the memory to lock down.
529	* @param cb Size of the memory block.
530	* @param fAccess Page access rights (S_READ, S_WRITE, S_EXEC)
531	*
532	* @returns IPRT status code.
533	*/
534	static int rtR0MemObjSolLock(void *pv, size_t cb, int fPageAccess)
535	{
536	/*
537	* Kernel memory mappings on x86/amd64 are always locked, only handle user-space memory.
538	*/
539	if (!SOL_IS_KRNL_ADDR(pv))
540	{
541	proc_t pProc = (proc_t )RTR0ProcHandleSelf();
542	AssertPtr(pProc);
543	faultcode_t rc = as_fault(pProc->p_as->a_hat, pProc->p_as, (caddr_t)pv, cb, F_SOFTLOCK, fPageAccess);
544	if (rc)
545	{
546	LogRel(("rtR0MemObjSolLock failed for pv=%pv cb=%lx fPageAccess=%d rc=%d\n", pv, cb, fPageAccess, rc));
547	return VERR_LOCK_FAILED;
548	}
549	}
550	return VINF_SUCCESS;
551	}
552
553
554	/**
555	* Unlock memory mappings for a virtual address.
556	*
557	* @param pv Pointer to the locked memory.
558	* @param cb Size of the memory block.
559	* @param fPageAccess Page access rights (S_READ, S_WRITE, S_EXEC).
560	*/
561	static void rtR0MemObjSolUnlock(void *pv, size_t cb, int fPageAccess)
562	{
563	if (!SOL_IS_KRNL_ADDR(pv))
564	{
565	proc_t pProcess = (proc_t )RTR0ProcHandleSelf();
566	AssertPtr(pProcess);
567	as_fault(pProcess->p_as->a_hat, pProcess->p_as, (caddr_t)pv, cb, F_SOFTUNLOCK, fPageAccess);
568	}
569	}
570
571
572	/**
573	* Maps a list of physical pages into user address space.
574	*
575	* @param pVirtAddr Where to store the virtual address of the mapping.
576	* @param fPageAccess Page access rights (PROT_READ, PROT_WRITE,
577	* PROT_EXEC)
578	* @param paPhysAddrs Array of physical addresses to pages.
579	* @param cb Size of memory being mapped.
580	*
581	* @returns IPRT status code.
582	*/
583	static int rtR0MemObjSolUserMap(caddr_t pVirtAddr, unsigned fPageAccess, uint64_t paPhysAddrs, size_t cb, size_t cbPageSize)
584	{
585	struct as pAddrSpace = ((proc_t )RTR0ProcHandleSelf())->p_as;
586	int rc;
587	SEGVBOX_CRARGS Args;
588
589	Args.paPhysAddrs = paPhysAddrs;
590	Args.fPageAccess = fPageAccess;
591	Args.cbPageSize = cbPageSize;
592
593	as_rangelock(pAddrSpace);
594	if (g_frtSolOldMapAddr)
595	g_rtSolMapAddr.u.pfnSol_map_addr_old(pVirtAddr, cb, 0 /* offset /, 0 / vacalign */, MAP_SHARED);
596	else
597	g_rtSolMapAddr.u.pfnSol_map_addr(pVirtAddr, cb, 0 /* offset */, MAP_SHARED);
598	if (*pVirtAddr != NULL)
599	rc = as_map(pAddrSpace, *pVirtAddr, cb, rtR0SegVBoxSolCreate, &Args);
600	else
601	rc = ENOMEM;
602	as_rangeunlock(pAddrSpace);
603
604	return RTErrConvertFromErrno(rc);
605	}
606
607
608	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
609	{
610	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)pMem;
611
612	switch (pMemSolaris->Core.enmType)
613	{
614	case RTR0MEMOBJTYPE_LOW:
615	rtR0SolMemFree(pMemSolaris->Core.pv, pMemSolaris->Core.cb);
616	break;
617
618	case RTR0MEMOBJTYPE_PHYS:
619	if (pMemSolaris->Core.u.Phys.fAllocated)
620	{
621	if (pMemSolaris->fLargePage)
622	rtR0MemObjSolLargePageFree(pMemSolaris->pvHandle, pMemSolaris->Core.cb);
623	else
624	rtR0SolMemFree(pMemSolaris->Core.pv, pMemSolaris->Core.cb);
625	}
626	break;
627
628	case RTR0MEMOBJTYPE_PHYS_NC:
629	if (pMemSolaris->fIndivPages)
630	rtR0MemObjSolPagesFree(pMemSolaris->pvHandle, pMemSolaris->Core.cb);
631	else
632	rtR0SolMemFree(pMemSolaris->Core.pv, pMemSolaris->Core.cb);
633	break;
634
635	case RTR0MEMOBJTYPE_PAGE:
636	if (!pMemSolaris->fExecutable)
637	ddi_umem_free(pMemSolaris->Cookie);
638	else
639	segkmem_free(heaptext_arena, pMemSolaris->Core.pv, pMemSolaris->Core.cb);
640	break;
641
642	case RTR0MEMOBJTYPE_LOCK:
643	rtR0MemObjSolUnlock(pMemSolaris->Core.pv, pMemSolaris->Core.cb, pMemSolaris->fAccess);
644	break;
645
646	case RTR0MEMOBJTYPE_MAPPING:
647	rtR0MemObjSolUnmap(pMemSolaris->Core.pv, pMemSolaris->Core.cb);
648	break;
649
650	case RTR0MEMOBJTYPE_RES_VIRT:
651	if (pMemSolaris->Core.u.ResVirt.R0Process == NIL_RTR0PROCESS)
652	vmem_xfree(heap_arena, pMemSolaris->Core.pv, pMemSolaris->Core.cb);
653	else
654	AssertFailed();
655	break;
656
657	case RTR0MEMOBJTYPE_CONT: /* we don't use this type here. */
658	default:
659	AssertMsgFailed(("enmType=%d\n", pMemSolaris->Core.enmType));
660	return VERR_INTERNAL_ERROR;
661	}
662
663	return VINF_SUCCESS;
664	}
665
666
667	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
668	{
669	/* Create the object. */
670	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PAGE, NULL, cb, pszTag);
671	if (pMemSolaris)
672	{
673	void *pvMem;
674	if (!fExecutable)
675	{
676	pMemSolaris->Core.fFlags \|= RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC;
677	pvMem = ddi_umem_alloc(cb, DDI_UMEM_SLEEP, &pMemSolaris->Cookie);
678	}
679	else
680	{
681	pMemSolaris->Core.fFlags \|= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC; /** @todo does segkmem_alloc zero the memory? */
682	pvMem = segkmem_alloc(heaptext_arena, cb, KM_SLEEP);
683	}
684	if (pvMem)
685	{
686	pMemSolaris->Core.pv = pvMem;
687	pMemSolaris->pvHandle = NULL;
688	pMemSolaris->fExecutable = fExecutable;
689	*ppMem = &pMemSolaris->Core;
690	return VINF_SUCCESS;
691	}
692	rtR0MemObjDelete(&pMemSolaris->Core);
693	return VERR_NO_PAGE_MEMORY;
694	}
695	return VERR_NO_MEMORY;
696	}
697
698
699	DECLHIDDEN(int) rtR0MemObjNativeAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags,
700	const char *pszTag)
701	{
702	return rtR0MemObjFallbackAllocLarge(ppMem, cb, cbLargePage, fFlags, pszTag);
703	}
704
705
706	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
707	{
708	AssertReturn(!fExecutable, VERR_NOT_SUPPORTED);
709
710	/* Create the object */
711	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOW, NULL, cb, pszTag);
712	if (pMemSolaris)
713	{
714	/* Allocate physically low page-aligned memory. */
715	uint64_t uPhysHi = _4G - 1;
716	void pvMem = rtR0SolMemAlloc(uPhysHi, NULL / puPhys /, cb, PAGE_SIZE, false / fContig */);
717	if (pvMem)
718	{
719	pMemSolaris->Core.fFlags \|= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC;
720	pMemSolaris->Core.pv = pvMem;
721	pMemSolaris->pvHandle = NULL;
722	*ppMem = &pMemSolaris->Core;
723	return VINF_SUCCESS;
724	}
725	rtR0MemObjDelete(&pMemSolaris->Core);
726	return VERR_NO_LOW_MEMORY;
727	}
728	return VERR_NO_MEMORY;
729	}
730
731
732	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest,
733	bool fExecutable, const char *pszTag)
734	{
735	AssertReturn(!fExecutable, VERR_NOT_SUPPORTED);
736	return rtR0MemObjNativeAllocPhys(ppMem, cb, PhysHighest, PAGE_SIZE /* alignment */, pszTag);
737	}
738
739
740	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, const char *pszTag)
741	{
742	#if HC_ARCH_BITS == 64
743	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS_NC, NULL, cb, pszTag);
744	if (pMemSolaris)
745	{
746	if (PhysHighest == NIL_RTHCPHYS)
747	{
748	uint64_t PhysAddr = UINT64_MAX;
749	void *pvPages = rtR0MemObjSolPagesAlloc(&PhysAddr, cb);
750	if (!pvPages)
751	{
752	LogRel(("rtR0MemObjNativeAllocPhysNC: rtR0MemObjSolPagesAlloc failed for cb=%u.\n", cb));
753	rtR0MemObjDelete(&pMemSolaris->Core);
754	return VERR_NO_MEMORY;
755	}
756	Assert(PhysAddr != UINT64_MAX);
757	Assert(!(PhysAddr & PAGE_OFFSET_MASK));
758
759	pMemSolaris->Core.pv = NULL;
760	pMemSolaris->pvHandle = pvPages;
761	pMemSolaris->fIndivPages = true;
762	}
763	else
764	{
765	/*
766	* If we must satisfy an upper limit constraint, it isn't feasible to grab individual pages.
767	* We fall back to using contig_alloc().
768	*/
769	uint64_t PhysAddr = UINT64_MAX;
770	void pvMem = rtR0SolMemAlloc(PhysHighest, &PhysAddr, cb, PAGE_SIZE, false / fContig */);
771	if (!pvMem)
772	{
773	LogRel(("rtR0MemObjNativeAllocPhysNC: rtR0SolMemAlloc failed for cb=%u PhysHighest=%RHp.\n", cb, PhysHighest));
774	rtR0MemObjDelete(&pMemSolaris->Core);
775	return VERR_NO_MEMORY;
776	}
777	Assert(PhysAddr != UINT64_MAX);
778	Assert(!(PhysAddr & PAGE_OFFSET_MASK));
779
780	pMemSolaris->Core.pv = pvMem;
781	pMemSolaris->pvHandle = NULL;
782	pMemSolaris->fIndivPages = false;
783	}
784	pMemSolaris->Core.fFlags \|= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC;
785	*ppMem = &pMemSolaris->Core;
786	return VINF_SUCCESS;
787	}
788	return VERR_NO_MEMORY;
789
790	#else /* 32 bit: */
791	return VERR_NOT_SUPPORTED; /* see the RTR0MemObjAllocPhysNC specs */
792	#endif
793	}
794
795
796	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment,
797	const char *pszTag)
798	{
799	AssertMsgReturn(PhysHighest >= 16 *_1M, ("PhysHigest=%RHp\n", PhysHighest), VERR_NOT_SUPPORTED);
800
801	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS, NULL, cb, pszTag);
802	if (RT_UNLIKELY(!pMemSolaris))
803	return VERR_NO_MEMORY;
804
805	/*
806	* Allocating one large page gets special treatment.
807	*/
808	static uint32_t s_cbLargePage = UINT32_MAX;
809	if (s_cbLargePage == UINT32_MAX)
810	{
811	if (page_num_pagesizes() > 1)
812	ASMAtomicWriteU32(&s_cbLargePage, page_get_pagesize(1)); /* Page-size code 1 maps to _2M on Solaris x86/amd64. */
813	else
814	ASMAtomicWriteU32(&s_cbLargePage, 0);
815	}
816
817	uint64_t PhysAddr;
818	if ( cb == s_cbLargePage
819	&& cb == uAlignment
820	&& PhysHighest == NIL_RTHCPHYS)
821	{
822	/*
823	* Allocate one large page (backed by physically contiguous memory).
824	*/
825	void *pvPages = rtR0MemObjSolLargePageAlloc(&PhysAddr, cb);
826	if (RT_LIKELY(pvPages))
827	{
828	AssertMsg(!(PhysAddr & (cb - 1)), ("%RHp\n", PhysAddr));
829	pMemSolaris->Core.fFlags \|= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC; /?/
830	pMemSolaris->Core.pv = NULL;
831	pMemSolaris->Core.u.Phys.PhysBase = PhysAddr;
832	pMemSolaris->Core.u.Phys.fAllocated = true;
833	pMemSolaris->pvHandle = pvPages;
834	pMemSolaris->fLargePage = true;
835
836	*ppMem = &pMemSolaris->Core;
837	return VINF_SUCCESS;
838	}
839	}
840	else
841	{
842	/*
843	* Allocate physically contiguous memory aligned as specified.
844	*/
845	AssertCompile(NIL_RTHCPHYS == UINT64_MAX); NOREF(RTASSERTVAR);
846	PhysAddr = PhysHighest;
847	void pvMem = rtR0SolMemAlloc(PhysHighest, &PhysAddr, cb, uAlignment, true / fContig */);
848	if (RT_LIKELY(pvMem))
849	{
850	Assert(!(PhysAddr & PAGE_OFFSET_MASK));
851	Assert(PhysAddr < PhysHighest);
852	Assert(PhysAddr + cb <= PhysHighest);
853
854	pMemSolaris->Core.fFlags \|= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC;
855	pMemSolaris->Core.pv = pvMem;
856	pMemSolaris->Core.u.Phys.PhysBase = PhysAddr;
857	pMemSolaris->Core.u.Phys.fAllocated = true;
858	pMemSolaris->pvHandle = NULL;
859	pMemSolaris->fLargePage = false;
860
861	*ppMem = &pMemSolaris->Core;
862	return VINF_SUCCESS;
863	}
864	}
865	rtR0MemObjDelete(&pMemSolaris->Core);
866	return VERR_NO_CONT_MEMORY;
867	}
868
869
870	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy,
871	const char *pszTag)
872	{
873	AssertReturn(uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE, VERR_NOT_SUPPORTED);
874
875	/* Create the object. */
876	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS, NULL, cb, pszTag);
877	if (!pMemSolaris)
878	return VERR_NO_MEMORY;
879
880	/* There is no allocation here, it needs to be mapped somewhere first. */
881	pMemSolaris->Core.u.Phys.fAllocated = false;
882	pMemSolaris->Core.u.Phys.PhysBase = Phys;
883	pMemSolaris->Core.u.Phys.uCachePolicy = uCachePolicy;
884	*ppMem = &pMemSolaris->Core;
885	return VINF_SUCCESS;
886	}
887
888
889	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess,
890	RTR0PROCESS R0Process, const char *pszTag)
891	{
892	AssertReturn(R0Process == RTR0ProcHandleSelf(), VERR_INVALID_PARAMETER);
893	NOREF(fAccess);
894
895	/* Create the locking object */
896	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOCK,
897	(void *)R3Ptr, cb, pszTag);
898	if (!pMemSolaris)
899	return VERR_NO_MEMORY;
900
901	/* Lock down user pages. */
902	int fPageAccess = S_READ;
903	if (fAccess & RTMEM_PROT_WRITE)
904	fPageAccess = S_WRITE;
905	if (fAccess & RTMEM_PROT_EXEC)
906	fPageAccess = S_EXEC;
907	int rc = rtR0MemObjSolLock((void *)R3Ptr, cb, fPageAccess);
908	if (RT_FAILURE(rc))
909	{
910	LogRel(("rtR0MemObjNativeLockUser: rtR0MemObjSolLock failed rc=%d\n", rc));
911	rtR0MemObjDelete(&pMemSolaris->Core);
912	return rc;
913	}
914
915	/* Fill in the object attributes and return successfully. */
916	pMemSolaris->Core.u.Lock.R0Process = R0Process;
917	pMemSolaris->pvHandle = NULL;
918	pMemSolaris->fAccess = fPageAccess;
919	*ppMem = &pMemSolaris->Core;
920	return VINF_SUCCESS;
921	}
922
923
924	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void pv, size_t cb, uint32_t fAccess, const char pszTag)
925	{
926	NOREF(fAccess);
927
928	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOCK, pv, cb, pszTag);
929	if (!pMemSolaris)
930	return VERR_NO_MEMORY;
931
932	/* Lock down kernel pages. */
933	int fPageAccess = S_READ;
934	if (fAccess & RTMEM_PROT_WRITE)
935	fPageAccess = S_WRITE;
936	if (fAccess & RTMEM_PROT_EXEC)
937	fPageAccess = S_EXEC;
938	int rc = rtR0MemObjSolLock(pv, cb, fPageAccess);
939	if (RT_FAILURE(rc))
940	{
941	LogRel(("rtR0MemObjNativeLockKernel: rtR0MemObjSolLock failed rc=%d\n", rc));
942	rtR0MemObjDelete(&pMemSolaris->Core);
943	return rc;
944	}
945
946	/* Fill in the object attributes and return successfully. */
947	pMemSolaris->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
948	pMemSolaris->pvHandle = NULL;
949	pMemSolaris->fAccess = fPageAccess;
950	*ppMem = &pMemSolaris->Core;
951	return VINF_SUCCESS;
952	}
953
954
955	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment,
956	const char *pszTag)
957	{
958	PRTR0MEMOBJSOL pMemSolaris;
959
960	/*
961	* Use xalloc.
962	*/
963	void pv = vmem_xalloc(heap_arena, cb, uAlignment, 0 / phase /, 0 / nocross */,
964	NULL /* minaddr /, NULL / maxaddr */, VM_SLEEP);
965	if (RT_UNLIKELY(!pv))
966	return VERR_NO_MEMORY;
967
968	/* Create the object. */
969	pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_RES_VIRT, pv, cb, pszTag);
970	if (!pMemSolaris)
971	{
972	LogRel(("rtR0MemObjNativeReserveKernel failed to alloc memory object.\n"));
973	vmem_xfree(heap_arena, pv, cb);
974	return VERR_NO_MEMORY;
975	}
976
977	pMemSolaris->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
978	*ppMem = &pMemSolaris->Core;
979	return VINF_SUCCESS;
980	}
981
982
983	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment,
984	RTR0PROCESS R0Process, const char *pszTag)
985	{
986	RT_NOREF(ppMem, R3PtrFixed, cb, uAlignment, R0Process, pszTag);
987	return VERR_NOT_SUPPORTED;
988	}
989
990
991	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
992	unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag)
993	{
994	/* Fail if requested to do something we can't. */
995	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
996	if (uAlignment > PAGE_SIZE)
997	return VERR_NOT_SUPPORTED;
998
999	/*
1000	* Use xalloc to get address space.
1001	*/
1002	if (!cbSub)
1003	cbSub = pMemToMap->cb;
1004	void pv = vmem_xalloc(heap_arena, cbSub, uAlignment, 0 / phase /, 0 / nocross */,
1005	NULL /* minaddr /, NULL / maxaddr */, VM_SLEEP);
1006	if (RT_UNLIKELY(!pv))
1007	return VERR_MAP_FAILED;
1008
1009	/*
1010	* Load the pages from the other object into it.
1011	*/
1012	uint32_t fAttr = HAT_UNORDERED_OK \| HAT_MERGING_OK \| HAT_LOADCACHING_OK \| HAT_STORECACHING_OK;
1013	if (fProt & RTMEM_PROT_READ)
1014	fAttr \|= PROT_READ;
1015	if (fProt & RTMEM_PROT_EXEC)
1016	fAttr \|= PROT_EXEC;
1017	if (fProt & RTMEM_PROT_WRITE)
1018	fAttr \|= PROT_WRITE;
1019	fAttr \|= HAT_NOSYNC;
1020
1021	int rc = VINF_SUCCESS;
1022	size_t off = 0;
1023	while (off < cbSub)
1024	{
1025	RTHCPHYS HCPhys = RTR0MemObjGetPagePhysAddr(pMemToMap, (offSub + off) >> PAGE_SHIFT);
1026	AssertBreakStmt(HCPhys != NIL_RTHCPHYS, rc = VERR_INTERNAL_ERROR_2);
1027	pfn_t pfn = HCPhys >> PAGE_SHIFT;
1028	AssertBreakStmt(((RTHCPHYS)pfn << PAGE_SHIFT) == HCPhys, rc = VERR_INTERNAL_ERROR_3);
1029
1030	hat_devload(kas.a_hat, (uint8_t *)pv + off, PAGE_SIZE, pfn, fAttr, HAT_LOAD_LOCK);
1031
1032	/* Advance. */
1033	off += PAGE_SIZE;
1034	}
1035	if (RT_SUCCESS(rc))
1036	{
1037	/*
1038	* Create a memory object for the mapping.
1039	*/
1040	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_MAPPING,
1041	pv, cbSub, pszTag);
1042	if (pMemSolaris)
1043	{
1044	pMemSolaris->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1045	*ppMem = &pMemSolaris->Core;
1046	return VINF_SUCCESS;
1047	}
1048
1049	LogRel(("rtR0MemObjNativeMapKernel failed to alloc memory object.\n"));
1050	rc = VERR_NO_MEMORY;
1051	}
1052
1053	if (off)
1054	hat_unload(kas.a_hat, pv, off, HAT_UNLOAD \| HAT_UNLOAD_UNLOCK);
1055	vmem_xfree(heap_arena, pv, cbSub);
1056	return rc;
1057	}
1058
1059
1060	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, PRTR0MEMOBJINTERNAL pMemToMap, RTR3PTR R3PtrFixed,
1061	size_t uAlignment, unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub,
1062	const char *pszTag)
1063	{
1064	/*
1065	* Fend off things we cannot do.
1066	*/
1067	AssertMsgReturn(R3PtrFixed == (RTR3PTR)-1, ("%p\n", R3PtrFixed), VERR_NOT_SUPPORTED);
1068	AssertMsgReturn(R0Process == RTR0ProcHandleSelf(), ("%p != %p\n", R0Process, RTR0ProcHandleSelf()), VERR_NOT_SUPPORTED);
1069	if (uAlignment != PAGE_SIZE)
1070	return VERR_NOT_SUPPORTED;
1071
1072	/*
1073	* Get parameters from the source object and offSub/cbSub.
1074	*/
1075	PRTR0MEMOBJSOL pMemToMapSolaris = (PRTR0MEMOBJSOL)pMemToMap;
1076	uint8_t pb = pMemToMapSolaris->Core.pv ? (uint8_t )pMemToMapSolaris->Core.pv + offSub : NULL;
1077	size_t const cb = cbSub ? cbSub : pMemToMapSolaris->Core.cb;
1078	size_t const cPages = cb >> PAGE_SHIFT;
1079	Assert(!offSub \|\| cbSub);
1080	Assert(!(cb & PAGE_OFFSET_MASK));
1081
1082	/*
1083	* Create the mapping object
1084	*/
1085	PRTR0MEMOBJSOL pMemSolaris;
1086	pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_MAPPING, pb, cb, pszTag);
1087	if (RT_UNLIKELY(!pMemSolaris))
1088	return VERR_NO_MEMORY;
1089
1090	/*
1091	* Gather the physical page address of the pages to be mapped.
1092	*/
1093	int rc = VINF_SUCCESS;
1094	uint64_t paPhysAddrs = kmem_zalloc(sizeof(uint64_t) cPages, KM_SLEEP);
1095	if (RT_LIKELY(paPhysAddrs))
1096	{
1097	if ( pMemToMapSolaris->Core.enmType == RTR0MEMOBJTYPE_PHYS_NC
1098	&& pMemToMapSolaris->fIndivPages)
1099	{
1100	/* Translate individual page_t to physical addresses. */
1101	page_t **papPages = pMemToMapSolaris->pvHandle;
1102	AssertPtr(papPages);
1103	papPages += offSub >> PAGE_SHIFT;
1104	for (size_t iPage = 0; iPage < cPages; iPage++)
1105	paPhysAddrs[iPage] = rtR0MemObjSolPagePhys(papPages[iPage]);
1106	}
1107	else if ( pMemToMapSolaris->Core.enmType == RTR0MEMOBJTYPE_PHYS
1108	&& pMemToMapSolaris->fLargePage)
1109	{
1110	/* Split up the large page into page-sized chunks. */
1111	RTHCPHYS Phys = pMemToMapSolaris->Core.u.Phys.PhysBase;
1112	Phys += offSub;
1113	for (size_t iPage = 0; iPage < cPages; iPage++, Phys += PAGE_SIZE)
1114	paPhysAddrs[iPage] = Phys;
1115	}
1116	else
1117	{
1118	/* Have kernel mapping, just translate virtual to physical. */
1119	AssertPtr(pb);
1120	for (size_t iPage = 0; iPage < cPages; iPage++)
1121	{
1122	paPhysAddrs[iPage] = rtR0MemObjSolVirtToPhys(pb);
1123	if (RT_UNLIKELY(paPhysAddrs[iPage] == -(uint64_t)1))
1124	{
1125	LogRel(("rtR0MemObjNativeMapUser: no page to map.\n"));
1126	rc = VERR_MAP_FAILED;
1127	break;
1128	}
1129	pb += PAGE_SIZE;
1130	}
1131	}
1132	if (RT_SUCCESS(rc))
1133	{
1134	/*
1135	* Perform the actual mapping.
1136	*/
1137	unsigned fPageAccess = PROT_READ;
1138	if (fProt & RTMEM_PROT_WRITE)
1139	fPageAccess \|= PROT_WRITE;
1140	if (fProt & RTMEM_PROT_EXEC)
1141	fPageAccess \|= PROT_EXEC;
1142
1143	caddr_t UserAddr = NULL;
1144	rc = rtR0MemObjSolUserMap(&UserAddr, fPageAccess, paPhysAddrs, cb, PAGE_SIZE);
1145	if (RT_SUCCESS(rc))
1146	{
1147	pMemSolaris->Core.u.Mapping.R0Process = R0Process;
1148	pMemSolaris->Core.pv = UserAddr;
1149
1150	*ppMem = &pMemSolaris->Core;
1151	kmem_free(paPhysAddrs, sizeof(uint64_t) * cPages);
1152	return VINF_SUCCESS;
1153	}
1154
1155	LogRel(("rtR0MemObjNativeMapUser: rtR0MemObjSolUserMap failed rc=%d.\n", rc));
1156	}
1157
1158	rc = VERR_MAP_FAILED;
1159	kmem_free(paPhysAddrs, sizeof(uint64_t) * cPages);
1160	}
1161	else
1162	rc = VERR_NO_MEMORY;
1163	rtR0MemObjDelete(&pMemSolaris->Core);
1164	return rc;
1165	}
1166
1167
1168	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1169	{
1170	NOREF(pMem);
1171	NOREF(offSub);
1172	NOREF(cbSub);
1173	NOREF(fProt);
1174	return VERR_NOT_SUPPORTED;
1175	}
1176
1177
1178	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1179	{
1180	PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)pMem;
1181
1182	switch (pMemSolaris->Core.enmType)
1183	{
1184	case RTR0MEMOBJTYPE_PHYS_NC:
1185	if ( pMemSolaris->Core.u.Phys.fAllocated
1186	\|\| !pMemSolaris->fIndivPages)
1187	{
1188	uint8_t pb = (uint8_t )pMemSolaris->Core.pv + ((size_t)iPage << PAGE_SHIFT);
1189	return rtR0MemObjSolVirtToPhys(pb);
1190	}
1191	page_t **ppPages = pMemSolaris->pvHandle;
1192	return rtR0MemObjSolPagePhys(ppPages[iPage]);
1193
1194	case RTR0MEMOBJTYPE_PAGE:
1195	case RTR0MEMOBJTYPE_LOW:
1196	case RTR0MEMOBJTYPE_LOCK:
1197	{
1198	uint8_t pb = (uint8_t )pMemSolaris->Core.pv + ((size_t)iPage << PAGE_SHIFT);
1199	return rtR0MemObjSolVirtToPhys(pb);
1200	}
1201
1202	/*
1203	* Although mapping can be handled by rtR0MemObjSolVirtToPhys(offset) like the above case,
1204	* request it from the parent so that we have a clear distinction between CONT/PHYS_NC.
1205	*/
1206	case RTR0MEMOBJTYPE_MAPPING:
1207	return rtR0MemObjNativeGetPagePhysAddr(pMemSolaris->Core.uRel.Child.pParent, iPage);
1208
1209	case RTR0MEMOBJTYPE_CONT:
1210	case RTR0MEMOBJTYPE_PHYS:
1211	AssertFailed(); /* handled by the caller */
1212	case RTR0MEMOBJTYPE_RES_VIRT:
1213	default:
1214	return NIL_RTHCPHYS;
1215	}
1216	}
1217
1218
1219	DECLHIDDEN(int) rtR0MemObjNativeZeroInitWithoutMapping(PRTR0MEMOBJINTERNAL pMem)
1220	{
1221	#ifdef RT_ARCH_AMD64
1222	PRTR0MEMOBJSOL const pMemSolaris = (PRTR0MEMOBJSOL)pMem;
1223	size_t const cPages = pMemSolaris->Core.cb >> PAGE_SHIFT;
1224	size_t iPage;
1225	for (iPage = 0; iPage < cPages; iPage++)
1226	{
1227	void *pvPage;
1228
1229	/* Get the physical address of the page. */
1230	RTHCPHYS HCPhys = rtR0MemObjNativeGetPagePhysAddr(&pMemSolaris->Core, iPage);
1231	AssertReturn(HCPhys != NIL_RTHCPHYS, VERR_INTERNAL_ERROR_3);
1232	Assert(!(HCPhys & PAGE_OFFSET_MASK));
1233
1234	/* Map it. */
1235	HCPhys >>= PAGE_SHIFT;
1236	AssertReturn(HCPhys <= physmax, VERR_INTERNAL_ERROR_3);
1237	pvPage = hat_kpm_pfn2va(HCPhys);
1238	AssertPtrReturn(pvPage, VERR_INTERNAL_ERROR_3);
1239
1240	/* Zero it. */
1241	RT_BZERO(pvPage, PAGE_SIZE);
1242	}
1243	return VINF_SUCCESS;
1244	#else
1245	RT_NOREF(pMem);
1246	return VERR_NOT_IMPLEMENTED;
1247	#endif
1248	}
1249

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/solaris/memobj-r0drv-solaris.c

Download in other formats: