memobj-r0drv-linux.c@ 85516

Last change on this file since 85516 was 85516, checked in by vboxsync, 4 years ago
IPRT/memobj-r0drv-linux.c: Enable IPRT_USE_ALLOC_VM_AREA_FOR_EXEC for linux 3.2 and later. bugref:9801
Property svn:eol-style set to `native` Property svn:keywords set to `Id Rev Revision`
File size: 64.4 KB

Line
1	/* $Id: memobj-r0drv-linux.c 85516 2020-07-29 10:47:38Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-linux-kernel.h"
32
33	#include <iprt/memobj.h>
34	#include <iprt/assert.h>
35	#include <iprt/err.h>
36	#include <iprt/log.h>
37	#include <iprt/mem.h>
38	#include <iprt/process.h>
39	#include <iprt/string.h>
40	#include "internal/memobj.h"
41	#include "internal/iprt.h"
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	/* early 2.6 kernels */
48	#ifndef PAGE_SHARED_EXEC
49	# define PAGE_SHARED_EXEC PAGE_SHARED
50	#endif
51	#ifndef PAGE_READONLY_EXEC
52	# define PAGE_READONLY_EXEC PAGE_READONLY
53	#endif
54
55	/** @def IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
56	* Whether we use alloc_vm_area (3.2+) for executable memory.
57	* This is a must for 5.8+, but we enable it all the way back to 3.2.x for
58	* better W^R compliance (fExecutable flag). */
59	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0) \|\| defined(DOXYGEN_RUNNING)
60	# define IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
61	#endif
62
63	/*
64	* 2.6.29+ kernels don't work with remap_pfn_range() anymore because
65	* track_pfn_vma_new() is apparently not defined for non-RAM pages.
66	* It should be safe to use vm_insert_page() older kernels as well.
67	*/
68	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
69	# define VBOX_USE_INSERT_PAGE
70	#endif
71	#if defined(CONFIG_X86_PAE) \
72	&& ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
73	\|\| ( LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \
74	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)))
75	# define VBOX_USE_PAE_HACK
76	#endif
77
78	/* gfp_t was introduced in 2.6.14, define it for earlier. */
79	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14)
80	# define gfp_t unsigned
81	#endif
82
83	/*
84	* Wrappers around mmap_lock/mmap_sem difference.
85	*/
86	#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)
87	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_lock)
88	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_lock)
89	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_lock)
90	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_lock)
91	#else
92	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_sem)
93	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_sem)
94	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_sem)
95	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_sem)
96	#endif
97
98
99	/*********************************************************************************************************************************
100	* Structures and Typedefs *
101	*********************************************************************************************************************************/
102	/**
103	* The Linux version of the memory object structure.
104	*/
105	typedef struct RTR0MEMOBJLNX
106	{
107	/** The core structure. */
108	RTR0MEMOBJINTERNAL Core;
109	/** Set if the allocation is contiguous.
110	* This means it has to be given back as one chunk. */
111	bool fContiguous;
112	/** Set if executable allocation. */
113	bool fExecutable;
114	/** Set if we've vmap'ed the memory into ring-0. */
115	bool fMappedToRing0;
116	#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
117	/** Return from alloc_vm_area() that we now need to use for executable
118	* memory. */
119	struct vm_struct *pArea;
120	/** PTE array that goes along with pArea (must be freed). */
121	pte_t **papPtesForArea;
122	#endif
123	/** The pages in the apPages array. */
124	size_t cPages;
125	/** Array of struct page pointers. (variable size) */
126	struct page *apPages[1];
127	} RTR0MEMOBJLNX;
128	/** Pointer to the linux memory object. */
129	typedef RTR0MEMOBJLNX *PRTR0MEMOBJLNX;
130
131
132	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
133
134
135	/**
136	* Helper that converts from a RTR0PROCESS handle to a linux task.
137	*
138	* @returns The corresponding Linux task.
139	* @param R0Process IPRT ring-0 process handle.
140	*/
141	static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
142	{
143	/** @todo fix rtR0ProcessToLinuxTask!! */
144	/** @todo many (all?) callers currently assume that we return 'current'! */
145	return R0Process == RTR0ProcHandleSelf() ? current : NULL;
146	}
147
148
149	/**
150	* Compute order. Some functions allocate 2^order pages.
151	*
152	* @returns order.
153	* @param cPages Number of pages.
154	*/
155	static int rtR0MemObjLinuxOrder(size_t cPages)
156	{
157	int iOrder;
158	size_t cTmp;
159
160	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
161	;
162	if (cPages & ~((size_t)1 << iOrder))
163	++iOrder;
164
165	return iOrder;
166	}
167
168
169	/**
170	* Converts from RTMEM_PROT_* to Linux PAGE_*.
171	*
172	* @returns Linux page protection constant.
173	* @param fProt The IPRT protection mask.
174	* @param fKernel Whether it applies to kernel or user space.
175	*/
176	static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
177	{
178	switch (fProt)
179	{
180	default:
181	AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
182	case RTMEM_PROT_NONE:
183	return PAGE_NONE;
184
185	case RTMEM_PROT_READ:
186	return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
187
188	case RTMEM_PROT_WRITE:
189	case RTMEM_PROT_WRITE \| RTMEM_PROT_READ:
190	return fKernel ? PAGE_KERNEL : PAGE_SHARED;
191
192	case RTMEM_PROT_EXEC:
193	case RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
194	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
195	if (fKernel)
196	{
197	pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
198	pgprot_val(fPg) &= ~_PAGE_RW;
199	return fPg;
200	}
201	return PAGE_READONLY_EXEC;
202	#else
203	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
204	#endif
205
206	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
207	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
208	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
209	}
210	}
211
212
213	/**
214	* Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
215	* an empty user space mapping.
216	*
217	* We acquire the mmap_sem/mmap_lock of the task!
218	*
219	* @returns Pointer to the mapping.
220	* (void *)-1 on failure.
221	* @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
222	* @param cb The size of the mapping.
223	* @param uAlignment The alignment of the mapping.
224	* @param pTask The Linux task to create this mapping in.
225	* @param fProt The RTMEM_PROT_* mask.
226	*/
227	static void rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct pTask, unsigned fProt)
228	{
229	unsigned fLnxProt;
230	unsigned long ulAddr;
231
232	Assert(pTask == current); /* do_mmap */
233	RT_NOREF_PV(pTask);
234
235	/*
236	* Convert from IPRT protection to mman.h PROT_ and call do_mmap.
237	*/
238	fProt &= (RTMEM_PROT_NONE \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC);
239	if (fProt == RTMEM_PROT_NONE)
240	fLnxProt = PROT_NONE;
241	else
242	{
243	fLnxProt = 0;
244	if (fProt & RTMEM_PROT_READ)
245	fLnxProt \|= PROT_READ;
246	if (fProt & RTMEM_PROT_WRITE)
247	fLnxProt \|= PROT_WRITE;
248	if (fProt & RTMEM_PROT_EXEC)
249	fLnxProt \|= PROT_EXEC;
250	}
251
252	if (R3PtrFixed != (RTR3PTR)-1)
253	{
254	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
255	ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
256	#else
257	LNX_MM_DOWN_WRITE(pTask->mm);
258	ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
259	LNX_MM_UP_WRITE(pTask->mm);
260	#endif
261	}
262	else
263	{
264	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
265	ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
266	#else
267	LNX_MM_DOWN_WRITE(pTask->mm);
268	ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
269	LNX_MM_UP_WRITE(pTask->mm);
270	#endif
271	if ( !(ulAddr & ~PAGE_MASK)
272	&& (ulAddr & (uAlignment - 1)))
273	{
274	/** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
275	* up alignment gaps. This is of course complicated by fragmentation (which we might have cause
276	* ourselves) and further by there begin two mmap strategies (top / bottom). */
277	/* For now, just ignore uAlignment requirements... */
278	}
279	}
280
281
282	if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
283	return (void *)-1;
284	return (void *)ulAddr;
285	}
286
287
288	/**
289	* Worker that destroys a user space mapping.
290	* Undoes what rtR0MemObjLinuxDoMmap did.
291	*
292	* We acquire the mmap_sem/mmap_lock of the task!
293	*
294	* @param pv The ring-3 mapping.
295	* @param cb The size of the mapping.
296	* @param pTask The Linux task to destroy this mapping in.
297	*/
298	static void rtR0MemObjLinuxDoMunmap(void pv, size_t cb, struct task_struct pTask)
299	{
300	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
301	Assert(pTask == current); RT_NOREF_PV(pTask);
302	vm_munmap((unsigned long)pv, cb);
303	#elif defined(USE_RHEL4_MUNMAP)
304	LNX_MM_DOWN_WRITE(pTask->mm);
305	do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
306	LNX_MM_UP_WRITE(pTask->mm);
307	#else
308	LNX_MM_DOWN_WRITE(pTask->mm);
309	do_munmap(pTask->mm, (unsigned long)pv, cb);
310	LNX_MM_UP_WRITE(pTask->mm);
311	#endif
312	}
313
314
315	/**
316	* Internal worker that allocates physical pages and creates the memory object for them.
317	*
318	* @returns IPRT status code.
319	* @param ppMemLnx Where to store the memory object pointer.
320	* @param enmType The object type.
321	* @param cb The number of bytes to allocate.
322	* @param uAlignment The alignment of the physical memory.
323	* Only valid if fContiguous == true, ignored otherwise.
324	* @param fFlagsLnx The page allocation flags (GPFs).
325	* @param fContiguous Whether the allocation must be contiguous.
326	* @param fExecutable Whether the memory must be executable.
327	* @param rcNoMem What to return when we're out of pages.
328	*/
329	static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
330	size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, bool fExecutable, int rcNoMem)
331	{
332	size_t iPage;
333	size_t const cPages = cb >> PAGE_SHIFT;
334	struct page *paPages;
335
336	/*
337	* Allocate a memory object structure that's large enough to contain
338	* the page pointer array.
339	*/
340	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType, NULL, cb);
341	if (!pMemLnx)
342	return VERR_NO_MEMORY;
343	pMemLnx->cPages = cPages;
344
345	if (cPages > 255)
346	{
347	# ifdef __GFP_REPEAT
348	/* Try hard to allocate the memory, but the allocation attempt might fail. */
349	fFlagsLnx \|= __GFP_REPEAT;
350	# endif
351	# ifdef __GFP_NOMEMALLOC
352	/* Introduced with Linux 2.6.12: Don't use emergency reserves */
353	fFlagsLnx \|= __GFP_NOMEMALLOC;
354	# endif
355	}
356
357	/*
358	* Allocate the pages.
359	* For small allocations we'll try contiguous first and then fall back on page by page.
360	*/
361	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
362	if ( fContiguous
363	\|\| cb <= PAGE_SIZE * 2)
364	{
365	# ifdef VBOX_USE_INSERT_PAGE
366	paPages = alloc_pages(fFlagsLnx \| __GFP_COMP \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
367	# else
368	paPages = alloc_pages(fFlagsLnx \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
369	# endif
370	if (paPages)
371	{
372	fContiguous = true;
373	for (iPage = 0; iPage < cPages; iPage++)
374	pMemLnx->apPages[iPage] = &paPages[iPage];
375	}
376	else if (fContiguous)
377	{
378	rtR0MemObjDelete(&pMemLnx->Core);
379	return rcNoMem;
380	}
381	}
382
383	if (!fContiguous)
384	{
385	for (iPage = 0; iPage < cPages; iPage++)
386	{
387	pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx \| __GFP_NOWARN);
388	if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
389	{
390	while (iPage-- > 0)
391	__free_page(pMemLnx->apPages[iPage]);
392	rtR0MemObjDelete(&pMemLnx->Core);
393	return rcNoMem;
394	}
395	}
396	}
397
398	#else /* < 2.4.22 */
399	/** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
400	paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
401	if (!paPages)
402	{
403	rtR0MemObjDelete(&pMemLnx->Core);
404	return rcNoMem;
405	}
406	for (iPage = 0; iPage < cPages; iPage++)
407	{
408	pMemLnx->apPages[iPage] = &paPages[iPage];
409	if (fExecutable)
410	MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
411	if (PageHighMem(pMemLnx->apPages[iPage]))
412	BUG();
413	}
414
415	fContiguous = true;
416	#endif /* < 2.4.22 */
417	pMemLnx->fContiguous = fContiguous;
418	pMemLnx->fExecutable = fExecutable;
419
420	#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
421	/*
422	* Reserve the pages.
423	*
424	* Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
425	* pages. According to Michal Hocko this shouldn't be necessary anyway because
426	* as pages which are not on the LRU list are never evictable.
427	*/
428	for (iPage = 0; iPage < cPages; iPage++)
429	SetPageReserved(pMemLnx->apPages[iPage]);
430	#endif
431
432	/*
433	* Note that the physical address of memory allocated with alloc_pages(flags, order)
434	* is always 2^(PAGE_SHIFT+order)-aligned.
435	*/
436	if ( fContiguous
437	&& uAlignment > PAGE_SIZE)
438	{
439	/*
440	* Check for alignment constraints. The physical address of memory allocated with
441	* alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
442	*/
443	if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
444	{
445	/*
446	* This should never happen!
447	*/
448	printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
449	(unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
450	rtR0MemObjLinuxFreePages(pMemLnx);
451	return rcNoMem;
452	}
453	}
454
455	*ppMemLnx = pMemLnx;
456	return VINF_SUCCESS;
457	}
458
459
460	/**
461	* Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
462	*
463	* This method does NOT free the object.
464	*
465	* @param pMemLnx The object which physical pages should be freed.
466	*/
467	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
468	{
469	size_t iPage = pMemLnx->cPages;
470	if (iPage > 0)
471	{
472	/*
473	* Restore the page flags.
474	*/
475	while (iPage-- > 0)
476	{
477	#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
478	/* See SetPageReserved() in rtR0MemObjLinuxAllocPages() */
479	ClearPageReserved(pMemLnx->apPages[iPage]);
480	#endif
481	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 22)
482	if (pMemLnx->fExecutable)
483	MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
484	#endif
485	}
486
487	/*
488	* Free the pages.
489	*/
490	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
491	if (!pMemLnx->fContiguous)
492	{
493	iPage = pMemLnx->cPages;
494	while (iPage-- > 0)
495	__free_page(pMemLnx->apPages[iPage]);
496	}
497	else
498	#endif
499	__free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
500
501	pMemLnx->cPages = 0;
502	}
503	}
504
505
506	/**
507	* Maps the allocation into ring-0.
508	*
509	* This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
510	*
511	* Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
512	* space, so we'll use that mapping if possible. If execute access is required, we'll
513	* play safe and do our own mapping.
514	*
515	* @returns IPRT status code.
516	* @param pMemLnx The linux memory object to map.
517	* @param fExecutable Whether execute access is required.
518	*/
519	static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
520	{
521	int rc = VINF_SUCCESS;
522
523	/*
524	* Choose mapping strategy.
525	*/
526	bool fMustMap = fExecutable
527	\|\| !pMemLnx->fContiguous;
528	if (!fMustMap)
529	{
530	size_t iPage = pMemLnx->cPages;
531	while (iPage-- > 0)
532	if (PageHighMem(pMemLnx->apPages[iPage]))
533	{
534	fMustMap = true;
535	break;
536	}
537	}
538
539	Assert(!pMemLnx->Core.pv);
540	Assert(!pMemLnx->fMappedToRing0);
541
542	if (fMustMap)
543	{
544	/*
545	* Use vmap - 2.4.22 and later.
546	*/
547	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
548	pgprot_t fPg;
549	pgprot_val(fPg) = _PAGE_PRESENT \| _PAGE_RW;
550	# ifdef _PAGE_NX
551	if (!fExecutable)
552	pgprot_val(fPg) \|= _PAGE_NX;
553	# endif
554
555	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
556	if (fExecutable)
557	{
558	pte_t papPtes = (pte_t )kmalloc_array(pMemLnx->cPages, sizeof(papPtes[0]), GFP_KERNEL);
559	if (papPtes)
560	{
561	pMemLnx->pArea = alloc_vm_area(pMemLnx->Core.cb, papPtes); /* Note! pArea->nr_pages is not set. */
562	if (pMemLnx->pArea)
563	{
564	size_t i;
565	Assert(pMemLnx->pArea->size >= pMemLnx->Core.cb); /* Note! includes guard page. */
566	Assert(pMemLnx->pArea->addr);
567	# ifdef _PAGE_NX
568	pgprot_val(fPg) \|= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
569	# endif
570	pMemLnx->papPtesForArea = papPtes;
571	for (i = 0; i < pMemLnx->cPages; i++)
572	*papPtes[i] = mk_pte(pMemLnx->apPages[i], fPg);
573	pMemLnx->Core.pv = pMemLnx->pArea->addr;
574	pMemLnx->fMappedToRing0 = true;
575	}
576	else
577	{
578	kfree(papPtes);
579	rc = VERR_MAP_FAILED;
580	}
581	}
582	else
583	rc = VERR_MAP_FAILED;
584	}
585	else
586	# endif
587	{
588	# ifdef VM_MAP
589	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
590	# else
591	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
592	# endif
593	if (pMemLnx->Core.pv)
594	pMemLnx->fMappedToRing0 = true;
595	else
596	rc = VERR_MAP_FAILED;
597	}
598	#else /* < 2.4.22 */
599	rc = VERR_NOT_SUPPORTED;
600	#endif
601	}
602	else
603	{
604	/*
605	* Use the kernel RAM mapping.
606	*/
607	pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
608	Assert(pMemLnx->Core.pv);
609	}
610
611	return rc;
612	}
613
614
615	/**
616	* Undoes what rtR0MemObjLinuxVMap() did.
617	*
618	* @param pMemLnx The linux memory object.
619	*/
620	static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
621	{
622	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
623	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
624	if (pMemLnx->pArea)
625	{
626	# if 0
627	pte_t **papPtes = pMemLnx->papPtesForArea;
628	size_t i;
629	for (i = 0; i < pMemLnx->cPages; i++)
630	*papPtes[i] = 0;
631	# endif
632	free_vm_area(pMemLnx->pArea);
633	kfree(pMemLnx->papPtesForArea);
634	pMemLnx->pArea = NULL;
635	pMemLnx->papPtesForArea = NULL;
636	}
637	else
638	# endif
639	if (pMemLnx->fMappedToRing0)
640	{
641	Assert(pMemLnx->Core.pv);
642	vunmap(pMemLnx->Core.pv);
643	pMemLnx->fMappedToRing0 = false;
644	}
645	#else /* < 2.4.22 */
646	Assert(!pMemLnx->fMappedToRing0);
647	#endif
648	pMemLnx->Core.pv = NULL;
649	}
650
651
652	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
653	{
654	IPRT_LINUX_SAVE_EFL_AC();
655	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
656
657	/*
658	* Release any memory that we've allocated or locked.
659	*/
660	switch (pMemLnx->Core.enmType)
661	{
662	case RTR0MEMOBJTYPE_LOW:
663	case RTR0MEMOBJTYPE_PAGE:
664	case RTR0MEMOBJTYPE_CONT:
665	case RTR0MEMOBJTYPE_PHYS:
666	case RTR0MEMOBJTYPE_PHYS_NC:
667	rtR0MemObjLinuxVUnmap(pMemLnx);
668	rtR0MemObjLinuxFreePages(pMemLnx);
669	break;
670
671	case RTR0MEMOBJTYPE_LOCK:
672	if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
673	{
674	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
675	size_t iPage;
676	Assert(pTask);
677	if (pTask && pTask->mm)
678	LNX_MM_DOWN_READ(pTask->mm);
679
680	iPage = pMemLnx->cPages;
681	while (iPage-- > 0)
682	{
683	if (!PageReserved(pMemLnx->apPages[iPage]))
684	SetPageDirty(pMemLnx->apPages[iPage]);
685	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
686	put_page(pMemLnx->apPages[iPage]);
687	#else
688	page_cache_release(pMemLnx->apPages[iPage]);
689	#endif
690	}
691
692	if (pTask && pTask->mm)
693	LNX_MM_UP_READ(pTask->mm);
694	}
695	/* else: kernel memory - nothing to do here. */
696	break;
697
698	case RTR0MEMOBJTYPE_RES_VIRT:
699	Assert(pMemLnx->Core.pv);
700	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
701	{
702	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
703	Assert(pTask);
704	if (pTask && pTask->mm)
705	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
706	}
707	else
708	{
709	vunmap(pMemLnx->Core.pv);
710
711	Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
712	__free_page(pMemLnx->apPages[0]);
713	pMemLnx->apPages[0] = NULL;
714	pMemLnx->cPages = 0;
715	}
716	pMemLnx->Core.pv = NULL;
717	break;
718
719	case RTR0MEMOBJTYPE_MAPPING:
720	Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
721	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
722	{
723	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
724	Assert(pTask);
725	if (pTask && pTask->mm)
726	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
727	}
728	else
729	vunmap(pMemLnx->Core.pv);
730	pMemLnx->Core.pv = NULL;
731	break;
732
733	default:
734	AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
735	return VERR_INTERNAL_ERROR;
736	}
737	IPRT_LINUX_RESTORE_EFL_ONLY_AC();
738	return VINF_SUCCESS;
739	}
740
741
742	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
743	{
744	IPRT_LINUX_SAVE_EFL_AC();
745	PRTR0MEMOBJLNX pMemLnx;
746	int rc;
747
748	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
749	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
750	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
751	#else
752	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
753	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
754	#endif
755	if (RT_SUCCESS(rc))
756	{
757	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
758	if (RT_SUCCESS(rc))
759	{
760	*ppMem = &pMemLnx->Core;
761	IPRT_LINUX_RESTORE_EFL_AC();
762	return rc;
763	}
764
765	rtR0MemObjLinuxFreePages(pMemLnx);
766	rtR0MemObjDelete(&pMemLnx->Core);
767	}
768
769	IPRT_LINUX_RESTORE_EFL_AC();
770	return rc;
771	}
772
773
774	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
775	{
776	IPRT_LINUX_SAVE_EFL_AC();
777	PRTR0MEMOBJLNX pMemLnx;
778	int rc;
779
780	/* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
781	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
782	/* ZONE_DMA32: 0-4GB */
783	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
784	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
785	if (RT_FAILURE(rc))
786	#endif
787	#ifdef RT_ARCH_AMD64
788	/* ZONE_DMA: 0-16MB */
789	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
790	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
791	#else
792	# ifdef CONFIG_X86_PAE
793	# endif
794	/* ZONE_NORMAL: 0-896MB */
795	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
796	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
797	#endif
798	if (RT_SUCCESS(rc))
799	{
800	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
801	if (RT_SUCCESS(rc))
802	{
803	*ppMem = &pMemLnx->Core;
804	IPRT_LINUX_RESTORE_EFL_AC();
805	return rc;
806	}
807
808	rtR0MemObjLinuxFreePages(pMemLnx);
809	rtR0MemObjDelete(&pMemLnx->Core);
810	}
811
812	IPRT_LINUX_RESTORE_EFL_AC();
813	return rc;
814	}
815
816
817	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
818	{
819	IPRT_LINUX_SAVE_EFL_AC();
820	PRTR0MEMOBJLNX pMemLnx;
821	int rc;
822
823	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
824	/* ZONE_DMA32: 0-4GB */
825	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
826	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
827	if (RT_FAILURE(rc))
828	#endif
829	#ifdef RT_ARCH_AMD64
830	/* ZONE_DMA: 0-16MB */
831	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
832	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
833	#else
834	/* ZONE_NORMAL (32-bit hosts): 0-896MB */
835	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
836	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
837	#endif
838	if (RT_SUCCESS(rc))
839	{
840	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
841	if (RT_SUCCESS(rc))
842	{
843	#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_HIGHMEM64G))
844	size_t iPage = pMemLnx->cPages;
845	while (iPage-- > 0)
846	Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
847	#endif
848	pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
849	*ppMem = &pMemLnx->Core;
850	IPRT_LINUX_RESTORE_EFL_AC();
851	return rc;
852	}
853
854	rtR0MemObjLinuxFreePages(pMemLnx);
855	rtR0MemObjDelete(&pMemLnx->Core);
856	}
857
858	IPRT_LINUX_RESTORE_EFL_AC();
859	return rc;
860	}
861
862
863	/**
864	* Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
865	*
866	* @returns IPRT status code.
867	* @param ppMemLnx Where to
868	* @param enmType The object type.
869	* @param cb The size of the allocation.
870	* @param uAlignment The alignment of the physical memory.
871	* Only valid for fContiguous == true, ignored otherwise.
872	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
873	* @param fGfp The Linux GFP flags to use for the allocation.
874	*/
875	static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
876	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, gfp_t fGfp)
877	{
878	PRTR0MEMOBJLNX pMemLnx;
879	int rc;
880
881	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
882	enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
883	false /fExecutable/, VERR_NO_PHYS_MEMORY);
884	if (RT_FAILURE(rc))
885	return rc;
886
887	/*
888	* Check the addresses if necessary. (Can be optimized a bit for PHYS.)
889	*/
890	if (PhysHighest != NIL_RTHCPHYS)
891	{
892	size_t iPage = pMemLnx->cPages;
893	while (iPage-- > 0)
894	if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
895	{
896	rtR0MemObjLinuxFreePages(pMemLnx);
897	rtR0MemObjDelete(&pMemLnx->Core);
898	return VERR_NO_MEMORY;
899	}
900	}
901
902	/*
903	* Complete the object.
904	*/
905	if (enmType == RTR0MEMOBJTYPE_PHYS)
906	{
907	pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
908	pMemLnx->Core.u.Phys.fAllocated = true;
909	}
910	*ppMem = &pMemLnx->Core;
911	return rc;
912	}
913
914
915	/**
916	* Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
917	*
918	* @returns IPRT status code.
919	* @param ppMem Where to store the memory object pointer on success.
920	* @param enmType The object type.
921	* @param cb The size of the allocation.
922	* @param uAlignment The alignment of the physical memory.
923	* Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
924	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
925	*/
926	static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
927	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest)
928	{
929	int rc;
930	IPRT_LINUX_SAVE_EFL_AC();
931
932	/*
933	* There are two clear cases and that's the <=16MB and anything-goes ones.
934	* When the physical address limit is somewhere in-between those two we'll
935	* just have to try, starting with HIGHUSER and working our way thru the
936	* different types, hoping we'll get lucky.
937	*
938	* We should probably move this physical address restriction logic up to
939	* the page alloc function as it would be more efficient there. But since
940	* we don't expect this to be a performance issue just yet it can wait.
941	*/
942	if (PhysHighest == NIL_RTHCPHYS)
943	/* ZONE_HIGHMEM: the whole physical memory */
944	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
945	else if (PhysHighest <= _1M * 16)
946	/* ZONE_DMA: 0-16MB */
947	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
948	else
949	{
950	rc = VERR_NO_MEMORY;
951	if (RT_FAILURE(rc))
952	/* ZONE_HIGHMEM: the whole physical memory */
953	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
954	if (RT_FAILURE(rc))
955	/* ZONE_NORMAL: 0-896MB */
956	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_USER);
957	#ifdef GFP_DMA32
958	if (RT_FAILURE(rc))
959	/* ZONE_DMA32: 0-4GB */
960	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA32);
961	#endif
962	if (RT_FAILURE(rc))
963	/* ZONE_DMA: 0-16MB */
964	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
965	}
966	IPRT_LINUX_RESTORE_EFL_AC();
967	return rc;
968	}
969
970
971	/**
972	* Translates a kernel virtual address to a linux page structure by walking the
973	* page tables.
974	*
975	* @note We do assume that the page tables will not change as we are walking
976	* them. This assumption is rather forced by the fact that I could not
977	* immediately see any way of preventing this from happening. So, we
978	* take some extra care when accessing them.
979	*
980	* Because of this, we don't want to use this function on memory where
981	* attribute changes to nearby pages is likely to cause large pages to
982	* be used or split up. So, don't use this for the linear mapping of
983	* physical memory.
984	*
985	* @returns Pointer to the page structur or NULL if it could not be found.
986	* @param pv The kernel virtual address.
987	*/
988	RTDECL(struct page ) rtR0MemObjLinuxVirtToPage(void pv)
989	{
990	unsigned long ulAddr = (unsigned long)pv;
991	unsigned long pfn;
992	struct page *pPage;
993	pte_t *pEntry;
994	union
995	{
996	pgd_t Global;
997	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
998	p4d_t Four;
999	#endif
1000	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1001	pud_t Upper;
1002	#endif
1003	pmd_t Middle;
1004	pte_t Entry;
1005	} u;
1006
1007	/* Should this happen in a situation this code will be called in? And if
1008	* so, can it change under our feet? See also
1009	* "Documentation/vm/active_mm.txt" in the kernel sources. */
1010	if (RT_UNLIKELY(!current->active_mm))
1011	return NULL;
1012	u.Global = *pgd_offset(current->active_mm, ulAddr);
1013	if (RT_UNLIKELY(pgd_none(u.Global)))
1014	return NULL;
1015	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1016	# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
1017	u.Four = *p4d_offset(&u.Global, ulAddr);
1018	if (RT_UNLIKELY(p4d_none(u.Four)))
1019	return NULL;
1020	if (p4d_large(u.Four))
1021	{
1022	pPage = p4d_page(u.Four);
1023	AssertReturn(pPage, NULL);
1024	pfn = page_to_pfn(pPage); /* doing the safe way... */
1025	AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
1026	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
1027	return pfn_to_page(pfn);
1028	}
1029	u.Upper = *pud_offset(&u.Four, ulAddr);
1030	# else /* < 4.12 */
1031	u.Upper = *pud_offset(&u.Global, ulAddr);
1032	# endif /* < 4.12 */
1033	if (RT_UNLIKELY(pud_none(u.Upper)))
1034	return NULL;
1035	# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
1036	if (pud_large(u.Upper))
1037	{
1038	pPage = pud_page(u.Upper);
1039	AssertReturn(pPage, NULL);
1040	pfn = page_to_pfn(pPage); /* doing the safe way... */
1041	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
1042	return pfn_to_page(pfn);
1043	}
1044	# endif
1045	u.Middle = *pmd_offset(&u.Upper, ulAddr);
1046	#else /* < 2.6.11 */
1047	u.Middle = *pmd_offset(&u.Global, ulAddr);
1048	#endif /* < 2.6.11 */
1049	if (RT_UNLIKELY(pmd_none(u.Middle)))
1050	return NULL;
1051	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0)
1052	if (pmd_large(u.Middle))
1053	{
1054	pPage = pmd_page(u.Middle);
1055	AssertReturn(pPage, NULL);
1056	pfn = page_to_pfn(pPage); /* doing the safe way... */
1057	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
1058	return pfn_to_page(pfn);
1059	}
1060	#endif
1061
1062	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) \|\| defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
1063	pEntry = pte_offset_map(&u.Middle, ulAddr);
1064	#else
1065	pEntry = pte_offset(&u.Middle, ulAddr);
1066	#endif
1067	if (RT_UNLIKELY(!pEntry))
1068	return NULL;
1069	u.Entry = *pEntry;
1070	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) \|\| defined(pte_offset_map)
1071	pte_unmap(pEntry);
1072	#endif
1073
1074	if (RT_UNLIKELY(!pte_present(u.Entry)))
1075	return NULL;
1076	return pte_page(u.Entry);
1077	}
1078	RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
1079
1080
1081	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
1082	{
1083	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest);
1084	}
1085
1086
1087	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
1088	{
1089	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest);
1090	}
1091
1092
1093	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
1094	{
1095	IPRT_LINUX_SAVE_EFL_AC();
1096
1097	/*
1098	* All we need to do here is to validate that we can use
1099	* ioremap on the specified address (32/64-bit dma_addr_t).
1100	*/
1101	PRTR0MEMOBJLNX pMemLnx;
1102	dma_addr_t PhysAddr = Phys;
1103	AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
1104
1105	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb);
1106	if (!pMemLnx)
1107	{
1108	IPRT_LINUX_RESTORE_EFL_AC();
1109	return VERR_NO_MEMORY;
1110	}
1111
1112	pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
1113	pMemLnx->Core.u.Phys.fAllocated = false;
1114	pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
1115	Assert(!pMemLnx->cPages);
1116	*ppMem = &pMemLnx->Core;
1117	IPRT_LINUX_RESTORE_EFL_AC();
1118	return VINF_SUCCESS;
1119	}
1120
1121	/* openSUSE Leap 42.3 detection :-/ */
1122	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) \
1123	&& LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) \
1124	&& defined(FAULT_FLAG_REMOTE)
1125	# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
1126	#else
1127	# define GET_USER_PAGES_API LINUX_VERSION_CODE
1128	#endif
1129
1130	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
1131	{
1132	IPRT_LINUX_SAVE_EFL_AC();
1133	const int cPages = cb >> PAGE_SHIFT;
1134	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1135	struct vm_area_struct **papVMAs;
1136	PRTR0MEMOBJLNX pMemLnx;
1137	int rc = VERR_NO_MEMORY;
1138	int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
1139
1140	/*
1141	* Check for valid task and size overflows.
1142	*/
1143	if (!pTask)
1144	return VERR_NOT_SUPPORTED;
1145	if (((size_t)cPages << PAGE_SHIFT) != cb)
1146	return VERR_OUT_OF_RANGE;
1147
1148	/*
1149	* Allocate the memory object and a temporary buffer for the VMAs.
1150	*/
1151	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, (void *)R3Ptr, cb);
1152	if (!pMemLnx)
1153	{
1154	IPRT_LINUX_RESTORE_EFL_AC();
1155	return VERR_NO_MEMORY;
1156	}
1157
1158	papVMAs = (struct vm_area_struct *)RTMemAlloc(sizeof(papVMAs) * cPages);
1159	if (papVMAs)
1160	{
1161	LNX_MM_DOWN_READ(pTask->mm);
1162
1163	/*
1164	* Get user pages.
1165	*/
1166	/** @todo r=bird: Should we not force read access too? */
1167	#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
1168	if (R0Process == RTR0ProcHandleSelf())
1169	rc = get_user_pages(R3Ptr, /* Where from. */
1170	cPages, /* How many pages. */
1171	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1172	fWrite ? FOLL_WRITE \| /* Write to memory. */
1173	FOLL_FORCE /* force write access. */
1174	: 0, /* Write to memory. */
1175	# else
1176	fWrite, /* Write to memory. */
1177	fWrite, /* force write access. */
1178	# endif
1179	&pMemLnx->apPages[0], /* Page array. */
1180	papVMAs); /* vmas */
1181	/*
1182	* Actually this should not happen at the moment as call this function
1183	* only for our own process.
1184	*/
1185	else
1186	rc = get_user_pages_remote(
1187	pTask, /* Task for fault accounting. */
1188	pTask->mm, /* Whose pages. */
1189	R3Ptr, /* Where from. */
1190	cPages, /* How many pages. */
1191	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1192	fWrite ? FOLL_WRITE \| /* Write to memory. */
1193	FOLL_FORCE /* force write access. */
1194	: 0, /* Write to memory. */
1195	# else
1196	fWrite, /* Write to memory. */
1197	fWrite, /* force write access. */
1198	# endif
1199	&pMemLnx->apPages[0], /* Page array. */
1200	papVMAs /* vmas */
1201	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
1202	, NULL /* locked */
1203	# endif
1204	);
1205	#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1206	rc = get_user_pages(pTask, /* Task for fault accounting. */
1207	pTask->mm, /* Whose pages. */
1208	R3Ptr, /* Where from. */
1209	cPages, /* How many pages. */
1210	/* The get_user_pages API change was back-ported to 4.4.168. */
1211	# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 168) \
1212	&& LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
1213	fWrite ? FOLL_WRITE \| /* Write to memory. */
1214	FOLL_FORCE /* force write access. */
1215	: 0, /* Write to memory. */
1216	# else
1217	fWrite, /* Write to memory. */
1218	fWrite, /* force write access. */
1219	# endif
1220	&pMemLnx->apPages[0], /* Page array. */
1221	papVMAs); /* vmas */
1222	#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1223	if (rc == cPages)
1224	{
1225	/*
1226	* Flush dcache (required?), protect against fork and _really_ pin the page
1227	* table entries. get_user_pages() will protect against swapping out the
1228	* pages but it will NOT protect against removing page table entries. This
1229	* can be achieved with
1230	* - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1231	* an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1232	* Usual Linux distributions support only a limited size of locked pages
1233	* (e.g. 32KB).
1234	* - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1235	* or by
1236	* - setting the VM_LOCKED flag. This is the same as doing mlock() without
1237	* a range check.
1238	*/
1239	/** @todo The Linux fork() protection will require more work if this API
1240	* is to be used for anything but locking VM pages. */
1241	while (rc-- > 0)
1242	{
1243	flush_dcache_page(pMemLnx->apPages[rc]);
1244	papVMAs[rc]->vm_flags \|= VM_DONTCOPY \| VM_LOCKED;
1245	}
1246
1247	LNX_MM_UP_READ(pTask->mm);
1248
1249	RTMemFree(papVMAs);
1250
1251	pMemLnx->Core.u.Lock.R0Process = R0Process;
1252	pMemLnx->cPages = cPages;
1253	Assert(!pMemLnx->fMappedToRing0);
1254	*ppMem = &pMemLnx->Core;
1255
1256	IPRT_LINUX_RESTORE_EFL_AC();
1257	return VINF_SUCCESS;
1258	}
1259
1260	/*
1261	* Failed - we need to unlock any pages that we succeeded to lock.
1262	*/
1263	while (rc-- > 0)
1264	{
1265	if (!PageReserved(pMemLnx->apPages[rc]))
1266	SetPageDirty(pMemLnx->apPages[rc]);
1267	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
1268	put_page(pMemLnx->apPages[rc]);
1269	#else
1270	page_cache_release(pMemLnx->apPages[rc]);
1271	#endif
1272	}
1273
1274	LNX_MM_UP_READ(pTask->mm);
1275
1276	RTMemFree(papVMAs);
1277	rc = VERR_LOCK_FAILED;
1278	}
1279
1280	rtR0MemObjDelete(&pMemLnx->Core);
1281	IPRT_LINUX_RESTORE_EFL_AC();
1282	return rc;
1283	}
1284
1285
1286	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1287	{
1288	IPRT_LINUX_SAVE_EFL_AC();
1289	void pvLast = (uint8_t )pv + cb - 1;
1290	size_t const cPages = cb >> PAGE_SHIFT;
1291	PRTR0MEMOBJLNX pMemLnx;
1292	bool fLinearMapping;
1293	int rc;
1294	uint8_t *pbPage;
1295	size_t iPage;
1296	NOREF(fAccess);
1297
1298	if ( !RTR0MemKernelIsValidAddr(pv)
1299	\|\| !RTR0MemKernelIsValidAddr(pv + cb))
1300	return VERR_INVALID_PARAMETER;
1301
1302	/*
1303	* The lower part of the kernel memory has a linear mapping between
1304	* physical and virtual addresses. So we take a short cut here. This is
1305	* assumed to be the cleanest way to handle those addresses (and the code
1306	* is well tested, though the test for determining it is not very nice).
1307	* If we ever decide it isn't we can still remove it.
1308	*/
1309	#if 0
1310	fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1311	#else
1312	fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1313	&& (unsigned long)pvLast < (unsigned long)high_memory;
1314	#endif
1315
1316	/*
1317	* Allocate the memory object.
1318	*/
1319	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, pv, cb);
1320	if (!pMemLnx)
1321	{
1322	IPRT_LINUX_RESTORE_EFL_AC();
1323	return VERR_NO_MEMORY;
1324	}
1325
1326	/*
1327	* Gather the pages.
1328	* We ASSUME all kernel pages are non-swappable and non-movable.
1329	*/
1330	rc = VINF_SUCCESS;
1331	pbPage = (uint8_t *)pvLast;
1332	iPage = cPages;
1333	if (!fLinearMapping)
1334	{
1335	while (iPage-- > 0)
1336	{
1337	struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1338	if (RT_UNLIKELY(!pPage))
1339	{
1340	rc = VERR_LOCK_FAILED;
1341	break;
1342	}
1343	pMemLnx->apPages[iPage] = pPage;
1344	pbPage -= PAGE_SIZE;
1345	}
1346	}
1347	else
1348	{
1349	while (iPage-- > 0)
1350	{
1351	pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1352	pbPage -= PAGE_SIZE;
1353	}
1354	}
1355	if (RT_SUCCESS(rc))
1356	{
1357	/*
1358	* Complete the memory object and return.
1359	*/
1360	pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1361	pMemLnx->cPages = cPages;
1362	Assert(!pMemLnx->fMappedToRing0);
1363	*ppMem = &pMemLnx->Core;
1364
1365	IPRT_LINUX_RESTORE_EFL_AC();
1366	return VINF_SUCCESS;
1367	}
1368
1369	rtR0MemObjDelete(&pMemLnx->Core);
1370	IPRT_LINUX_RESTORE_EFL_AC();
1371	return rc;
1372	}
1373
1374
1375	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1376	{
1377	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1378	IPRT_LINUX_SAVE_EFL_AC();
1379	const size_t cPages = cb >> PAGE_SHIFT;
1380	struct page *pDummyPage;
1381	struct page **papPages;
1382
1383	/* check for unsupported stuff. */
1384	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1385	if (uAlignment > PAGE_SIZE)
1386	return VERR_NOT_SUPPORTED;
1387
1388	/*
1389	* Allocate a dummy page and create a page pointer array for vmap such that
1390	* the dummy page is mapped all over the reserved area.
1391	*/
1392	pDummyPage = alloc_page(GFP_HIGHUSER \| __GFP_NOWARN);
1393	if (pDummyPage)
1394	{
1395	papPages = RTMemAlloc(sizeof(papPages) cPages);
1396	if (papPages)
1397	{
1398	void *pv;
1399	size_t iPage = cPages;
1400	while (iPage-- > 0)
1401	papPages[iPage] = pDummyPage;
1402	# ifdef VM_MAP
1403	pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1404	# else
1405	pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1406	# endif
1407	RTMemFree(papPages);
1408	if (pv)
1409	{
1410	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1411	if (pMemLnx)
1412	{
1413	pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1414	pMemLnx->cPages = 1;
1415	pMemLnx->apPages[0] = pDummyPage;
1416	*ppMem = &pMemLnx->Core;
1417	IPRT_LINUX_RESTORE_EFL_AC();
1418	return VINF_SUCCESS;
1419	}
1420	vunmap(pv);
1421	}
1422	}
1423	__free_page(pDummyPage);
1424	}
1425	IPRT_LINUX_RESTORE_EFL_AC();
1426	return VERR_NO_MEMORY;
1427
1428	#else /* < 2.4.22 */
1429	/*
1430	* Could probably use ioremap here, but the caller is in a better position than us
1431	* to select some safe physical memory.
1432	*/
1433	return VERR_NOT_SUPPORTED;
1434	#endif
1435	}
1436
1437
1438	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1439	{
1440	IPRT_LINUX_SAVE_EFL_AC();
1441	PRTR0MEMOBJLNX pMemLnx;
1442	void *pv;
1443	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1444	if (!pTask)
1445	return VERR_NOT_SUPPORTED;
1446
1447	/*
1448	* Check that the specified alignment is supported.
1449	*/
1450	if (uAlignment > PAGE_SIZE)
1451	return VERR_NOT_SUPPORTED;
1452
1453	/*
1454	* Let rtR0MemObjLinuxDoMmap do the difficult bits.
1455	*/
1456	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1457	if (pv == (void *)-1)
1458	{
1459	IPRT_LINUX_RESTORE_EFL_AC();
1460	return VERR_NO_MEMORY;
1461	}
1462
1463	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1464	if (!pMemLnx)
1465	{
1466	rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1467	IPRT_LINUX_RESTORE_EFL_AC();
1468	return VERR_NO_MEMORY;
1469	}
1470
1471	pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1472	*ppMem = &pMemLnx->Core;
1473	IPRT_LINUX_RESTORE_EFL_AC();
1474	return VINF_SUCCESS;
1475	}
1476
1477
1478	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap,
1479	void *pvFixed, size_t uAlignment,
1480	unsigned fProt, size_t offSub, size_t cbSub)
1481	{
1482	int rc = VERR_NO_MEMORY;
1483	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1484	PRTR0MEMOBJLNX pMemLnx;
1485	IPRT_LINUX_SAVE_EFL_AC();
1486
1487	/* Fail if requested to do something we can't. */
1488	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1489	if (uAlignment > PAGE_SIZE)
1490	return VERR_NOT_SUPPORTED;
1491
1492	/*
1493	* Create the IPRT memory object.
1494	*/
1495	if (!cbSub)
1496	cbSub = pMemLnxToMap->Core.cb - offSub;
1497	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1498	if (pMemLnx)
1499	{
1500	if (pMemLnxToMap->cPages)
1501	{
1502	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1503	/*
1504	* Use vmap - 2.4.22 and later.
1505	*/
1506	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1507	/** @todo We don't really care too much for EXEC here... 5.8 always adds NX. */
1508	Assert(((offSub + cbSub) >> PAGE_SHIFT) <= pMemLnxToMap->cPages);
1509	# ifdef VM_MAP
1510	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_MAP, fPg);
1511	# else
1512	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_ALLOC, fPg);
1513	# endif
1514	if (pMemLnx->Core.pv)
1515	{
1516	pMemLnx->fMappedToRing0 = true;
1517	rc = VINF_SUCCESS;
1518	}
1519	else
1520	rc = VERR_MAP_FAILED;
1521
1522	#else /* < 2.4.22 */
1523	/*
1524	* Only option here is to share mappings if possible and forget about fProt.
1525	*/
1526	if (rtR0MemObjIsRing3(pMemToMap))
1527	rc = VERR_NOT_SUPPORTED;
1528	else
1529	{
1530	rc = VINF_SUCCESS;
1531	if (!pMemLnxToMap->Core.pv)
1532	rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1533	if (RT_SUCCESS(rc))
1534	{
1535	Assert(pMemLnxToMap->Core.pv);
1536	pMemLnx->Core.pv = (uint8_t *)pMemLnxToMap->Core.pv + offSub;
1537	}
1538	}
1539	#endif
1540	}
1541	else
1542	{
1543	/*
1544	* MMIO / physical memory.
1545	*/
1546	Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1547	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
1548	/*
1549	* ioremap() defaults to no caching since the 2.6 kernels.
1550	* ioremap_nocache() has been removed finally in 5.6-rc1.
1551	*/
1552	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1553	? ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1554	: ioremap_cache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1555	#else /* KERNEL_VERSION < 2.6.25 */
1556	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1557	? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1558	: ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1559	#endif /* KERNEL_VERSION < 2.6.25 */
1560	if (pMemLnx->Core.pv)
1561	{
1562	/** @todo fix protection. */
1563	rc = VINF_SUCCESS;
1564	}
1565	}
1566	if (RT_SUCCESS(rc))
1567	{
1568	pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1569	*ppMem = &pMemLnx->Core;
1570	IPRT_LINUX_RESTORE_EFL_AC();
1571	return VINF_SUCCESS;
1572	}
1573	rtR0MemObjDelete(&pMemLnx->Core);
1574	}
1575
1576	IPRT_LINUX_RESTORE_EFL_AC();
1577	return rc;
1578	}
1579
1580
1581	#ifdef VBOX_USE_PAE_HACK
1582	/**
1583	* Replace the PFN of a PTE with the address of the actual page.
1584	*
1585	* The caller maps a reserved dummy page at the address with the desired access
1586	* and flags.
1587	*
1588	* This hack is required for older Linux kernels which don't provide
1589	* remap_pfn_range().
1590	*
1591	* @returns 0 on success, -ENOMEM on failure.
1592	* @param mm The memory context.
1593	* @param ulAddr The mapping address.
1594	* @param Phys The physical address of the page to map.
1595	*/
1596	static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1597	{
1598	int rc = -ENOMEM;
1599	pgd_t *pgd;
1600
1601	spin_lock(&mm->page_table_lock);
1602
1603	pgd = pgd_offset(mm, ulAddr);
1604	if (!pgd_none(pgd) && !pgd_bad(pgd))
1605	{
1606	pmd_t *pmd = pmd_offset(pgd, ulAddr);
1607	if (!pmd_none(*pmd))
1608	{
1609	pte_t *ptep = pte_offset_map(pmd, ulAddr);
1610	if (ptep)
1611	{
1612	pte_t pte = *ptep;
1613	pte.pte_high &= 0xfff00000;
1614	pte.pte_high \|= ((Phys >> 32) & 0x000fffff);
1615	pte.pte_low &= 0x00000fff;
1616	pte.pte_low \|= (Phys & 0xfffff000);
1617	set_pte(ptep, pte);
1618	pte_unmap(ptep);
1619	rc = 0;
1620	}
1621	}
1622	}
1623
1624	spin_unlock(&mm->page_table_lock);
1625	return rc;
1626	}
1627	#endif /* VBOX_USE_PAE_HACK */
1628
1629
1630	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1631	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub)
1632	{
1633	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1634	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1635	int rc = VERR_NO_MEMORY;
1636	PRTR0MEMOBJLNX pMemLnx;
1637	#ifdef VBOX_USE_PAE_HACK
1638	struct page *pDummyPage;
1639	RTHCPHYS DummyPhys;
1640	#endif
1641	IPRT_LINUX_SAVE_EFL_AC();
1642
1643	/*
1644	* Check for restrictions.
1645	*/
1646	if (!pTask)
1647	return VERR_NOT_SUPPORTED;
1648	if (uAlignment > PAGE_SIZE)
1649	return VERR_NOT_SUPPORTED;
1650
1651	#ifdef VBOX_USE_PAE_HACK
1652	/*
1653	* Allocate a dummy page for use when mapping the memory.
1654	*/
1655	pDummyPage = alloc_page(GFP_USER \| __GFP_NOWARN);
1656	if (!pDummyPage)
1657	{
1658	IPRT_LINUX_RESTORE_EFL_AC();
1659	return VERR_NO_MEMORY;
1660	}
1661	SetPageReserved(pDummyPage);
1662	DummyPhys = page_to_phys(pDummyPage);
1663	#endif
1664
1665	/*
1666	* Create the IPRT memory object.
1667	*/
1668	Assert(!offSub \|\| cbSub);
1669	if (cbSub == 0)
1670	cbSub = pMemLnxToMap->Core.cb;
1671	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1672	if (pMemLnx)
1673	{
1674	/*
1675	* Allocate user space mapping.
1676	*/
1677	void *pv;
1678	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cbSub, uAlignment, pTask, fProt);
1679	if (pv != (void *)-1)
1680	{
1681	/*
1682	* Map page by page into the mmap area.
1683	* This is generic, paranoid and not very efficient.
1684	*/
1685	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1686	unsigned long ulAddrCur = (unsigned long)pv;
1687	const size_t cPages = (offSub + cbSub) >> PAGE_SHIFT;
1688	size_t iPage;
1689
1690	LNX_MM_DOWN_WRITE(pTask->mm);
1691
1692	rc = VINF_SUCCESS;
1693	if (pMemLnxToMap->cPages)
1694	{
1695	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1696	{
1697	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)
1698	RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1699	#endif
1700	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1701	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1702	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1703	#endif
1704	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1705	/* remap_page_range() limitation on x86 */
1706	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1707	#endif
1708
1709	#if defined(VBOX_USE_INSERT_PAGE) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
1710	rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1711	/* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1712	* See remap_pfn_range() in mm/memory.c */
1713	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
1714	vma->vm_flags \|= VM_DONTEXPAND \| VM_DONTDUMP;
1715	#else
1716	vma->vm_flags \|= VM_RESERVED;
1717	#endif
1718	#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1719	rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1720	#elif defined(VBOX_USE_PAE_HACK)
1721	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1722	if (!rc)
1723	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1724	#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1725	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1726	#else /* 2.4 */
1727	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1728	#endif
1729	if (rc)
1730	{
1731	rc = VERR_NO_MEMORY;
1732	break;
1733	}
1734	}
1735	}
1736	else
1737	{
1738	RTHCPHYS Phys;
1739	if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
1740	Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
1741	else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
1742	Phys = pMemLnxToMap->Core.u.Cont.Phys;
1743	else
1744	{
1745	AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
1746	Phys = NIL_RTHCPHYS;
1747	}
1748	if (Phys != NIL_RTHCPHYS)
1749	{
1750	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
1751	{
1752	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1753	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1754	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1755	#endif
1756	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1757	/* remap_page_range() limitation on x86 */
1758	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1759	#endif
1760
1761	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1762	rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1763	#elif defined(VBOX_USE_PAE_HACK)
1764	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1765	if (!rc)
1766	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1767	#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1768	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1769	#else /* 2.4 */
1770	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1771	#endif
1772	if (rc)
1773	{
1774	rc = VERR_NO_MEMORY;
1775	break;
1776	}
1777	}
1778	}
1779	}
1780
1781	#ifdef CONFIG_NUMA_BALANCING
1782	# if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0)
1783	# ifdef RHEL_RELEASE_CODE
1784	# if RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7, 0)
1785	# define VBOX_NUMA_HACK_OLD
1786	# endif
1787	# endif
1788	# endif
1789	if (RT_SUCCESS(rc))
1790	{
1791	/** @todo Ugly hack! But right now we have no other means to
1792	* disable automatic NUMA page balancing. */
1793	# ifdef RT_OS_X86
1794	# ifdef VBOX_NUMA_HACK_OLD
1795	pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
1796	# endif
1797	pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
1798	# else
1799	# ifdef VBOX_NUMA_HACK_OLD
1800	pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
1801	# endif
1802	pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
1803	# endif
1804	}
1805	#endif /* CONFIG_NUMA_BALANCING */
1806
1807	LNX_MM_UP_WRITE(pTask->mm);
1808
1809	if (RT_SUCCESS(rc))
1810	{
1811	#ifdef VBOX_USE_PAE_HACK
1812	__free_page(pDummyPage);
1813	#endif
1814	pMemLnx->Core.pv = pv;
1815	pMemLnx->Core.u.Mapping.R0Process = R0Process;
1816	*ppMem = &pMemLnx->Core;
1817	IPRT_LINUX_RESTORE_EFL_AC();
1818	return VINF_SUCCESS;
1819	}
1820
1821	/*
1822	* Bail out.
1823	*/
1824	rtR0MemObjLinuxDoMunmap(pv, cbSub, pTask);
1825	}
1826	rtR0MemObjDelete(&pMemLnx->Core);
1827	}
1828	#ifdef VBOX_USE_PAE_HACK
1829	__free_page(pDummyPage);
1830	#endif
1831
1832	IPRT_LINUX_RESTORE_EFL_AC();
1833	return rc;
1834	}
1835
1836
1837	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1838	{
1839	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
1840	/*
1841	* Currently only supported when we've got addresses PTEs from the kernel.
1842	*/
1843	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1844	if (pMemLnx->pArea && pMemLnx->papPtesForArea)
1845	{
1846	pgprot_t const fPg = rtR0MemObjLinuxConvertProt(fProt, true /fKernel/);
1847	size_t const cPages = (offSub + cbSub) >> PAGE_SHIFT;
1848	pte_t **papPtes = pMemLnx->papPtesForArea;
1849	size_t i;
1850
1851	for (i = offSub >> PAGE_SHIFT; i < cPages; i++)
1852	{
1853	set_pte(papPtes[i], mk_pte(pMemLnx->apPages[i], fPg));
1854	}
1855	preempt_disable();
1856	__flush_tlb_all();
1857	preempt_enable();
1858	return VINF_SUCCESS;
1859	}
1860	# endif
1861
1862	NOREF(pMem);
1863	NOREF(offSub);
1864	NOREF(cbSub);
1865	NOREF(fProt);
1866	return VERR_NOT_SUPPORTED;
1867	}
1868
1869
1870	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1871	{
1872	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1873
1874	if (pMemLnx->cPages)
1875	return page_to_phys(pMemLnx->apPages[iPage]);
1876
1877	switch (pMemLnx->Core.enmType)
1878	{
1879	case RTR0MEMOBJTYPE_CONT:
1880	return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
1881
1882	case RTR0MEMOBJTYPE_PHYS:
1883	return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
1884
1885	/* the parent knows */
1886	case RTR0MEMOBJTYPE_MAPPING:
1887	return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
1888
1889	/* cPages > 0 */
1890	case RTR0MEMOBJTYPE_LOW:
1891	case RTR0MEMOBJTYPE_LOCK:
1892	case RTR0MEMOBJTYPE_PHYS_NC:
1893	case RTR0MEMOBJTYPE_PAGE:
1894	default:
1895	AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
1896	/* fall thru */
1897
1898	case RTR0MEMOBJTYPE_RES_VIRT:
1899	return NIL_RTHCPHYS;
1900	}
1901	}
1902

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c@ 85516

Download in other formats: