memobj-r0drv-linux.c@ 86615

Last change on this file since 86615 was 86542, checked in by vboxsync, 5 years ago
Additions/linux, Runtime/r0drv/linux: Adjustments for Linux 5.9.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Rev Revision`
File size: 63.2 KB

Line
1	/* $Id: memobj-r0drv-linux.c 86542 2020-10-12 13:35:53Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-linux-kernel.h"
32
33	#include <iprt/memobj.h>
34	#include <iprt/assert.h>
35	#include <iprt/err.h>
36	#include <iprt/log.h>
37	#include <iprt/mem.h>
38	#include <iprt/process.h>
39	#include <iprt/string.h>
40	#include "internal/memobj.h"
41	#include "internal/iprt.h"
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	/* early 2.6 kernels */
48	#ifndef PAGE_SHARED_EXEC
49	# define PAGE_SHARED_EXEC PAGE_SHARED
50	#endif
51	#ifndef PAGE_READONLY_EXEC
52	# define PAGE_READONLY_EXEC PAGE_READONLY
53	#endif
54
55	/** @def IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
56	* Whether we use alloc_vm_area (3.2+) for executable memory.
57	* This is a must for 5.8+, but we enable it all the way back to 3.2.x for
58	* better W^R compliance (fExecutable flag). */
59	#if RTLNX_VER_MIN(3,2,0) \|\| defined(DOXYGEN_RUNNING)
60	# define IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
61	#endif
62
63	/*
64	* 2.6.29+ kernels don't work with remap_pfn_range() anymore because
65	* track_pfn_vma_new() is apparently not defined for non-RAM pages.
66	* It should be safe to use vm_insert_page() older kernels as well.
67	*/
68	#if RTLNX_VER_MIN(2,6,23)
69	# define VBOX_USE_INSERT_PAGE
70	#endif
71	#if defined(CONFIG_X86_PAE) \
72	&& ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
73	\|\| RTLNX_VER_RANGE(2,6,0, 2,6,11) )
74	# define VBOX_USE_PAE_HACK
75	#endif
76
77	/* gfp_t was introduced in 2.6.14, define it for earlier. */
78	#if RTLNX_VER_MAX(2,6,14)
79	# define gfp_t unsigned
80	#endif
81
82	/*
83	* Wrappers around mmap_lock/mmap_sem difference.
84	*/
85	#if RTLNX_VER_MIN(5,8,0)
86	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_lock)
87	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_lock)
88	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_lock)
89	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_lock)
90	#else
91	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_sem)
92	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_sem)
93	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_sem)
94	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_sem)
95	#endif
96
97
98	/*********************************************************************************************************************************
99	* Structures and Typedefs *
100	*********************************************************************************************************************************/
101	/**
102	* The Linux version of the memory object structure.
103	*/
104	typedef struct RTR0MEMOBJLNX
105	{
106	/** The core structure. */
107	RTR0MEMOBJINTERNAL Core;
108	/** Set if the allocation is contiguous.
109	* This means it has to be given back as one chunk. */
110	bool fContiguous;
111	/** Set if executable allocation. */
112	bool fExecutable;
113	/** Set if we've vmap'ed the memory into ring-0. */
114	bool fMappedToRing0;
115	#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
116	/** Return from alloc_vm_area() that we now need to use for executable
117	* memory. */
118	struct vm_struct *pArea;
119	/** PTE array that goes along with pArea (must be freed). */
120	pte_t **papPtesForArea;
121	#endif
122	/** The pages in the apPages array. */
123	size_t cPages;
124	/** Array of struct page pointers. (variable size) */
125	struct page *apPages[1];
126	} RTR0MEMOBJLNX;
127	/** Pointer to the linux memory object. */
128	typedef RTR0MEMOBJLNX *PRTR0MEMOBJLNX;
129
130
131	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
132
133
134	/**
135	* Helper that converts from a RTR0PROCESS handle to a linux task.
136	*
137	* @returns The corresponding Linux task.
138	* @param R0Process IPRT ring-0 process handle.
139	*/
140	static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
141	{
142	/** @todo fix rtR0ProcessToLinuxTask!! */
143	/** @todo many (all?) callers currently assume that we return 'current'! */
144	return R0Process == RTR0ProcHandleSelf() ? current : NULL;
145	}
146
147
148	/**
149	* Compute order. Some functions allocate 2^order pages.
150	*
151	* @returns order.
152	* @param cPages Number of pages.
153	*/
154	static int rtR0MemObjLinuxOrder(size_t cPages)
155	{
156	int iOrder;
157	size_t cTmp;
158
159	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
160	;
161	if (cPages & ~((size_t)1 << iOrder))
162	++iOrder;
163
164	return iOrder;
165	}
166
167
168	/**
169	* Converts from RTMEM_PROT_* to Linux PAGE_*.
170	*
171	* @returns Linux page protection constant.
172	* @param fProt The IPRT protection mask.
173	* @param fKernel Whether it applies to kernel or user space.
174	*/
175	static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
176	{
177	switch (fProt)
178	{
179	default:
180	AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
181	case RTMEM_PROT_NONE:
182	return PAGE_NONE;
183
184	case RTMEM_PROT_READ:
185	return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
186
187	case RTMEM_PROT_WRITE:
188	case RTMEM_PROT_WRITE \| RTMEM_PROT_READ:
189	return fKernel ? PAGE_KERNEL : PAGE_SHARED;
190
191	case RTMEM_PROT_EXEC:
192	case RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
193	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
194	if (fKernel)
195	{
196	pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
197	pgprot_val(fPg) &= ~_PAGE_RW;
198	return fPg;
199	}
200	return PAGE_READONLY_EXEC;
201	#else
202	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
203	#endif
204
205	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
206	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
207	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
208	}
209	}
210
211
212	/**
213	* Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
214	* an empty user space mapping.
215	*
216	* We acquire the mmap_sem/mmap_lock of the task!
217	*
218	* @returns Pointer to the mapping.
219	* (void *)-1 on failure.
220	* @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
221	* @param cb The size of the mapping.
222	* @param uAlignment The alignment of the mapping.
223	* @param pTask The Linux task to create this mapping in.
224	* @param fProt The RTMEM_PROT_* mask.
225	*/
226	static void rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct pTask, unsigned fProt)
227	{
228	unsigned fLnxProt;
229	unsigned long ulAddr;
230
231	Assert(pTask == current); /* do_mmap */
232	RT_NOREF_PV(pTask);
233
234	/*
235	* Convert from IPRT protection to mman.h PROT_ and call do_mmap.
236	*/
237	fProt &= (RTMEM_PROT_NONE \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC);
238	if (fProt == RTMEM_PROT_NONE)
239	fLnxProt = PROT_NONE;
240	else
241	{
242	fLnxProt = 0;
243	if (fProt & RTMEM_PROT_READ)
244	fLnxProt \|= PROT_READ;
245	if (fProt & RTMEM_PROT_WRITE)
246	fLnxProt \|= PROT_WRITE;
247	if (fProt & RTMEM_PROT_EXEC)
248	fLnxProt \|= PROT_EXEC;
249	}
250
251	if (R3PtrFixed != (RTR3PTR)-1)
252	{
253	#if RTLNX_VER_MIN(3,5,0)
254	ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
255	#else
256	LNX_MM_DOWN_WRITE(pTask->mm);
257	ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
258	LNX_MM_UP_WRITE(pTask->mm);
259	#endif
260	}
261	else
262	{
263	#if RTLNX_VER_MIN(3,5,0)
264	ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
265	#else
266	LNX_MM_DOWN_WRITE(pTask->mm);
267	ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
268	LNX_MM_UP_WRITE(pTask->mm);
269	#endif
270	if ( !(ulAddr & ~PAGE_MASK)
271	&& (ulAddr & (uAlignment - 1)))
272	{
273	/** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
274	* up alignment gaps. This is of course complicated by fragmentation (which we might have cause
275	* ourselves) and further by there begin two mmap strategies (top / bottom). */
276	/* For now, just ignore uAlignment requirements... */
277	}
278	}
279
280
281	if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
282	return (void *)-1;
283	return (void *)ulAddr;
284	}
285
286
287	/**
288	* Worker that destroys a user space mapping.
289	* Undoes what rtR0MemObjLinuxDoMmap did.
290	*
291	* We acquire the mmap_sem/mmap_lock of the task!
292	*
293	* @param pv The ring-3 mapping.
294	* @param cb The size of the mapping.
295	* @param pTask The Linux task to destroy this mapping in.
296	*/
297	static void rtR0MemObjLinuxDoMunmap(void pv, size_t cb, struct task_struct pTask)
298	{
299	#if RTLNX_VER_MIN(3,5,0)
300	Assert(pTask == current); RT_NOREF_PV(pTask);
301	vm_munmap((unsigned long)pv, cb);
302	#elif defined(USE_RHEL4_MUNMAP)
303	LNX_MM_DOWN_WRITE(pTask->mm);
304	do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
305	LNX_MM_UP_WRITE(pTask->mm);
306	#else
307	LNX_MM_DOWN_WRITE(pTask->mm);
308	do_munmap(pTask->mm, (unsigned long)pv, cb);
309	LNX_MM_UP_WRITE(pTask->mm);
310	#endif
311	}
312
313
314	/**
315	* Internal worker that allocates physical pages and creates the memory object for them.
316	*
317	* @returns IPRT status code.
318	* @param ppMemLnx Where to store the memory object pointer.
319	* @param enmType The object type.
320	* @param cb The number of bytes to allocate.
321	* @param uAlignment The alignment of the physical memory.
322	* Only valid if fContiguous == true, ignored otherwise.
323	* @param fFlagsLnx The page allocation flags (GPFs).
324	* @param fContiguous Whether the allocation must be contiguous.
325	* @param fExecutable Whether the memory must be executable.
326	* @param rcNoMem What to return when we're out of pages.
327	*/
328	static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
329	size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, bool fExecutable, int rcNoMem)
330	{
331	size_t iPage;
332	size_t const cPages = cb >> PAGE_SHIFT;
333	struct page *paPages;
334
335	/*
336	* Allocate a memory object structure that's large enough to contain
337	* the page pointer array.
338	*/
339	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType, NULL, cb);
340	if (!pMemLnx)
341	return VERR_NO_MEMORY;
342	pMemLnx->cPages = cPages;
343
344	if (cPages > 255)
345	{
346	# ifdef __GFP_REPEAT
347	/* Try hard to allocate the memory, but the allocation attempt might fail. */
348	fFlagsLnx \|= __GFP_REPEAT;
349	# endif
350	# ifdef __GFP_NOMEMALLOC
351	/* Introduced with Linux 2.6.12: Don't use emergency reserves */
352	fFlagsLnx \|= __GFP_NOMEMALLOC;
353	# endif
354	}
355
356	/*
357	* Allocate the pages.
358	* For small allocations we'll try contiguous first and then fall back on page by page.
359	*/
360	#if RTLNX_VER_MIN(2,4,22)
361	if ( fContiguous
362	\|\| cb <= PAGE_SIZE * 2)
363	{
364	# ifdef VBOX_USE_INSERT_PAGE
365	paPages = alloc_pages(fFlagsLnx \| __GFP_COMP \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
366	# else
367	paPages = alloc_pages(fFlagsLnx \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
368	# endif
369	if (paPages)
370	{
371	fContiguous = true;
372	for (iPage = 0; iPage < cPages; iPage++)
373	pMemLnx->apPages[iPage] = &paPages[iPage];
374	}
375	else if (fContiguous)
376	{
377	rtR0MemObjDelete(&pMemLnx->Core);
378	return rcNoMem;
379	}
380	}
381
382	if (!fContiguous)
383	{
384	for (iPage = 0; iPage < cPages; iPage++)
385	{
386	pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx \| __GFP_NOWARN);
387	if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
388	{
389	while (iPage-- > 0)
390	__free_page(pMemLnx->apPages[iPage]);
391	rtR0MemObjDelete(&pMemLnx->Core);
392	return rcNoMem;
393	}
394	}
395	}
396
397	#else /* < 2.4.22 */
398	/** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
399	paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
400	if (!paPages)
401	{
402	rtR0MemObjDelete(&pMemLnx->Core);
403	return rcNoMem;
404	}
405	for (iPage = 0; iPage < cPages; iPage++)
406	{
407	pMemLnx->apPages[iPage] = &paPages[iPage];
408	if (fExecutable)
409	MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
410	if (PageHighMem(pMemLnx->apPages[iPage]))
411	BUG();
412	}
413
414	fContiguous = true;
415	#endif /* < 2.4.22 */
416	pMemLnx->fContiguous = fContiguous;
417	pMemLnx->fExecutable = fExecutable;
418
419	#if RTLNX_VER_MAX(4,5,0)
420	/*
421	* Reserve the pages.
422	*
423	* Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
424	* pages. According to Michal Hocko this shouldn't be necessary anyway because
425	* as pages which are not on the LRU list are never evictable.
426	*/
427	for (iPage = 0; iPage < cPages; iPage++)
428	SetPageReserved(pMemLnx->apPages[iPage]);
429	#endif
430
431	/*
432	* Note that the physical address of memory allocated with alloc_pages(flags, order)
433	* is always 2^(PAGE_SHIFT+order)-aligned.
434	*/
435	if ( fContiguous
436	&& uAlignment > PAGE_SIZE)
437	{
438	/*
439	* Check for alignment constraints. The physical address of memory allocated with
440	* alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
441	*/
442	if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
443	{
444	/*
445	* This should never happen!
446	*/
447	printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
448	(unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
449	rtR0MemObjLinuxFreePages(pMemLnx);
450	return rcNoMem;
451	}
452	}
453
454	*ppMemLnx = pMemLnx;
455	return VINF_SUCCESS;
456	}
457
458
459	/**
460	* Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
461	*
462	* This method does NOT free the object.
463	*
464	* @param pMemLnx The object which physical pages should be freed.
465	*/
466	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
467	{
468	size_t iPage = pMemLnx->cPages;
469	if (iPage > 0)
470	{
471	/*
472	* Restore the page flags.
473	*/
474	while (iPage-- > 0)
475	{
476	#if RTLNX_VER_MAX(4,5,0)
477	/* See SetPageReserved() in rtR0MemObjLinuxAllocPages() */
478	ClearPageReserved(pMemLnx->apPages[iPage]);
479	#endif
480	#if RTLNX_VER_MAX(2,4,22)
481	if (pMemLnx->fExecutable)
482	MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
483	#endif
484	}
485
486	/*
487	* Free the pages.
488	*/
489	#if RTLNX_VER_MIN(2,4,22)
490	if (!pMemLnx->fContiguous)
491	{
492	iPage = pMemLnx->cPages;
493	while (iPage-- > 0)
494	__free_page(pMemLnx->apPages[iPage]);
495	}
496	else
497	#endif
498	__free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
499
500	pMemLnx->cPages = 0;
501	}
502	}
503
504
505	/**
506	* Maps the allocation into ring-0.
507	*
508	* This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
509	*
510	* Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
511	* space, so we'll use that mapping if possible. If execute access is required, we'll
512	* play safe and do our own mapping.
513	*
514	* @returns IPRT status code.
515	* @param pMemLnx The linux memory object to map.
516	* @param fExecutable Whether execute access is required.
517	*/
518	static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
519	{
520	int rc = VINF_SUCCESS;
521
522	/*
523	* Choose mapping strategy.
524	*/
525	bool fMustMap = fExecutable
526	\|\| !pMemLnx->fContiguous;
527	if (!fMustMap)
528	{
529	size_t iPage = pMemLnx->cPages;
530	while (iPage-- > 0)
531	if (PageHighMem(pMemLnx->apPages[iPage]))
532	{
533	fMustMap = true;
534	break;
535	}
536	}
537
538	Assert(!pMemLnx->Core.pv);
539	Assert(!pMemLnx->fMappedToRing0);
540
541	if (fMustMap)
542	{
543	/*
544	* Use vmap - 2.4.22 and later.
545	*/
546	#if RTLNX_VER_MIN(2,4,22)
547	pgprot_t fPg;
548	pgprot_val(fPg) = _PAGE_PRESENT \| _PAGE_RW;
549	# ifdef _PAGE_NX
550	if (!fExecutable)
551	pgprot_val(fPg) \|= _PAGE_NX;
552	# endif
553
554	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
555	if (fExecutable)
556	{
557	pte_t papPtes = (pte_t )kmalloc_array(pMemLnx->cPages, sizeof(papPtes[0]), GFP_KERNEL);
558	if (papPtes)
559	{
560	pMemLnx->pArea = alloc_vm_area(pMemLnx->Core.cb, papPtes); /* Note! pArea->nr_pages is not set. */
561	if (pMemLnx->pArea)
562	{
563	size_t i;
564	Assert(pMemLnx->pArea->size >= pMemLnx->Core.cb); /* Note! includes guard page. */
565	Assert(pMemLnx->pArea->addr);
566	# ifdef _PAGE_NX
567	pgprot_val(fPg) \|= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
568	# endif
569	pMemLnx->papPtesForArea = papPtes;
570	for (i = 0; i < pMemLnx->cPages; i++)
571	*papPtes[i] = mk_pte(pMemLnx->apPages[i], fPg);
572	pMemLnx->Core.pv = pMemLnx->pArea->addr;
573	pMemLnx->fMappedToRing0 = true;
574	}
575	else
576	{
577	kfree(papPtes);
578	rc = VERR_MAP_FAILED;
579	}
580	}
581	else
582	rc = VERR_MAP_FAILED;
583	}
584	else
585	# endif
586	{
587	# ifdef VM_MAP
588	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
589	# else
590	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
591	# endif
592	if (pMemLnx->Core.pv)
593	pMemLnx->fMappedToRing0 = true;
594	else
595	rc = VERR_MAP_FAILED;
596	}
597	#else /* < 2.4.22 */
598	rc = VERR_NOT_SUPPORTED;
599	#endif
600	}
601	else
602	{
603	/*
604	* Use the kernel RAM mapping.
605	*/
606	pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
607	Assert(pMemLnx->Core.pv);
608	}
609
610	return rc;
611	}
612
613
614	/**
615	* Undoes what rtR0MemObjLinuxVMap() did.
616	*
617	* @param pMemLnx The linux memory object.
618	*/
619	static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
620	{
621	#if RTLNX_VER_MIN(2,4,22)
622	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
623	if (pMemLnx->pArea)
624	{
625	# if 0
626	pte_t **papPtes = pMemLnx->papPtesForArea;
627	size_t i;
628	for (i = 0; i < pMemLnx->cPages; i++)
629	*papPtes[i] = 0;
630	# endif
631	free_vm_area(pMemLnx->pArea);
632	kfree(pMemLnx->papPtesForArea);
633	pMemLnx->pArea = NULL;
634	pMemLnx->papPtesForArea = NULL;
635	}
636	else
637	# endif
638	if (pMemLnx->fMappedToRing0)
639	{
640	Assert(pMemLnx->Core.pv);
641	vunmap(pMemLnx->Core.pv);
642	pMemLnx->fMappedToRing0 = false;
643	}
644	#else /* < 2.4.22 */
645	Assert(!pMemLnx->fMappedToRing0);
646	#endif
647	pMemLnx->Core.pv = NULL;
648	}
649
650
651	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
652	{
653	IPRT_LINUX_SAVE_EFL_AC();
654	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
655
656	/*
657	* Release any memory that we've allocated or locked.
658	*/
659	switch (pMemLnx->Core.enmType)
660	{
661	case RTR0MEMOBJTYPE_LOW:
662	case RTR0MEMOBJTYPE_PAGE:
663	case RTR0MEMOBJTYPE_CONT:
664	case RTR0MEMOBJTYPE_PHYS:
665	case RTR0MEMOBJTYPE_PHYS_NC:
666	rtR0MemObjLinuxVUnmap(pMemLnx);
667	rtR0MemObjLinuxFreePages(pMemLnx);
668	break;
669
670	case RTR0MEMOBJTYPE_LOCK:
671	if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
672	{
673	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
674	size_t iPage;
675	Assert(pTask);
676	if (pTask && pTask->mm)
677	LNX_MM_DOWN_READ(pTask->mm);
678
679	iPage = pMemLnx->cPages;
680	while (iPage-- > 0)
681	{
682	if (!PageReserved(pMemLnx->apPages[iPage]))
683	SetPageDirty(pMemLnx->apPages[iPage]);
684	#if RTLNX_VER_MIN(4,6,0)
685	put_page(pMemLnx->apPages[iPage]);
686	#else
687	page_cache_release(pMemLnx->apPages[iPage]);
688	#endif
689	}
690
691	if (pTask && pTask->mm)
692	LNX_MM_UP_READ(pTask->mm);
693	}
694	/* else: kernel memory - nothing to do here. */
695	break;
696
697	case RTR0MEMOBJTYPE_RES_VIRT:
698	Assert(pMemLnx->Core.pv);
699	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
700	{
701	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
702	Assert(pTask);
703	if (pTask && pTask->mm)
704	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
705	}
706	else
707	{
708	vunmap(pMemLnx->Core.pv);
709
710	Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
711	__free_page(pMemLnx->apPages[0]);
712	pMemLnx->apPages[0] = NULL;
713	pMemLnx->cPages = 0;
714	}
715	pMemLnx->Core.pv = NULL;
716	break;
717
718	case RTR0MEMOBJTYPE_MAPPING:
719	Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
720	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
721	{
722	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
723	Assert(pTask);
724	if (pTask && pTask->mm)
725	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
726	}
727	else
728	vunmap(pMemLnx->Core.pv);
729	pMemLnx->Core.pv = NULL;
730	break;
731
732	default:
733	AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
734	return VERR_INTERNAL_ERROR;
735	}
736	IPRT_LINUX_RESTORE_EFL_ONLY_AC();
737	return VINF_SUCCESS;
738	}
739
740
741	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
742	{
743	IPRT_LINUX_SAVE_EFL_AC();
744	PRTR0MEMOBJLNX pMemLnx;
745	int rc;
746
747	#if RTLNX_VER_MIN(2,4,22)
748	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
749	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
750	#else
751	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
752	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
753	#endif
754	if (RT_SUCCESS(rc))
755	{
756	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
757	if (RT_SUCCESS(rc))
758	{
759	*ppMem = &pMemLnx->Core;
760	IPRT_LINUX_RESTORE_EFL_AC();
761	return rc;
762	}
763
764	rtR0MemObjLinuxFreePages(pMemLnx);
765	rtR0MemObjDelete(&pMemLnx->Core);
766	}
767
768	IPRT_LINUX_RESTORE_EFL_AC();
769	return rc;
770	}
771
772
773	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
774	{
775	IPRT_LINUX_SAVE_EFL_AC();
776	PRTR0MEMOBJLNX pMemLnx;
777	int rc;
778
779	/* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
780	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
781	/* ZONE_DMA32: 0-4GB */
782	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
783	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
784	if (RT_FAILURE(rc))
785	#endif
786	#ifdef RT_ARCH_AMD64
787	/* ZONE_DMA: 0-16MB */
788	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
789	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
790	#else
791	# ifdef CONFIG_X86_PAE
792	# endif
793	/* ZONE_NORMAL: 0-896MB */
794	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
795	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
796	#endif
797	if (RT_SUCCESS(rc))
798	{
799	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
800	if (RT_SUCCESS(rc))
801	{
802	*ppMem = &pMemLnx->Core;
803	IPRT_LINUX_RESTORE_EFL_AC();
804	return rc;
805	}
806
807	rtR0MemObjLinuxFreePages(pMemLnx);
808	rtR0MemObjDelete(&pMemLnx->Core);
809	}
810
811	IPRT_LINUX_RESTORE_EFL_AC();
812	return rc;
813	}
814
815
816	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
817	{
818	IPRT_LINUX_SAVE_EFL_AC();
819	PRTR0MEMOBJLNX pMemLnx;
820	int rc;
821
822	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
823	/* ZONE_DMA32: 0-4GB */
824	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
825	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
826	if (RT_FAILURE(rc))
827	#endif
828	#ifdef RT_ARCH_AMD64
829	/* ZONE_DMA: 0-16MB */
830	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
831	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
832	#else
833	/* ZONE_NORMAL (32-bit hosts): 0-896MB */
834	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
835	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
836	#endif
837	if (RT_SUCCESS(rc))
838	{
839	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
840	if (RT_SUCCESS(rc))
841	{
842	#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_HIGHMEM64G))
843	size_t iPage = pMemLnx->cPages;
844	while (iPage-- > 0)
845	Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
846	#endif
847	pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
848	*ppMem = &pMemLnx->Core;
849	IPRT_LINUX_RESTORE_EFL_AC();
850	return rc;
851	}
852
853	rtR0MemObjLinuxFreePages(pMemLnx);
854	rtR0MemObjDelete(&pMemLnx->Core);
855	}
856
857	IPRT_LINUX_RESTORE_EFL_AC();
858	return rc;
859	}
860
861
862	/**
863	* Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
864	*
865	* @returns IPRT status code.
866	* @param ppMemLnx Where to
867	* @param enmType The object type.
868	* @param cb The size of the allocation.
869	* @param uAlignment The alignment of the physical memory.
870	* Only valid for fContiguous == true, ignored otherwise.
871	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
872	* @param fGfp The Linux GFP flags to use for the allocation.
873	*/
874	static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
875	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, gfp_t fGfp)
876	{
877	PRTR0MEMOBJLNX pMemLnx;
878	int rc;
879
880	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
881	enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
882	false /fExecutable/, VERR_NO_PHYS_MEMORY);
883	if (RT_FAILURE(rc))
884	return rc;
885
886	/*
887	* Check the addresses if necessary. (Can be optimized a bit for PHYS.)
888	*/
889	if (PhysHighest != NIL_RTHCPHYS)
890	{
891	size_t iPage = pMemLnx->cPages;
892	while (iPage-- > 0)
893	if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
894	{
895	rtR0MemObjLinuxFreePages(pMemLnx);
896	rtR0MemObjDelete(&pMemLnx->Core);
897	return VERR_NO_MEMORY;
898	}
899	}
900
901	/*
902	* Complete the object.
903	*/
904	if (enmType == RTR0MEMOBJTYPE_PHYS)
905	{
906	pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
907	pMemLnx->Core.u.Phys.fAllocated = true;
908	}
909	*ppMem = &pMemLnx->Core;
910	return rc;
911	}
912
913
914	/**
915	* Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
916	*
917	* @returns IPRT status code.
918	* @param ppMem Where to store the memory object pointer on success.
919	* @param enmType The object type.
920	* @param cb The size of the allocation.
921	* @param uAlignment The alignment of the physical memory.
922	* Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
923	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
924	*/
925	static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
926	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest)
927	{
928	int rc;
929	IPRT_LINUX_SAVE_EFL_AC();
930
931	/*
932	* There are two clear cases and that's the <=16MB and anything-goes ones.
933	* When the physical address limit is somewhere in-between those two we'll
934	* just have to try, starting with HIGHUSER and working our way thru the
935	* different types, hoping we'll get lucky.
936	*
937	* We should probably move this physical address restriction logic up to
938	* the page alloc function as it would be more efficient there. But since
939	* we don't expect this to be a performance issue just yet it can wait.
940	*/
941	if (PhysHighest == NIL_RTHCPHYS)
942	/* ZONE_HIGHMEM: the whole physical memory */
943	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
944	else if (PhysHighest <= _1M * 16)
945	/* ZONE_DMA: 0-16MB */
946	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
947	else
948	{
949	rc = VERR_NO_MEMORY;
950	if (RT_FAILURE(rc))
951	/* ZONE_HIGHMEM: the whole physical memory */
952	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
953	if (RT_FAILURE(rc))
954	/* ZONE_NORMAL: 0-896MB */
955	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_USER);
956	#ifdef GFP_DMA32
957	if (RT_FAILURE(rc))
958	/* ZONE_DMA32: 0-4GB */
959	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA32);
960	#endif
961	if (RT_FAILURE(rc))
962	/* ZONE_DMA: 0-16MB */
963	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
964	}
965	IPRT_LINUX_RESTORE_EFL_AC();
966	return rc;
967	}
968
969
970	/**
971	* Translates a kernel virtual address to a linux page structure by walking the
972	* page tables.
973	*
974	* @note We do assume that the page tables will not change as we are walking
975	* them. This assumption is rather forced by the fact that I could not
976	* immediately see any way of preventing this from happening. So, we
977	* take some extra care when accessing them.
978	*
979	* Because of this, we don't want to use this function on memory where
980	* attribute changes to nearby pages is likely to cause large pages to
981	* be used or split up. So, don't use this for the linear mapping of
982	* physical memory.
983	*
984	* @returns Pointer to the page structur or NULL if it could not be found.
985	* @param pv The kernel virtual address.
986	*/
987	RTDECL(struct page ) rtR0MemObjLinuxVirtToPage(void pv)
988	{
989	unsigned long ulAddr = (unsigned long)pv;
990	unsigned long pfn;
991	struct page *pPage;
992	pte_t *pEntry;
993	union
994	{
995	pgd_t Global;
996	#if RTLNX_VER_MIN(4,12,0)
997	p4d_t Four;
998	#endif
999	#if RTLNX_VER_MIN(2,6,11)
1000	pud_t Upper;
1001	#endif
1002	pmd_t Middle;
1003	pte_t Entry;
1004	} u;
1005
1006	/* Should this happen in a situation this code will be called in? And if
1007	* so, can it change under our feet? See also
1008	* "Documentation/vm/active_mm.txt" in the kernel sources. */
1009	if (RT_UNLIKELY(!current->active_mm))
1010	return NULL;
1011	u.Global = *pgd_offset(current->active_mm, ulAddr);
1012	if (RT_UNLIKELY(pgd_none(u.Global)))
1013	return NULL;
1014	#if RTLNX_VER_MIN(2,6,11)
1015	# if RTLNX_VER_MIN(4,12,0)
1016	u.Four = *p4d_offset(&u.Global, ulAddr);
1017	if (RT_UNLIKELY(p4d_none(u.Four)))
1018	return NULL;
1019	if (p4d_large(u.Four))
1020	{
1021	pPage = p4d_page(u.Four);
1022	AssertReturn(pPage, NULL);
1023	pfn = page_to_pfn(pPage); /* doing the safe way... */
1024	AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
1025	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
1026	return pfn_to_page(pfn);
1027	}
1028	u.Upper = *pud_offset(&u.Four, ulAddr);
1029	# else /* < 4.12 */
1030	u.Upper = *pud_offset(&u.Global, ulAddr);
1031	# endif /* < 4.12 */
1032	if (RT_UNLIKELY(pud_none(u.Upper)))
1033	return NULL;
1034	# if RTLNX_VER_MIN(2,6,25)
1035	if (pud_large(u.Upper))
1036	{
1037	pPage = pud_page(u.Upper);
1038	AssertReturn(pPage, NULL);
1039	pfn = page_to_pfn(pPage); /* doing the safe way... */
1040	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
1041	return pfn_to_page(pfn);
1042	}
1043	# endif
1044	u.Middle = *pmd_offset(&u.Upper, ulAddr);
1045	#else /* < 2.6.11 */
1046	u.Middle = *pmd_offset(&u.Global, ulAddr);
1047	#endif /* < 2.6.11 */
1048	if (RT_UNLIKELY(pmd_none(u.Middle)))
1049	return NULL;
1050	#if RTLNX_VER_MIN(2,6,0)
1051	if (pmd_large(u.Middle))
1052	{
1053	pPage = pmd_page(u.Middle);
1054	AssertReturn(pPage, NULL);
1055	pfn = page_to_pfn(pPage); /* doing the safe way... */
1056	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
1057	return pfn_to_page(pfn);
1058	}
1059	#endif
1060
1061	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
1062	pEntry = pte_offset_map(&u.Middle, ulAddr);
1063	#else
1064	pEntry = pte_offset(&u.Middle, ulAddr);
1065	#endif
1066	if (RT_UNLIKELY(!pEntry))
1067	return NULL;
1068	u.Entry = *pEntry;
1069	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map)
1070	pte_unmap(pEntry);
1071	#endif
1072
1073	if (RT_UNLIKELY(!pte_present(u.Entry)))
1074	return NULL;
1075	return pte_page(u.Entry);
1076	}
1077	RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
1078
1079
1080	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
1081	{
1082	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest);
1083	}
1084
1085
1086	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
1087	{
1088	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest);
1089	}
1090
1091
1092	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
1093	{
1094	IPRT_LINUX_SAVE_EFL_AC();
1095
1096	/*
1097	* All we need to do here is to validate that we can use
1098	* ioremap on the specified address (32/64-bit dma_addr_t).
1099	*/
1100	PRTR0MEMOBJLNX pMemLnx;
1101	dma_addr_t PhysAddr = Phys;
1102	AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
1103
1104	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb);
1105	if (!pMemLnx)
1106	{
1107	IPRT_LINUX_RESTORE_EFL_AC();
1108	return VERR_NO_MEMORY;
1109	}
1110
1111	pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
1112	pMemLnx->Core.u.Phys.fAllocated = false;
1113	pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
1114	Assert(!pMemLnx->cPages);
1115	*ppMem = &pMemLnx->Core;
1116	IPRT_LINUX_RESTORE_EFL_AC();
1117	return VINF_SUCCESS;
1118	}
1119
1120	/* openSUSE Leap 42.3 detection :-/ */
1121	#if RTLNX_VER_RANGE(4,4,0, 4,6,0) && defined(FAULT_FLAG_REMOTE)
1122	# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
1123	#else
1124	# define GET_USER_PAGES_API LINUX_VERSION_CODE
1125	#endif
1126
1127	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
1128	{
1129	IPRT_LINUX_SAVE_EFL_AC();
1130	const int cPages = cb >> PAGE_SHIFT;
1131	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1132	struct vm_area_struct **papVMAs;
1133	PRTR0MEMOBJLNX pMemLnx;
1134	int rc = VERR_NO_MEMORY;
1135	int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
1136
1137	/*
1138	* Check for valid task and size overflows.
1139	*/
1140	if (!pTask)
1141	return VERR_NOT_SUPPORTED;
1142	if (((size_t)cPages << PAGE_SHIFT) != cb)
1143	return VERR_OUT_OF_RANGE;
1144
1145	/*
1146	* Allocate the memory object and a temporary buffer for the VMAs.
1147	*/
1148	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, (void *)R3Ptr, cb);
1149	if (!pMemLnx)
1150	{
1151	IPRT_LINUX_RESTORE_EFL_AC();
1152	return VERR_NO_MEMORY;
1153	}
1154
1155	papVMAs = (struct vm_area_struct *)RTMemAlloc(sizeof(papVMAs) * cPages);
1156	if (papVMAs)
1157	{
1158	LNX_MM_DOWN_READ(pTask->mm);
1159
1160	/*
1161	* Get user pages.
1162	*/
1163	/** @todo r=bird: Should we not force read access too? */
1164	#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
1165	if (R0Process == RTR0ProcHandleSelf())
1166	rc = get_user_pages(R3Ptr, /* Where from. */
1167	cPages, /* How many pages. */
1168	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1169	fWrite ? FOLL_WRITE \| /* Write to memory. */
1170	FOLL_FORCE /* force write access. */
1171	: 0, /* Write to memory. */
1172	# else
1173	fWrite, /* Write to memory. */
1174	fWrite, /* force write access. */
1175	# endif
1176	&pMemLnx->apPages[0], /* Page array. */
1177	papVMAs); /* vmas */
1178	/*
1179	* Actually this should not happen at the moment as call this function
1180	* only for our own process.
1181	*/
1182	else
1183	rc = get_user_pages_remote(
1184	# if GET_USER_PAGES_API < KERNEL_VERSION(5, 9, 0)
1185	pTask, /* Task for fault accounting. */
1186	# endif
1187	pTask->mm, /* Whose pages. */
1188	R3Ptr, /* Where from. */
1189	cPages, /* How many pages. */
1190	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1191	fWrite ? FOLL_WRITE \| /* Write to memory. */
1192	FOLL_FORCE /* force write access. */
1193	: 0, /* Write to memory. */
1194	# else
1195	fWrite, /* Write to memory. */
1196	fWrite, /* force write access. */
1197	# endif
1198	&pMemLnx->apPages[0], /* Page array. */
1199	papVMAs /* vmas */
1200	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
1201	, NULL /* locked */
1202	# endif
1203	);
1204	#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1205	rc = get_user_pages(pTask, /* Task for fault accounting. */
1206	pTask->mm, /* Whose pages. */
1207	R3Ptr, /* Where from. */
1208	cPages, /* How many pages. */
1209	/* The get_user_pages API change was back-ported to 4.4.168. */
1210	# if RTLNX_VER_RANGE(4,4,168, 4,5,0)
1211	fWrite ? FOLL_WRITE \| /* Write to memory. */
1212	FOLL_FORCE /* force write access. */
1213	: 0, /* Write to memory. */
1214	# else
1215	fWrite, /* Write to memory. */
1216	fWrite, /* force write access. */
1217	# endif
1218	&pMemLnx->apPages[0], /* Page array. */
1219	papVMAs); /* vmas */
1220	#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1221	if (rc == cPages)
1222	{
1223	/*
1224	* Flush dcache (required?), protect against fork and _really_ pin the page
1225	* table entries. get_user_pages() will protect against swapping out the
1226	* pages but it will NOT protect against removing page table entries. This
1227	* can be achieved with
1228	* - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1229	* an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1230	* Usual Linux distributions support only a limited size of locked pages
1231	* (e.g. 32KB).
1232	* - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1233	* or by
1234	* - setting the VM_LOCKED flag. This is the same as doing mlock() without
1235	* a range check.
1236	*/
1237	/** @todo The Linux fork() protection will require more work if this API
1238	* is to be used for anything but locking VM pages. */
1239	while (rc-- > 0)
1240	{
1241	flush_dcache_page(pMemLnx->apPages[rc]);
1242	papVMAs[rc]->vm_flags \|= VM_DONTCOPY \| VM_LOCKED;
1243	}
1244
1245	LNX_MM_UP_READ(pTask->mm);
1246
1247	RTMemFree(papVMAs);
1248
1249	pMemLnx->Core.u.Lock.R0Process = R0Process;
1250	pMemLnx->cPages = cPages;
1251	Assert(!pMemLnx->fMappedToRing0);
1252	*ppMem = &pMemLnx->Core;
1253
1254	IPRT_LINUX_RESTORE_EFL_AC();
1255	return VINF_SUCCESS;
1256	}
1257
1258	/*
1259	* Failed - we need to unlock any pages that we succeeded to lock.
1260	*/
1261	while (rc-- > 0)
1262	{
1263	if (!PageReserved(pMemLnx->apPages[rc]))
1264	SetPageDirty(pMemLnx->apPages[rc]);
1265	#if RTLNX_VER_MIN(4,6,0)
1266	put_page(pMemLnx->apPages[rc]);
1267	#else
1268	page_cache_release(pMemLnx->apPages[rc]);
1269	#endif
1270	}
1271
1272	LNX_MM_UP_READ(pTask->mm);
1273
1274	RTMemFree(papVMAs);
1275	rc = VERR_LOCK_FAILED;
1276	}
1277
1278	rtR0MemObjDelete(&pMemLnx->Core);
1279	IPRT_LINUX_RESTORE_EFL_AC();
1280	return rc;
1281	}
1282
1283
1284	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1285	{
1286	IPRT_LINUX_SAVE_EFL_AC();
1287	void pvLast = (uint8_t )pv + cb - 1;
1288	size_t const cPages = cb >> PAGE_SHIFT;
1289	PRTR0MEMOBJLNX pMemLnx;
1290	bool fLinearMapping;
1291	int rc;
1292	uint8_t *pbPage;
1293	size_t iPage;
1294	NOREF(fAccess);
1295
1296	if ( !RTR0MemKernelIsValidAddr(pv)
1297	\|\| !RTR0MemKernelIsValidAddr(pv + cb))
1298	return VERR_INVALID_PARAMETER;
1299
1300	/*
1301	* The lower part of the kernel memory has a linear mapping between
1302	* physical and virtual addresses. So we take a short cut here. This is
1303	* assumed to be the cleanest way to handle those addresses (and the code
1304	* is well tested, though the test for determining it is not very nice).
1305	* If we ever decide it isn't we can still remove it.
1306	*/
1307	#if 0
1308	fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1309	#else
1310	fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1311	&& (unsigned long)pvLast < (unsigned long)high_memory;
1312	#endif
1313
1314	/*
1315	* Allocate the memory object.
1316	*/
1317	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, pv, cb);
1318	if (!pMemLnx)
1319	{
1320	IPRT_LINUX_RESTORE_EFL_AC();
1321	return VERR_NO_MEMORY;
1322	}
1323
1324	/*
1325	* Gather the pages.
1326	* We ASSUME all kernel pages are non-swappable and non-movable.
1327	*/
1328	rc = VINF_SUCCESS;
1329	pbPage = (uint8_t *)pvLast;
1330	iPage = cPages;
1331	if (!fLinearMapping)
1332	{
1333	while (iPage-- > 0)
1334	{
1335	struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1336	if (RT_UNLIKELY(!pPage))
1337	{
1338	rc = VERR_LOCK_FAILED;
1339	break;
1340	}
1341	pMemLnx->apPages[iPage] = pPage;
1342	pbPage -= PAGE_SIZE;
1343	}
1344	}
1345	else
1346	{
1347	while (iPage-- > 0)
1348	{
1349	pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1350	pbPage -= PAGE_SIZE;
1351	}
1352	}
1353	if (RT_SUCCESS(rc))
1354	{
1355	/*
1356	* Complete the memory object and return.
1357	*/
1358	pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1359	pMemLnx->cPages = cPages;
1360	Assert(!pMemLnx->fMappedToRing0);
1361	*ppMem = &pMemLnx->Core;
1362
1363	IPRT_LINUX_RESTORE_EFL_AC();
1364	return VINF_SUCCESS;
1365	}
1366
1367	rtR0MemObjDelete(&pMemLnx->Core);
1368	IPRT_LINUX_RESTORE_EFL_AC();
1369	return rc;
1370	}
1371
1372
1373	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1374	{
1375	#if RTLNX_VER_MIN(2,4,22)
1376	IPRT_LINUX_SAVE_EFL_AC();
1377	const size_t cPages = cb >> PAGE_SHIFT;
1378	struct page *pDummyPage;
1379	struct page **papPages;
1380
1381	/* check for unsupported stuff. */
1382	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1383	if (uAlignment > PAGE_SIZE)
1384	return VERR_NOT_SUPPORTED;
1385
1386	/*
1387	* Allocate a dummy page and create a page pointer array for vmap such that
1388	* the dummy page is mapped all over the reserved area.
1389	*/
1390	pDummyPage = alloc_page(GFP_HIGHUSER \| __GFP_NOWARN);
1391	if (pDummyPage)
1392	{
1393	papPages = RTMemAlloc(sizeof(papPages) cPages);
1394	if (papPages)
1395	{
1396	void *pv;
1397	size_t iPage = cPages;
1398	while (iPage-- > 0)
1399	papPages[iPage] = pDummyPage;
1400	# ifdef VM_MAP
1401	pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1402	# else
1403	pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1404	# endif
1405	RTMemFree(papPages);
1406	if (pv)
1407	{
1408	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1409	if (pMemLnx)
1410	{
1411	pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1412	pMemLnx->cPages = 1;
1413	pMemLnx->apPages[0] = pDummyPage;
1414	*ppMem = &pMemLnx->Core;
1415	IPRT_LINUX_RESTORE_EFL_AC();
1416	return VINF_SUCCESS;
1417	}
1418	vunmap(pv);
1419	}
1420	}
1421	__free_page(pDummyPage);
1422	}
1423	IPRT_LINUX_RESTORE_EFL_AC();
1424	return VERR_NO_MEMORY;
1425
1426	#else /* < 2.4.22 */
1427	/*
1428	* Could probably use ioremap here, but the caller is in a better position than us
1429	* to select some safe physical memory.
1430	*/
1431	return VERR_NOT_SUPPORTED;
1432	#endif
1433	}
1434
1435
1436	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1437	{
1438	IPRT_LINUX_SAVE_EFL_AC();
1439	PRTR0MEMOBJLNX pMemLnx;
1440	void *pv;
1441	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1442	if (!pTask)
1443	return VERR_NOT_SUPPORTED;
1444
1445	/*
1446	* Check that the specified alignment is supported.
1447	*/
1448	if (uAlignment > PAGE_SIZE)
1449	return VERR_NOT_SUPPORTED;
1450
1451	/*
1452	* Let rtR0MemObjLinuxDoMmap do the difficult bits.
1453	*/
1454	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1455	if (pv == (void *)-1)
1456	{
1457	IPRT_LINUX_RESTORE_EFL_AC();
1458	return VERR_NO_MEMORY;
1459	}
1460
1461	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1462	if (!pMemLnx)
1463	{
1464	rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1465	IPRT_LINUX_RESTORE_EFL_AC();
1466	return VERR_NO_MEMORY;
1467	}
1468
1469	pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1470	*ppMem = &pMemLnx->Core;
1471	IPRT_LINUX_RESTORE_EFL_AC();
1472	return VINF_SUCCESS;
1473	}
1474
1475
1476	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap,
1477	void *pvFixed, size_t uAlignment,
1478	unsigned fProt, size_t offSub, size_t cbSub)
1479	{
1480	int rc = VERR_NO_MEMORY;
1481	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1482	PRTR0MEMOBJLNX pMemLnx;
1483	IPRT_LINUX_SAVE_EFL_AC();
1484
1485	/* Fail if requested to do something we can't. */
1486	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1487	if (uAlignment > PAGE_SIZE)
1488	return VERR_NOT_SUPPORTED;
1489
1490	/*
1491	* Create the IPRT memory object.
1492	*/
1493	if (!cbSub)
1494	cbSub = pMemLnxToMap->Core.cb - offSub;
1495	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1496	if (pMemLnx)
1497	{
1498	if (pMemLnxToMap->cPages)
1499	{
1500	#if RTLNX_VER_MIN(2,4,22)
1501	/*
1502	* Use vmap - 2.4.22 and later.
1503	*/
1504	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1505	/** @todo We don't really care too much for EXEC here... 5.8 always adds NX. */
1506	Assert(((offSub + cbSub) >> PAGE_SHIFT) <= pMemLnxToMap->cPages);
1507	# ifdef VM_MAP
1508	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_MAP, fPg);
1509	# else
1510	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_ALLOC, fPg);
1511	# endif
1512	if (pMemLnx->Core.pv)
1513	{
1514	pMemLnx->fMappedToRing0 = true;
1515	rc = VINF_SUCCESS;
1516	}
1517	else
1518	rc = VERR_MAP_FAILED;
1519
1520	#else /* < 2.4.22 */
1521	/*
1522	* Only option here is to share mappings if possible and forget about fProt.
1523	*/
1524	if (rtR0MemObjIsRing3(pMemToMap))
1525	rc = VERR_NOT_SUPPORTED;
1526	else
1527	{
1528	rc = VINF_SUCCESS;
1529	if (!pMemLnxToMap->Core.pv)
1530	rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1531	if (RT_SUCCESS(rc))
1532	{
1533	Assert(pMemLnxToMap->Core.pv);
1534	pMemLnx->Core.pv = (uint8_t *)pMemLnxToMap->Core.pv + offSub;
1535	}
1536	}
1537	#endif
1538	}
1539	else
1540	{
1541	/*
1542	* MMIO / physical memory.
1543	*/
1544	Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1545	#if RTLNX_VER_MIN(2,6,25)
1546	/*
1547	* ioremap() defaults to no caching since the 2.6 kernels.
1548	* ioremap_nocache() has been removed finally in 5.6-rc1.
1549	*/
1550	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1551	? ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1552	: ioremap_cache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1553	#else /* KERNEL_VERSION < 2.6.25 */
1554	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1555	? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1556	: ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1557	#endif /* KERNEL_VERSION < 2.6.25 */
1558	if (pMemLnx->Core.pv)
1559	{
1560	/** @todo fix protection. */
1561	rc = VINF_SUCCESS;
1562	}
1563	}
1564	if (RT_SUCCESS(rc))
1565	{
1566	pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1567	*ppMem = &pMemLnx->Core;
1568	IPRT_LINUX_RESTORE_EFL_AC();
1569	return VINF_SUCCESS;
1570	}
1571	rtR0MemObjDelete(&pMemLnx->Core);
1572	}
1573
1574	IPRT_LINUX_RESTORE_EFL_AC();
1575	return rc;
1576	}
1577
1578
1579	#ifdef VBOX_USE_PAE_HACK
1580	/**
1581	* Replace the PFN of a PTE with the address of the actual page.
1582	*
1583	* The caller maps a reserved dummy page at the address with the desired access
1584	* and flags.
1585	*
1586	* This hack is required for older Linux kernels which don't provide
1587	* remap_pfn_range().
1588	*
1589	* @returns 0 on success, -ENOMEM on failure.
1590	* @param mm The memory context.
1591	* @param ulAddr The mapping address.
1592	* @param Phys The physical address of the page to map.
1593	*/
1594	static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1595	{
1596	int rc = -ENOMEM;
1597	pgd_t *pgd;
1598
1599	spin_lock(&mm->page_table_lock);
1600
1601	pgd = pgd_offset(mm, ulAddr);
1602	if (!pgd_none(pgd) && !pgd_bad(pgd))
1603	{
1604	pmd_t *pmd = pmd_offset(pgd, ulAddr);
1605	if (!pmd_none(*pmd))
1606	{
1607	pte_t *ptep = pte_offset_map(pmd, ulAddr);
1608	if (ptep)
1609	{
1610	pte_t pte = *ptep;
1611	pte.pte_high &= 0xfff00000;
1612	pte.pte_high \|= ((Phys >> 32) & 0x000fffff);
1613	pte.pte_low &= 0x00000fff;
1614	pte.pte_low \|= (Phys & 0xfffff000);
1615	set_pte(ptep, pte);
1616	pte_unmap(ptep);
1617	rc = 0;
1618	}
1619	}
1620	}
1621
1622	spin_unlock(&mm->page_table_lock);
1623	return rc;
1624	}
1625	#endif /* VBOX_USE_PAE_HACK */
1626
1627
1628	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1629	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub)
1630	{
1631	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1632	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1633	int rc = VERR_NO_MEMORY;
1634	PRTR0MEMOBJLNX pMemLnx;
1635	#ifdef VBOX_USE_PAE_HACK
1636	struct page *pDummyPage;
1637	RTHCPHYS DummyPhys;
1638	#endif
1639	IPRT_LINUX_SAVE_EFL_AC();
1640
1641	/*
1642	* Check for restrictions.
1643	*/
1644	if (!pTask)
1645	return VERR_NOT_SUPPORTED;
1646	if (uAlignment > PAGE_SIZE)
1647	return VERR_NOT_SUPPORTED;
1648
1649	#ifdef VBOX_USE_PAE_HACK
1650	/*
1651	* Allocate a dummy page for use when mapping the memory.
1652	*/
1653	pDummyPage = alloc_page(GFP_USER \| __GFP_NOWARN);
1654	if (!pDummyPage)
1655	{
1656	IPRT_LINUX_RESTORE_EFL_AC();
1657	return VERR_NO_MEMORY;
1658	}
1659	SetPageReserved(pDummyPage);
1660	DummyPhys = page_to_phys(pDummyPage);
1661	#endif
1662
1663	/*
1664	* Create the IPRT memory object.
1665	*/
1666	Assert(!offSub \|\| cbSub);
1667	if (cbSub == 0)
1668	cbSub = pMemLnxToMap->Core.cb;
1669	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1670	if (pMemLnx)
1671	{
1672	/*
1673	* Allocate user space mapping.
1674	*/
1675	void *pv;
1676	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cbSub, uAlignment, pTask, fProt);
1677	if (pv != (void *)-1)
1678	{
1679	/*
1680	* Map page by page into the mmap area.
1681	* This is generic, paranoid and not very efficient.
1682	*/
1683	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1684	unsigned long ulAddrCur = (unsigned long)pv;
1685	const size_t cPages = (offSub + cbSub) >> PAGE_SHIFT;
1686	size_t iPage;
1687
1688	LNX_MM_DOWN_WRITE(pTask->mm);
1689
1690	rc = VINF_SUCCESS;
1691	if (pMemLnxToMap->cPages)
1692	{
1693	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1694	{
1695	#if RTLNX_VER_MAX(2,6,11)
1696	RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1697	#endif
1698	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1699	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1700	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1701	#endif
1702	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1703	/* remap_page_range() limitation on x86 */
1704	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1705	#endif
1706
1707	#if defined(VBOX_USE_INSERT_PAGE) && RTLNX_VER_MIN(2,6,22)
1708	rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1709	/* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1710	* See remap_pfn_range() in mm/memory.c */
1711	#if RTLNX_VER_MIN(3,7,0)
1712	vma->vm_flags \|= VM_DONTEXPAND \| VM_DONTDUMP;
1713	#else
1714	vma->vm_flags \|= VM_RESERVED;
1715	#endif
1716	#elif RTLNX_VER_MIN(2,6,11)
1717	rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1718	#elif defined(VBOX_USE_PAE_HACK)
1719	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1720	if (!rc)
1721	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1722	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1723	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1724	#else /* 2.4 */
1725	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1726	#endif
1727	if (rc)
1728	{
1729	rc = VERR_NO_MEMORY;
1730	break;
1731	}
1732	}
1733	}
1734	else
1735	{
1736	RTHCPHYS Phys;
1737	if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
1738	Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
1739	else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
1740	Phys = pMemLnxToMap->Core.u.Cont.Phys;
1741	else
1742	{
1743	AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
1744	Phys = NIL_RTHCPHYS;
1745	}
1746	if (Phys != NIL_RTHCPHYS)
1747	{
1748	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
1749	{
1750	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1751	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1752	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1753	#endif
1754	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1755	/* remap_page_range() limitation on x86 */
1756	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1757	#endif
1758
1759	#if RTLNX_VER_MIN(2,6,11)
1760	rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1761	#elif defined(VBOX_USE_PAE_HACK)
1762	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1763	if (!rc)
1764	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1765	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1766	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1767	#else /* 2.4 */
1768	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1769	#endif
1770	if (rc)
1771	{
1772	rc = VERR_NO_MEMORY;
1773	break;
1774	}
1775	}
1776	}
1777	}
1778
1779	#ifdef CONFIG_NUMA_BALANCING
1780	# if RTLNX_VER_MAX(3,13,0) && RTLNX_RHEL_MAX(7,0)
1781	# define VBOX_NUMA_HACK_OLD
1782	# endif
1783	if (RT_SUCCESS(rc))
1784	{
1785	/** @todo Ugly hack! But right now we have no other means to
1786	* disable automatic NUMA page balancing. */
1787	# ifdef RT_OS_X86
1788	# ifdef VBOX_NUMA_HACK_OLD
1789	pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
1790	# endif
1791	pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
1792	# else
1793	# ifdef VBOX_NUMA_HACK_OLD
1794	pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
1795	# endif
1796	pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
1797	# endif
1798	}
1799	#endif /* CONFIG_NUMA_BALANCING */
1800
1801	LNX_MM_UP_WRITE(pTask->mm);
1802
1803	if (RT_SUCCESS(rc))
1804	{
1805	#ifdef VBOX_USE_PAE_HACK
1806	__free_page(pDummyPage);
1807	#endif
1808	pMemLnx->Core.pv = pv;
1809	pMemLnx->Core.u.Mapping.R0Process = R0Process;
1810	*ppMem = &pMemLnx->Core;
1811	IPRT_LINUX_RESTORE_EFL_AC();
1812	return VINF_SUCCESS;
1813	}
1814
1815	/*
1816	* Bail out.
1817	*/
1818	rtR0MemObjLinuxDoMunmap(pv, cbSub, pTask);
1819	}
1820	rtR0MemObjDelete(&pMemLnx->Core);
1821	}
1822	#ifdef VBOX_USE_PAE_HACK
1823	__free_page(pDummyPage);
1824	#endif
1825
1826	IPRT_LINUX_RESTORE_EFL_AC();
1827	return rc;
1828	}
1829
1830
1831	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1832	{
1833	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
1834	/*
1835	* Currently only supported when we've got addresses PTEs from the kernel.
1836	*/
1837	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1838	if (pMemLnx->pArea && pMemLnx->papPtesForArea)
1839	{
1840	pgprot_t const fPg = rtR0MemObjLinuxConvertProt(fProt, true /fKernel/);
1841	size_t const cPages = (offSub + cbSub) >> PAGE_SHIFT;
1842	pte_t **papPtes = pMemLnx->papPtesForArea;
1843	size_t i;
1844
1845	for (i = offSub >> PAGE_SHIFT; i < cPages; i++)
1846	{
1847	set_pte(papPtes[i], mk_pte(pMemLnx->apPages[i], fPg));
1848	}
1849	preempt_disable();
1850	__flush_tlb_all();
1851	preempt_enable();
1852	return VINF_SUCCESS;
1853	}
1854	# endif
1855
1856	NOREF(pMem);
1857	NOREF(offSub);
1858	NOREF(cbSub);
1859	NOREF(fProt);
1860	return VERR_NOT_SUPPORTED;
1861	}
1862
1863
1864	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1865	{
1866	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1867
1868	if (pMemLnx->cPages)
1869	return page_to_phys(pMemLnx->apPages[iPage]);
1870
1871	switch (pMemLnx->Core.enmType)
1872	{
1873	case RTR0MEMOBJTYPE_CONT:
1874	return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
1875
1876	case RTR0MEMOBJTYPE_PHYS:
1877	return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
1878
1879	/* the parent knows */
1880	case RTR0MEMOBJTYPE_MAPPING:
1881	return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
1882
1883	/* cPages > 0 */
1884	case RTR0MEMOBJTYPE_LOW:
1885	case RTR0MEMOBJTYPE_LOCK:
1886	case RTR0MEMOBJTYPE_PHYS_NC:
1887	case RTR0MEMOBJTYPE_PAGE:
1888	default:
1889	AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
1890	/* fall thru */
1891
1892	case RTR0MEMOBJTYPE_RES_VIRT:
1893	return NIL_RTHCPHYS;
1894	}
1895	}
1896

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c@ 86615

Download in other formats: