memobj-r0drv-linux.c@ 87074

Last change on this file since 87074 was 87074, checked in by vboxsync, 4 years ago
Runtime/memobj-r0drv-linux.c: Changes to support the upcoming 5.10 kernel, bugref:9879
Property svn:eol-style set to `native` Property svn:keywords set to `Id Rev Revision`
File size: 65.6 KB

Line
1	/* $Id: memobj-r0drv-linux.c 87074 2020-12-09 18:59:04Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-linux-kernel.h"
32
33	#include <iprt/memobj.h>
34	#include <iprt/assert.h>
35	#include <iprt/err.h>
36	#include <iprt/log.h>
37	#include <iprt/mem.h>
38	#include <iprt/process.h>
39	#include <iprt/string.h>
40	#include "internal/memobj.h"
41	#include "internal/iprt.h"
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	/* early 2.6 kernels */
48	#ifndef PAGE_SHARED_EXEC
49	# define PAGE_SHARED_EXEC PAGE_SHARED
50	#endif
51	#ifndef PAGE_READONLY_EXEC
52	# define PAGE_READONLY_EXEC PAGE_READONLY
53	#endif
54
55	/** @def IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
56	* Whether we use alloc_vm_area (3.2+) for executable memory.
57	* This is a must for 5.8+, but we enable it all the way back to 3.2.x for
58	* better W^R compliance (fExecutable flag). */
59	#if RTLNX_VER_RANGE(3,2,0, 5,10,0) \|\| defined(DOXYGEN_RUNNING)
60	# define IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
61	#endif
62	/** @def IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
63	* alloc_vm_area was removed with 5.10 so we have to resort to a different way
64	* to allocate executable memory.
65	* It would be possible to remove IPRT_USE_ALLOC_VM_AREA_FOR_EXEC and use
66	* this path execlusively for 3.2+ but no time to test it really works on every
67	* supported kernel, so better play safe for now.
68	*/
69	#if RTLNX_VER_MIN(5,10,0) \|\| defined(DOXYGEN_RUNNING)
70	# define IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
71	#endif
72
73	/*
74	* 2.6.29+ kernels don't work with remap_pfn_range() anymore because
75	* track_pfn_vma_new() is apparently not defined for non-RAM pages.
76	* It should be safe to use vm_insert_page() older kernels as well.
77	*/
78	#if RTLNX_VER_MIN(2,6,23)
79	# define VBOX_USE_INSERT_PAGE
80	#endif
81	#if defined(CONFIG_X86_PAE) \
82	&& ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
83	\|\| RTLNX_VER_RANGE(2,6,0, 2,6,11) )
84	# define VBOX_USE_PAE_HACK
85	#endif
86
87	/* gfp_t was introduced in 2.6.14, define it for earlier. */
88	#if RTLNX_VER_MAX(2,6,14)
89	# define gfp_t unsigned
90	#endif
91
92	/*
93	* Wrappers around mmap_lock/mmap_sem difference.
94	*/
95	#if RTLNX_VER_MIN(5,8,0)
96	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_lock)
97	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_lock)
98	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_lock)
99	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_lock)
100	#else
101	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_sem)
102	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_sem)
103	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_sem)
104	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_sem)
105	#endif
106
107
108	/*********************************************************************************************************************************
109	* Structures and Typedefs *
110	*********************************************************************************************************************************/
111	/**
112	* The Linux version of the memory object structure.
113	*/
114	typedef struct RTR0MEMOBJLNX
115	{
116	/** The core structure. */
117	RTR0MEMOBJINTERNAL Core;
118	/** Set if the allocation is contiguous.
119	* This means it has to be given back as one chunk. */
120	bool fContiguous;
121	/** Set if executable allocation. */
122	bool fExecutable;
123	/** Set if we've vmap'ed the memory into ring-0. */
124	bool fMappedToRing0;
125	#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
126	/** Return from alloc_vm_area() that we now need to use for executable
127	* memory. */
128	struct vm_struct *pArea;
129	/** PTE array that goes along with pArea (must be freed). */
130	pte_t **papPtesForArea;
131	#endif
132	/** The pages in the apPages array. */
133	size_t cPages;
134	/** Array of struct page pointers. (variable size) */
135	struct page *apPages[1];
136	} RTR0MEMOBJLNX;
137	/** Pointer to the linux memory object. */
138	typedef RTR0MEMOBJLNX *PRTR0MEMOBJLNX;
139
140
141	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
142
143
144	/**
145	* Helper that converts from a RTR0PROCESS handle to a linux task.
146	*
147	* @returns The corresponding Linux task.
148	* @param R0Process IPRT ring-0 process handle.
149	*/
150	static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
151	{
152	/** @todo fix rtR0ProcessToLinuxTask!! */
153	/** @todo many (all?) callers currently assume that we return 'current'! */
154	return R0Process == RTR0ProcHandleSelf() ? current : NULL;
155	}
156
157
158	/**
159	* Compute order. Some functions allocate 2^order pages.
160	*
161	* @returns order.
162	* @param cPages Number of pages.
163	*/
164	static int rtR0MemObjLinuxOrder(size_t cPages)
165	{
166	int iOrder;
167	size_t cTmp;
168
169	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
170	;
171	if (cPages & ~((size_t)1 << iOrder))
172	++iOrder;
173
174	return iOrder;
175	}
176
177
178	/**
179	* Converts from RTMEM_PROT_* to Linux PAGE_*.
180	*
181	* @returns Linux page protection constant.
182	* @param fProt The IPRT protection mask.
183	* @param fKernel Whether it applies to kernel or user space.
184	*/
185	static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
186	{
187	switch (fProt)
188	{
189	default:
190	AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
191	case RTMEM_PROT_NONE:
192	return PAGE_NONE;
193
194	case RTMEM_PROT_READ:
195	return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
196
197	case RTMEM_PROT_WRITE:
198	case RTMEM_PROT_WRITE \| RTMEM_PROT_READ:
199	return fKernel ? PAGE_KERNEL : PAGE_SHARED;
200
201	case RTMEM_PROT_EXEC:
202	case RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
203	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
204	if (fKernel)
205	{
206	pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
207	pgprot_val(fPg) &= ~_PAGE_RW;
208	return fPg;
209	}
210	return PAGE_READONLY_EXEC;
211	#else
212	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
213	#endif
214
215	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
216	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
217	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
218	}
219	}
220
221
222	/**
223	* Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
224	* an empty user space mapping.
225	*
226	* We acquire the mmap_sem/mmap_lock of the task!
227	*
228	* @returns Pointer to the mapping.
229	* (void *)-1 on failure.
230	* @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
231	* @param cb The size of the mapping.
232	* @param uAlignment The alignment of the mapping.
233	* @param pTask The Linux task to create this mapping in.
234	* @param fProt The RTMEM_PROT_* mask.
235	*/
236	static void rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct pTask, unsigned fProt)
237	{
238	unsigned fLnxProt;
239	unsigned long ulAddr;
240
241	Assert(pTask == current); /* do_mmap */
242	RT_NOREF_PV(pTask);
243
244	/*
245	* Convert from IPRT protection to mman.h PROT_ and call do_mmap.
246	*/
247	fProt &= (RTMEM_PROT_NONE \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC);
248	if (fProt == RTMEM_PROT_NONE)
249	fLnxProt = PROT_NONE;
250	else
251	{
252	fLnxProt = 0;
253	if (fProt & RTMEM_PROT_READ)
254	fLnxProt \|= PROT_READ;
255	if (fProt & RTMEM_PROT_WRITE)
256	fLnxProt \|= PROT_WRITE;
257	if (fProt & RTMEM_PROT_EXEC)
258	fLnxProt \|= PROT_EXEC;
259	}
260
261	if (R3PtrFixed != (RTR3PTR)-1)
262	{
263	#if RTLNX_VER_MIN(3,5,0)
264	ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
265	#else
266	LNX_MM_DOWN_WRITE(pTask->mm);
267	ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
268	LNX_MM_UP_WRITE(pTask->mm);
269	#endif
270	}
271	else
272	{
273	#if RTLNX_VER_MIN(3,5,0)
274	ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
275	#else
276	LNX_MM_DOWN_WRITE(pTask->mm);
277	ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
278	LNX_MM_UP_WRITE(pTask->mm);
279	#endif
280	if ( !(ulAddr & ~PAGE_MASK)
281	&& (ulAddr & (uAlignment - 1)))
282	{
283	/** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
284	* up alignment gaps. This is of course complicated by fragmentation (which we might have cause
285	* ourselves) and further by there begin two mmap strategies (top / bottom). */
286	/* For now, just ignore uAlignment requirements... */
287	}
288	}
289
290
291	if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
292	return (void *)-1;
293	return (void *)ulAddr;
294	}
295
296
297	/**
298	* Worker that destroys a user space mapping.
299	* Undoes what rtR0MemObjLinuxDoMmap did.
300	*
301	* We acquire the mmap_sem/mmap_lock of the task!
302	*
303	* @param pv The ring-3 mapping.
304	* @param cb The size of the mapping.
305	* @param pTask The Linux task to destroy this mapping in.
306	*/
307	static void rtR0MemObjLinuxDoMunmap(void pv, size_t cb, struct task_struct pTask)
308	{
309	#if RTLNX_VER_MIN(3,5,0)
310	Assert(pTask == current); RT_NOREF_PV(pTask);
311	vm_munmap((unsigned long)pv, cb);
312	#elif defined(USE_RHEL4_MUNMAP)
313	LNX_MM_DOWN_WRITE(pTask->mm);
314	do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
315	LNX_MM_UP_WRITE(pTask->mm);
316	#else
317	LNX_MM_DOWN_WRITE(pTask->mm);
318	do_munmap(pTask->mm, (unsigned long)pv, cb);
319	LNX_MM_UP_WRITE(pTask->mm);
320	#endif
321	}
322
323
324	/**
325	* Internal worker that allocates physical pages and creates the memory object for them.
326	*
327	* @returns IPRT status code.
328	* @param ppMemLnx Where to store the memory object pointer.
329	* @param enmType The object type.
330	* @param cb The number of bytes to allocate.
331	* @param uAlignment The alignment of the physical memory.
332	* Only valid if fContiguous == true, ignored otherwise.
333	* @param fFlagsLnx The page allocation flags (GPFs).
334	* @param fContiguous Whether the allocation must be contiguous.
335	* @param fExecutable Whether the memory must be executable.
336	* @param rcNoMem What to return when we're out of pages.
337	*/
338	static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
339	size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, bool fExecutable, int rcNoMem)
340	{
341	size_t iPage;
342	size_t const cPages = cb >> PAGE_SHIFT;
343	struct page *paPages;
344
345	/*
346	* Allocate a memory object structure that's large enough to contain
347	* the page pointer array.
348	*/
349	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType, NULL, cb);
350	if (!pMemLnx)
351	return VERR_NO_MEMORY;
352	pMemLnx->cPages = cPages;
353
354	if (cPages > 255)
355	{
356	# ifdef __GFP_REPEAT
357	/* Try hard to allocate the memory, but the allocation attempt might fail. */
358	fFlagsLnx \|= __GFP_REPEAT;
359	# endif
360	# ifdef __GFP_NOMEMALLOC
361	/* Introduced with Linux 2.6.12: Don't use emergency reserves */
362	fFlagsLnx \|= __GFP_NOMEMALLOC;
363	# endif
364	}
365
366	/*
367	* Allocate the pages.
368	* For small allocations we'll try contiguous first and then fall back on page by page.
369	*/
370	#if RTLNX_VER_MIN(2,4,22)
371	if ( fContiguous
372	\|\| cb <= PAGE_SIZE * 2)
373	{
374	# ifdef VBOX_USE_INSERT_PAGE
375	paPages = alloc_pages(fFlagsLnx \| __GFP_COMP \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
376	# else
377	paPages = alloc_pages(fFlagsLnx \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
378	# endif
379	if (paPages)
380	{
381	fContiguous = true;
382	for (iPage = 0; iPage < cPages; iPage++)
383	pMemLnx->apPages[iPage] = &paPages[iPage];
384	}
385	else if (fContiguous)
386	{
387	rtR0MemObjDelete(&pMemLnx->Core);
388	return rcNoMem;
389	}
390	}
391
392	if (!fContiguous)
393	{
394	for (iPage = 0; iPage < cPages; iPage++)
395	{
396	pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx \| __GFP_NOWARN);
397	if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
398	{
399	while (iPage-- > 0)
400	__free_page(pMemLnx->apPages[iPage]);
401	rtR0MemObjDelete(&pMemLnx->Core);
402	return rcNoMem;
403	}
404	}
405	}
406
407	#else /* < 2.4.22 */
408	/** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
409	paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
410	if (!paPages)
411	{
412	rtR0MemObjDelete(&pMemLnx->Core);
413	return rcNoMem;
414	}
415	for (iPage = 0; iPage < cPages; iPage++)
416	{
417	pMemLnx->apPages[iPage] = &paPages[iPage];
418	if (fExecutable)
419	MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
420	if (PageHighMem(pMemLnx->apPages[iPage]))
421	BUG();
422	}
423
424	fContiguous = true;
425	#endif /* < 2.4.22 */
426	pMemLnx->fContiguous = fContiguous;
427	pMemLnx->fExecutable = fExecutable;
428
429	#if RTLNX_VER_MAX(4,5,0)
430	/*
431	* Reserve the pages.
432	*
433	* Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
434	* pages. According to Michal Hocko this shouldn't be necessary anyway because
435	* as pages which are not on the LRU list are never evictable.
436	*/
437	for (iPage = 0; iPage < cPages; iPage++)
438	SetPageReserved(pMemLnx->apPages[iPage]);
439	#endif
440
441	/*
442	* Note that the physical address of memory allocated with alloc_pages(flags, order)
443	* is always 2^(PAGE_SHIFT+order)-aligned.
444	*/
445	if ( fContiguous
446	&& uAlignment > PAGE_SIZE)
447	{
448	/*
449	* Check for alignment constraints. The physical address of memory allocated with
450	* alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
451	*/
452	if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
453	{
454	/*
455	* This should never happen!
456	*/
457	printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
458	(unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
459	rtR0MemObjLinuxFreePages(pMemLnx);
460	return rcNoMem;
461	}
462	}
463
464	*ppMemLnx = pMemLnx;
465	return VINF_SUCCESS;
466	}
467
468
469	/**
470	* Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
471	*
472	* This method does NOT free the object.
473	*
474	* @param pMemLnx The object which physical pages should be freed.
475	*/
476	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
477	{
478	size_t iPage = pMemLnx->cPages;
479	if (iPage > 0)
480	{
481	/*
482	* Restore the page flags.
483	*/
484	while (iPage-- > 0)
485	{
486	#if RTLNX_VER_MAX(4,5,0)
487	/* See SetPageReserved() in rtR0MemObjLinuxAllocPages() */
488	ClearPageReserved(pMemLnx->apPages[iPage]);
489	#endif
490	#if RTLNX_VER_MAX(2,4,22)
491	if (pMemLnx->fExecutable)
492	MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
493	#endif
494	}
495
496	/*
497	* Free the pages.
498	*/
499	#if RTLNX_VER_MIN(2,4,22)
500	if (!pMemLnx->fContiguous)
501	{
502	iPage = pMemLnx->cPages;
503	while (iPage-- > 0)
504	__free_page(pMemLnx->apPages[iPage]);
505	}
506	else
507	#endif
508	__free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
509
510	pMemLnx->cPages = 0;
511	}
512	}
513
514
515	#ifdef IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
516	/**
517	* User data passed to the apply_to_page_range() callback.
518	*/
519	typedef struct LNXAPPLYPGRANGE
520	{
521	/** Pointer to the memory object. */
522	PRTR0MEMOBJLNX pMemLnx;
523	/** The page protection flags to apply. */
524	pgprot_t fPg;
525	} LNXAPPLYPGRANGE;
526	/** Pointer to the user data. */
527	typedef LNXAPPLYPGRANGE *PLNXAPPLYPGRANGE;
528	/** Pointer to the const user data. */
529	typedef const LNXAPPLYPGRANGE *PCLNXAPPLYPGRANGE;
530
531	/**
532	* Callback called in apply_to_page_range().
533	*
534	* @returns Linux status code.
535	* @param pPte Pointer to the page table entry for the given address.
536	* @param uAddr The address to apply the new protection to.
537	* @param pvUser The opaque user data.
538	*/
539	static DECLCALLBACK(int) rtR0MemObjLinuxApplyPageRange(pte_t pPte, unsigned long uAddr, void pvUser)
540	{
541	PCLNXAPPLYPGRANGE pArgs = (PCLNXAPPLYPGRANGE)pvUser;
542	PRTR0MEMOBJLNX pMemLnx = pArgs->pMemLnx;
543	size_t idxPg = (uAddr - (unsigned long)pMemLnx->Core.pv) >> PAGE_SHIFT;
544
545	set_pte(pPte, mk_pte(pMemLnx->apPages[idxPg], pArgs->fPg));
546	return 0;
547	}
548	#endif
549
550
551	/**
552	* Maps the allocation into ring-0.
553	*
554	* This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
555	*
556	* Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
557	* space, so we'll use that mapping if possible. If execute access is required, we'll
558	* play safe and do our own mapping.
559	*
560	* @returns IPRT status code.
561	* @param pMemLnx The linux memory object to map.
562	* @param fExecutable Whether execute access is required.
563	*/
564	static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
565	{
566	int rc = VINF_SUCCESS;
567
568	/*
569	* Choose mapping strategy.
570	*/
571	bool fMustMap = fExecutable
572	\|\| !pMemLnx->fContiguous;
573	if (!fMustMap)
574	{
575	size_t iPage = pMemLnx->cPages;
576	while (iPage-- > 0)
577	if (PageHighMem(pMemLnx->apPages[iPage]))
578	{
579	fMustMap = true;
580	break;
581	}
582	}
583
584	Assert(!pMemLnx->Core.pv);
585	Assert(!pMemLnx->fMappedToRing0);
586
587	if (fMustMap)
588	{
589	/*
590	* Use vmap - 2.4.22 and later.
591	*/
592	#if RTLNX_VER_MIN(2,4,22)
593	pgprot_t fPg;
594	pgprot_val(fPg) = _PAGE_PRESENT \| _PAGE_RW;
595	# ifdef _PAGE_NX
596	if (!fExecutable)
597	pgprot_val(fPg) \|= _PAGE_NX;
598	# endif
599
600	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
601	if (fExecutable)
602	{
603	pte_t papPtes = (pte_t )kmalloc_array(pMemLnx->cPages, sizeof(papPtes[0]), GFP_KERNEL);
604	if (papPtes)
605	{
606	pMemLnx->pArea = alloc_vm_area(pMemLnx->Core.cb, papPtes); /* Note! pArea->nr_pages is not set. */
607	if (pMemLnx->pArea)
608	{
609	size_t i;
610	Assert(pMemLnx->pArea->size >= pMemLnx->Core.cb); /* Note! includes guard page. */
611	Assert(pMemLnx->pArea->addr);
612	# ifdef _PAGE_NX
613	pgprot_val(fPg) \|= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
614	# endif
615	pMemLnx->papPtesForArea = papPtes;
616	for (i = 0; i < pMemLnx->cPages; i++)
617	*papPtes[i] = mk_pte(pMemLnx->apPages[i], fPg);
618	pMemLnx->Core.pv = pMemLnx->pArea->addr;
619	pMemLnx->fMappedToRing0 = true;
620	}
621	else
622	{
623	kfree(papPtes);
624	rc = VERR_MAP_FAILED;
625	}
626	}
627	else
628	rc = VERR_MAP_FAILED;
629	}
630	else
631	# endif
632	{
633	# if defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
634	if (fExecutable)
635	pgprot_val(fPg) \|= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
636	# endif
637
638	# ifdef VM_MAP
639	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
640	# else
641	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
642	# endif
643	if (pMemLnx->Core.pv)
644	pMemLnx->fMappedToRing0 = true;
645	else
646	rc = VERR_MAP_FAILED;
647	}
648	#else /* < 2.4.22 */
649	rc = VERR_NOT_SUPPORTED;
650	#endif
651	}
652	else
653	{
654	/*
655	* Use the kernel RAM mapping.
656	*/
657	pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
658	Assert(pMemLnx->Core.pv);
659	}
660
661	return rc;
662	}
663
664
665	/**
666	* Undoes what rtR0MemObjLinuxVMap() did.
667	*
668	* @param pMemLnx The linux memory object.
669	*/
670	static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
671	{
672	#if RTLNX_VER_MIN(2,4,22)
673	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
674	if (pMemLnx->pArea)
675	{
676	# if 0
677	pte_t **papPtes = pMemLnx->papPtesForArea;
678	size_t i;
679	for (i = 0; i < pMemLnx->cPages; i++)
680	*papPtes[i] = 0;
681	# endif
682	free_vm_area(pMemLnx->pArea);
683	kfree(pMemLnx->papPtesForArea);
684	pMemLnx->pArea = NULL;
685	pMemLnx->papPtesForArea = NULL;
686	}
687	else
688	# endif
689	if (pMemLnx->fMappedToRing0)
690	{
691	Assert(pMemLnx->Core.pv);
692	vunmap(pMemLnx->Core.pv);
693	pMemLnx->fMappedToRing0 = false;
694	}
695	#else /* < 2.4.22 */
696	Assert(!pMemLnx->fMappedToRing0);
697	#endif
698	pMemLnx->Core.pv = NULL;
699	}
700
701
702	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
703	{
704	IPRT_LINUX_SAVE_EFL_AC();
705	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
706
707	/*
708	* Release any memory that we've allocated or locked.
709	*/
710	switch (pMemLnx->Core.enmType)
711	{
712	case RTR0MEMOBJTYPE_LOW:
713	case RTR0MEMOBJTYPE_PAGE:
714	case RTR0MEMOBJTYPE_CONT:
715	case RTR0MEMOBJTYPE_PHYS:
716	case RTR0MEMOBJTYPE_PHYS_NC:
717	rtR0MemObjLinuxVUnmap(pMemLnx);
718	rtR0MemObjLinuxFreePages(pMemLnx);
719	break;
720
721	case RTR0MEMOBJTYPE_LOCK:
722	if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
723	{
724	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
725	size_t iPage;
726	Assert(pTask);
727	if (pTask && pTask->mm)
728	LNX_MM_DOWN_READ(pTask->mm);
729
730	iPage = pMemLnx->cPages;
731	while (iPage-- > 0)
732	{
733	if (!PageReserved(pMemLnx->apPages[iPage]))
734	SetPageDirty(pMemLnx->apPages[iPage]);
735	#if RTLNX_VER_MIN(4,6,0)
736	put_page(pMemLnx->apPages[iPage]);
737	#else
738	page_cache_release(pMemLnx->apPages[iPage]);
739	#endif
740	}
741
742	if (pTask && pTask->mm)
743	LNX_MM_UP_READ(pTask->mm);
744	}
745	/* else: kernel memory - nothing to do here. */
746	break;
747
748	case RTR0MEMOBJTYPE_RES_VIRT:
749	Assert(pMemLnx->Core.pv);
750	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
751	{
752	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
753	Assert(pTask);
754	if (pTask && pTask->mm)
755	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
756	}
757	else
758	{
759	vunmap(pMemLnx->Core.pv);
760
761	Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
762	__free_page(pMemLnx->apPages[0]);
763	pMemLnx->apPages[0] = NULL;
764	pMemLnx->cPages = 0;
765	}
766	pMemLnx->Core.pv = NULL;
767	break;
768
769	case RTR0MEMOBJTYPE_MAPPING:
770	Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
771	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
772	{
773	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
774	Assert(pTask);
775	if (pTask && pTask->mm)
776	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
777	}
778	else
779	vunmap(pMemLnx->Core.pv);
780	pMemLnx->Core.pv = NULL;
781	break;
782
783	default:
784	AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
785	return VERR_INTERNAL_ERROR;
786	}
787	IPRT_LINUX_RESTORE_EFL_ONLY_AC();
788	return VINF_SUCCESS;
789	}
790
791
792	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
793	{
794	IPRT_LINUX_SAVE_EFL_AC();
795	PRTR0MEMOBJLNX pMemLnx;
796	int rc;
797
798	#if RTLNX_VER_MIN(2,4,22)
799	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
800	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
801	#else
802	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
803	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
804	#endif
805	if (RT_SUCCESS(rc))
806	{
807	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
808	if (RT_SUCCESS(rc))
809	{
810	*ppMem = &pMemLnx->Core;
811	IPRT_LINUX_RESTORE_EFL_AC();
812	return rc;
813	}
814
815	rtR0MemObjLinuxFreePages(pMemLnx);
816	rtR0MemObjDelete(&pMemLnx->Core);
817	}
818
819	IPRT_LINUX_RESTORE_EFL_AC();
820	return rc;
821	}
822
823
824	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
825	{
826	IPRT_LINUX_SAVE_EFL_AC();
827	PRTR0MEMOBJLNX pMemLnx;
828	int rc;
829
830	/* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
831	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
832	/* ZONE_DMA32: 0-4GB */
833	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
834	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
835	if (RT_FAILURE(rc))
836	#endif
837	#ifdef RT_ARCH_AMD64
838	/* ZONE_DMA: 0-16MB */
839	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
840	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
841	#else
842	# ifdef CONFIG_X86_PAE
843	# endif
844	/* ZONE_NORMAL: 0-896MB */
845	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
846	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
847	#endif
848	if (RT_SUCCESS(rc))
849	{
850	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
851	if (RT_SUCCESS(rc))
852	{
853	*ppMem = &pMemLnx->Core;
854	IPRT_LINUX_RESTORE_EFL_AC();
855	return rc;
856	}
857
858	rtR0MemObjLinuxFreePages(pMemLnx);
859	rtR0MemObjDelete(&pMemLnx->Core);
860	}
861
862	IPRT_LINUX_RESTORE_EFL_AC();
863	return rc;
864	}
865
866
867	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
868	{
869	IPRT_LINUX_SAVE_EFL_AC();
870	PRTR0MEMOBJLNX pMemLnx;
871	int rc;
872
873	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
874	/* ZONE_DMA32: 0-4GB */
875	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
876	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
877	if (RT_FAILURE(rc))
878	#endif
879	#ifdef RT_ARCH_AMD64
880	/* ZONE_DMA: 0-16MB */
881	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
882	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
883	#else
884	/* ZONE_NORMAL (32-bit hosts): 0-896MB */
885	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
886	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
887	#endif
888	if (RT_SUCCESS(rc))
889	{
890	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
891	if (RT_SUCCESS(rc))
892	{
893	#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_HIGHMEM64G))
894	size_t iPage = pMemLnx->cPages;
895	while (iPage-- > 0)
896	Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
897	#endif
898	pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
899	*ppMem = &pMemLnx->Core;
900	IPRT_LINUX_RESTORE_EFL_AC();
901	return rc;
902	}
903
904	rtR0MemObjLinuxFreePages(pMemLnx);
905	rtR0MemObjDelete(&pMemLnx->Core);
906	}
907
908	IPRT_LINUX_RESTORE_EFL_AC();
909	return rc;
910	}
911
912
913	/**
914	* Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
915	*
916	* @returns IPRT status code.
917	* @param ppMemLnx Where to
918	* @param enmType The object type.
919	* @param cb The size of the allocation.
920	* @param uAlignment The alignment of the physical memory.
921	* Only valid for fContiguous == true, ignored otherwise.
922	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
923	* @param fGfp The Linux GFP flags to use for the allocation.
924	*/
925	static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
926	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, gfp_t fGfp)
927	{
928	PRTR0MEMOBJLNX pMemLnx;
929	int rc;
930
931	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
932	enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
933	false /fExecutable/, VERR_NO_PHYS_MEMORY);
934	if (RT_FAILURE(rc))
935	return rc;
936
937	/*
938	* Check the addresses if necessary. (Can be optimized a bit for PHYS.)
939	*/
940	if (PhysHighest != NIL_RTHCPHYS)
941	{
942	size_t iPage = pMemLnx->cPages;
943	while (iPage-- > 0)
944	if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
945	{
946	rtR0MemObjLinuxFreePages(pMemLnx);
947	rtR0MemObjDelete(&pMemLnx->Core);
948	return VERR_NO_MEMORY;
949	}
950	}
951
952	/*
953	* Complete the object.
954	*/
955	if (enmType == RTR0MEMOBJTYPE_PHYS)
956	{
957	pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
958	pMemLnx->Core.u.Phys.fAllocated = true;
959	}
960	*ppMem = &pMemLnx->Core;
961	return rc;
962	}
963
964
965	/**
966	* Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
967	*
968	* @returns IPRT status code.
969	* @param ppMem Where to store the memory object pointer on success.
970	* @param enmType The object type.
971	* @param cb The size of the allocation.
972	* @param uAlignment The alignment of the physical memory.
973	* Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
974	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
975	*/
976	static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
977	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest)
978	{
979	int rc;
980	IPRT_LINUX_SAVE_EFL_AC();
981
982	/*
983	* There are two clear cases and that's the <=16MB and anything-goes ones.
984	* When the physical address limit is somewhere in-between those two we'll
985	* just have to try, starting with HIGHUSER and working our way thru the
986	* different types, hoping we'll get lucky.
987	*
988	* We should probably move this physical address restriction logic up to
989	* the page alloc function as it would be more efficient there. But since
990	* we don't expect this to be a performance issue just yet it can wait.
991	*/
992	if (PhysHighest == NIL_RTHCPHYS)
993	/* ZONE_HIGHMEM: the whole physical memory */
994	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
995	else if (PhysHighest <= _1M * 16)
996	/* ZONE_DMA: 0-16MB */
997	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
998	else
999	{
1000	rc = VERR_NO_MEMORY;
1001	if (RT_FAILURE(rc))
1002	/* ZONE_HIGHMEM: the whole physical memory */
1003	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
1004	if (RT_FAILURE(rc))
1005	/* ZONE_NORMAL: 0-896MB */
1006	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_USER);
1007	#ifdef GFP_DMA32
1008	if (RT_FAILURE(rc))
1009	/* ZONE_DMA32: 0-4GB */
1010	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA32);
1011	#endif
1012	if (RT_FAILURE(rc))
1013	/* ZONE_DMA: 0-16MB */
1014	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
1015	}
1016	IPRT_LINUX_RESTORE_EFL_AC();
1017	return rc;
1018	}
1019
1020
1021	/**
1022	* Translates a kernel virtual address to a linux page structure by walking the
1023	* page tables.
1024	*
1025	* @note We do assume that the page tables will not change as we are walking
1026	* them. This assumption is rather forced by the fact that I could not
1027	* immediately see any way of preventing this from happening. So, we
1028	* take some extra care when accessing them.
1029	*
1030	* Because of this, we don't want to use this function on memory where
1031	* attribute changes to nearby pages is likely to cause large pages to
1032	* be used or split up. So, don't use this for the linear mapping of
1033	* physical memory.
1034	*
1035	* @returns Pointer to the page structur or NULL if it could not be found.
1036	* @param pv The kernel virtual address.
1037	*/
1038	RTDECL(struct page ) rtR0MemObjLinuxVirtToPage(void pv)
1039	{
1040	unsigned long ulAddr = (unsigned long)pv;
1041	unsigned long pfn;
1042	struct page *pPage;
1043	pte_t *pEntry;
1044	union
1045	{
1046	pgd_t Global;
1047	#if RTLNX_VER_MIN(4,12,0)
1048	p4d_t Four;
1049	#endif
1050	#if RTLNX_VER_MIN(2,6,11)
1051	pud_t Upper;
1052	#endif
1053	pmd_t Middle;
1054	pte_t Entry;
1055	} u;
1056
1057	/* Should this happen in a situation this code will be called in? And if
1058	* so, can it change under our feet? See also
1059	* "Documentation/vm/active_mm.txt" in the kernel sources. */
1060	if (RT_UNLIKELY(!current->active_mm))
1061	return NULL;
1062	u.Global = *pgd_offset(current->active_mm, ulAddr);
1063	if (RT_UNLIKELY(pgd_none(u.Global)))
1064	return NULL;
1065	#if RTLNX_VER_MIN(2,6,11)
1066	# if RTLNX_VER_MIN(4,12,0)
1067	u.Four = *p4d_offset(&u.Global, ulAddr);
1068	if (RT_UNLIKELY(p4d_none(u.Four)))
1069	return NULL;
1070	if (p4d_large(u.Four))
1071	{
1072	pPage = p4d_page(u.Four);
1073	AssertReturn(pPage, NULL);
1074	pfn = page_to_pfn(pPage); /* doing the safe way... */
1075	AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
1076	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
1077	return pfn_to_page(pfn);
1078	}
1079	u.Upper = *pud_offset(&u.Four, ulAddr);
1080	# else /* < 4.12 */
1081	u.Upper = *pud_offset(&u.Global, ulAddr);
1082	# endif /* < 4.12 */
1083	if (RT_UNLIKELY(pud_none(u.Upper)))
1084	return NULL;
1085	# if RTLNX_VER_MIN(2,6,25)
1086	if (pud_large(u.Upper))
1087	{
1088	pPage = pud_page(u.Upper);
1089	AssertReturn(pPage, NULL);
1090	pfn = page_to_pfn(pPage); /* doing the safe way... */
1091	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
1092	return pfn_to_page(pfn);
1093	}
1094	# endif
1095	u.Middle = *pmd_offset(&u.Upper, ulAddr);
1096	#else /* < 2.6.11 */
1097	u.Middle = *pmd_offset(&u.Global, ulAddr);
1098	#endif /* < 2.6.11 */
1099	if (RT_UNLIKELY(pmd_none(u.Middle)))
1100	return NULL;
1101	#if RTLNX_VER_MIN(2,6,0)
1102	if (pmd_large(u.Middle))
1103	{
1104	pPage = pmd_page(u.Middle);
1105	AssertReturn(pPage, NULL);
1106	pfn = page_to_pfn(pPage); /* doing the safe way... */
1107	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
1108	return pfn_to_page(pfn);
1109	}
1110	#endif
1111
1112	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
1113	pEntry = pte_offset_map(&u.Middle, ulAddr);
1114	#else
1115	pEntry = pte_offset(&u.Middle, ulAddr);
1116	#endif
1117	if (RT_UNLIKELY(!pEntry))
1118	return NULL;
1119	u.Entry = *pEntry;
1120	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map)
1121	pte_unmap(pEntry);
1122	#endif
1123
1124	if (RT_UNLIKELY(!pte_present(u.Entry)))
1125	return NULL;
1126	return pte_page(u.Entry);
1127	}
1128	RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
1129
1130
1131	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
1132	{
1133	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest);
1134	}
1135
1136
1137	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
1138	{
1139	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest);
1140	}
1141
1142
1143	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
1144	{
1145	IPRT_LINUX_SAVE_EFL_AC();
1146
1147	/*
1148	* All we need to do here is to validate that we can use
1149	* ioremap on the specified address (32/64-bit dma_addr_t).
1150	*/
1151	PRTR0MEMOBJLNX pMemLnx;
1152	dma_addr_t PhysAddr = Phys;
1153	AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
1154
1155	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb);
1156	if (!pMemLnx)
1157	{
1158	IPRT_LINUX_RESTORE_EFL_AC();
1159	return VERR_NO_MEMORY;
1160	}
1161
1162	pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
1163	pMemLnx->Core.u.Phys.fAllocated = false;
1164	pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
1165	Assert(!pMemLnx->cPages);
1166	*ppMem = &pMemLnx->Core;
1167	IPRT_LINUX_RESTORE_EFL_AC();
1168	return VINF_SUCCESS;
1169	}
1170
1171	/* openSUSE Leap 42.3 detection :-/ */
1172	#if RTLNX_VER_RANGE(4,4,0, 4,6,0) && defined(FAULT_FLAG_REMOTE)
1173	# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
1174	#else
1175	# define GET_USER_PAGES_API LINUX_VERSION_CODE
1176	#endif
1177
1178	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
1179	{
1180	IPRT_LINUX_SAVE_EFL_AC();
1181	const int cPages = cb >> PAGE_SHIFT;
1182	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1183	struct vm_area_struct **papVMAs;
1184	PRTR0MEMOBJLNX pMemLnx;
1185	int rc = VERR_NO_MEMORY;
1186	int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
1187
1188	/*
1189	* Check for valid task and size overflows.
1190	*/
1191	if (!pTask)
1192	return VERR_NOT_SUPPORTED;
1193	if (((size_t)cPages << PAGE_SHIFT) != cb)
1194	return VERR_OUT_OF_RANGE;
1195
1196	/*
1197	* Allocate the memory object and a temporary buffer for the VMAs.
1198	*/
1199	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, (void *)R3Ptr, cb);
1200	if (!pMemLnx)
1201	{
1202	IPRT_LINUX_RESTORE_EFL_AC();
1203	return VERR_NO_MEMORY;
1204	}
1205
1206	papVMAs = (struct vm_area_struct *)RTMemAlloc(sizeof(papVMAs) * cPages);
1207	if (papVMAs)
1208	{
1209	LNX_MM_DOWN_READ(pTask->mm);
1210
1211	/*
1212	* Get user pages.
1213	*/
1214	/** @todo r=bird: Should we not force read access too? */
1215	#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
1216	if (R0Process == RTR0ProcHandleSelf())
1217	rc = get_user_pages(R3Ptr, /* Where from. */
1218	cPages, /* How many pages. */
1219	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1220	fWrite ? FOLL_WRITE \| /* Write to memory. */
1221	FOLL_FORCE /* force write access. */
1222	: 0, /* Write to memory. */
1223	# else
1224	fWrite, /* Write to memory. */
1225	fWrite, /* force write access. */
1226	# endif
1227	&pMemLnx->apPages[0], /* Page array. */
1228	papVMAs); /* vmas */
1229	/*
1230	* Actually this should not happen at the moment as call this function
1231	* only for our own process.
1232	*/
1233	else
1234	rc = get_user_pages_remote(
1235	# if GET_USER_PAGES_API < KERNEL_VERSION(5, 9, 0)
1236	pTask, /* Task for fault accounting. */
1237	# endif
1238	pTask->mm, /* Whose pages. */
1239	R3Ptr, /* Where from. */
1240	cPages, /* How many pages. */
1241	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1242	fWrite ? FOLL_WRITE \| /* Write to memory. */
1243	FOLL_FORCE /* force write access. */
1244	: 0, /* Write to memory. */
1245	# else
1246	fWrite, /* Write to memory. */
1247	fWrite, /* force write access. */
1248	# endif
1249	&pMemLnx->apPages[0], /* Page array. */
1250	papVMAs /* vmas */
1251	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
1252	, NULL /* locked */
1253	# endif
1254	);
1255	#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1256	rc = get_user_pages(pTask, /* Task for fault accounting. */
1257	pTask->mm, /* Whose pages. */
1258	R3Ptr, /* Where from. */
1259	cPages, /* How many pages. */
1260	/* The get_user_pages API change was back-ported to 4.4.168. */
1261	# if RTLNX_VER_RANGE(4,4,168, 4,5,0)
1262	fWrite ? FOLL_WRITE \| /* Write to memory. */
1263	FOLL_FORCE /* force write access. */
1264	: 0, /* Write to memory. */
1265	# else
1266	fWrite, /* Write to memory. */
1267	fWrite, /* force write access. */
1268	# endif
1269	&pMemLnx->apPages[0], /* Page array. */
1270	papVMAs); /* vmas */
1271	#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1272	if (rc == cPages)
1273	{
1274	/*
1275	* Flush dcache (required?), protect against fork and _really_ pin the page
1276	* table entries. get_user_pages() will protect against swapping out the
1277	* pages but it will NOT protect against removing page table entries. This
1278	* can be achieved with
1279	* - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1280	* an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1281	* Usual Linux distributions support only a limited size of locked pages
1282	* (e.g. 32KB).
1283	* - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1284	* or by
1285	* - setting the VM_LOCKED flag. This is the same as doing mlock() without
1286	* a range check.
1287	*/
1288	/** @todo The Linux fork() protection will require more work if this API
1289	* is to be used for anything but locking VM pages. */
1290	while (rc-- > 0)
1291	{
1292	flush_dcache_page(pMemLnx->apPages[rc]);
1293	papVMAs[rc]->vm_flags \|= VM_DONTCOPY \| VM_LOCKED;
1294	}
1295
1296	LNX_MM_UP_READ(pTask->mm);
1297
1298	RTMemFree(papVMAs);
1299
1300	pMemLnx->Core.u.Lock.R0Process = R0Process;
1301	pMemLnx->cPages = cPages;
1302	Assert(!pMemLnx->fMappedToRing0);
1303	*ppMem = &pMemLnx->Core;
1304
1305	IPRT_LINUX_RESTORE_EFL_AC();
1306	return VINF_SUCCESS;
1307	}
1308
1309	/*
1310	* Failed - we need to unlock any pages that we succeeded to lock.
1311	*/
1312	while (rc-- > 0)
1313	{
1314	if (!PageReserved(pMemLnx->apPages[rc]))
1315	SetPageDirty(pMemLnx->apPages[rc]);
1316	#if RTLNX_VER_MIN(4,6,0)
1317	put_page(pMemLnx->apPages[rc]);
1318	#else
1319	page_cache_release(pMemLnx->apPages[rc]);
1320	#endif
1321	}
1322
1323	LNX_MM_UP_READ(pTask->mm);
1324
1325	RTMemFree(papVMAs);
1326	rc = VERR_LOCK_FAILED;
1327	}
1328
1329	rtR0MemObjDelete(&pMemLnx->Core);
1330	IPRT_LINUX_RESTORE_EFL_AC();
1331	return rc;
1332	}
1333
1334
1335	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1336	{
1337	IPRT_LINUX_SAVE_EFL_AC();
1338	void pvLast = (uint8_t )pv + cb - 1;
1339	size_t const cPages = cb >> PAGE_SHIFT;
1340	PRTR0MEMOBJLNX pMemLnx;
1341	bool fLinearMapping;
1342	int rc;
1343	uint8_t *pbPage;
1344	size_t iPage;
1345	NOREF(fAccess);
1346
1347	if ( !RTR0MemKernelIsValidAddr(pv)
1348	\|\| !RTR0MemKernelIsValidAddr(pv + cb))
1349	return VERR_INVALID_PARAMETER;
1350
1351	/*
1352	* The lower part of the kernel memory has a linear mapping between
1353	* physical and virtual addresses. So we take a short cut here. This is
1354	* assumed to be the cleanest way to handle those addresses (and the code
1355	* is well tested, though the test for determining it is not very nice).
1356	* If we ever decide it isn't we can still remove it.
1357	*/
1358	#if 0
1359	fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1360	#else
1361	fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1362	&& (unsigned long)pvLast < (unsigned long)high_memory;
1363	#endif
1364
1365	/*
1366	* Allocate the memory object.
1367	*/
1368	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, pv, cb);
1369	if (!pMemLnx)
1370	{
1371	IPRT_LINUX_RESTORE_EFL_AC();
1372	return VERR_NO_MEMORY;
1373	}
1374
1375	/*
1376	* Gather the pages.
1377	* We ASSUME all kernel pages are non-swappable and non-movable.
1378	*/
1379	rc = VINF_SUCCESS;
1380	pbPage = (uint8_t *)pvLast;
1381	iPage = cPages;
1382	if (!fLinearMapping)
1383	{
1384	while (iPage-- > 0)
1385	{
1386	struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1387	if (RT_UNLIKELY(!pPage))
1388	{
1389	rc = VERR_LOCK_FAILED;
1390	break;
1391	}
1392	pMemLnx->apPages[iPage] = pPage;
1393	pbPage -= PAGE_SIZE;
1394	}
1395	}
1396	else
1397	{
1398	while (iPage-- > 0)
1399	{
1400	pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1401	pbPage -= PAGE_SIZE;
1402	}
1403	}
1404	if (RT_SUCCESS(rc))
1405	{
1406	/*
1407	* Complete the memory object and return.
1408	*/
1409	pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1410	pMemLnx->cPages = cPages;
1411	Assert(!pMemLnx->fMappedToRing0);
1412	*ppMem = &pMemLnx->Core;
1413
1414	IPRT_LINUX_RESTORE_EFL_AC();
1415	return VINF_SUCCESS;
1416	}
1417
1418	rtR0MemObjDelete(&pMemLnx->Core);
1419	IPRT_LINUX_RESTORE_EFL_AC();
1420	return rc;
1421	}
1422
1423
1424	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1425	{
1426	#if RTLNX_VER_MIN(2,4,22)
1427	IPRT_LINUX_SAVE_EFL_AC();
1428	const size_t cPages = cb >> PAGE_SHIFT;
1429	struct page *pDummyPage;
1430	struct page **papPages;
1431
1432	/* check for unsupported stuff. */
1433	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1434	if (uAlignment > PAGE_SIZE)
1435	return VERR_NOT_SUPPORTED;
1436
1437	/*
1438	* Allocate a dummy page and create a page pointer array for vmap such that
1439	* the dummy page is mapped all over the reserved area.
1440	*/
1441	pDummyPage = alloc_page(GFP_HIGHUSER \| __GFP_NOWARN);
1442	if (pDummyPage)
1443	{
1444	papPages = RTMemAlloc(sizeof(papPages) cPages);
1445	if (papPages)
1446	{
1447	void *pv;
1448	size_t iPage = cPages;
1449	while (iPage-- > 0)
1450	papPages[iPage] = pDummyPage;
1451	# ifdef VM_MAP
1452	pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1453	# else
1454	pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1455	# endif
1456	RTMemFree(papPages);
1457	if (pv)
1458	{
1459	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1460	if (pMemLnx)
1461	{
1462	pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1463	pMemLnx->cPages = 1;
1464	pMemLnx->apPages[0] = pDummyPage;
1465	*ppMem = &pMemLnx->Core;
1466	IPRT_LINUX_RESTORE_EFL_AC();
1467	return VINF_SUCCESS;
1468	}
1469	vunmap(pv);
1470	}
1471	}
1472	__free_page(pDummyPage);
1473	}
1474	IPRT_LINUX_RESTORE_EFL_AC();
1475	return VERR_NO_MEMORY;
1476
1477	#else /* < 2.4.22 */
1478	/*
1479	* Could probably use ioremap here, but the caller is in a better position than us
1480	* to select some safe physical memory.
1481	*/
1482	return VERR_NOT_SUPPORTED;
1483	#endif
1484	}
1485
1486
1487	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1488	{
1489	IPRT_LINUX_SAVE_EFL_AC();
1490	PRTR0MEMOBJLNX pMemLnx;
1491	void *pv;
1492	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1493	if (!pTask)
1494	return VERR_NOT_SUPPORTED;
1495
1496	/*
1497	* Check that the specified alignment is supported.
1498	*/
1499	if (uAlignment > PAGE_SIZE)
1500	return VERR_NOT_SUPPORTED;
1501
1502	/*
1503	* Let rtR0MemObjLinuxDoMmap do the difficult bits.
1504	*/
1505	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1506	if (pv == (void *)-1)
1507	{
1508	IPRT_LINUX_RESTORE_EFL_AC();
1509	return VERR_NO_MEMORY;
1510	}
1511
1512	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1513	if (!pMemLnx)
1514	{
1515	rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1516	IPRT_LINUX_RESTORE_EFL_AC();
1517	return VERR_NO_MEMORY;
1518	}
1519
1520	pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1521	*ppMem = &pMemLnx->Core;
1522	IPRT_LINUX_RESTORE_EFL_AC();
1523	return VINF_SUCCESS;
1524	}
1525
1526
1527	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap,
1528	void *pvFixed, size_t uAlignment,
1529	unsigned fProt, size_t offSub, size_t cbSub)
1530	{
1531	int rc = VERR_NO_MEMORY;
1532	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1533	PRTR0MEMOBJLNX pMemLnx;
1534	IPRT_LINUX_SAVE_EFL_AC();
1535
1536	/* Fail if requested to do something we can't. */
1537	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1538	if (uAlignment > PAGE_SIZE)
1539	return VERR_NOT_SUPPORTED;
1540
1541	/*
1542	* Create the IPRT memory object.
1543	*/
1544	if (!cbSub)
1545	cbSub = pMemLnxToMap->Core.cb - offSub;
1546	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1547	if (pMemLnx)
1548	{
1549	if (pMemLnxToMap->cPages)
1550	{
1551	#if RTLNX_VER_MIN(2,4,22)
1552	/*
1553	* Use vmap - 2.4.22 and later.
1554	*/
1555	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1556	/** @todo We don't really care too much for EXEC here... 5.8 always adds NX. */
1557	Assert(((offSub + cbSub) >> PAGE_SHIFT) <= pMemLnxToMap->cPages);
1558	# ifdef VM_MAP
1559	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_MAP, fPg);
1560	# else
1561	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_ALLOC, fPg);
1562	# endif
1563	if (pMemLnx->Core.pv)
1564	{
1565	pMemLnx->fMappedToRing0 = true;
1566	rc = VINF_SUCCESS;
1567	}
1568	else
1569	rc = VERR_MAP_FAILED;
1570
1571	#else /* < 2.4.22 */
1572	/*
1573	* Only option here is to share mappings if possible and forget about fProt.
1574	*/
1575	if (rtR0MemObjIsRing3(pMemToMap))
1576	rc = VERR_NOT_SUPPORTED;
1577	else
1578	{
1579	rc = VINF_SUCCESS;
1580	if (!pMemLnxToMap->Core.pv)
1581	rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1582	if (RT_SUCCESS(rc))
1583	{
1584	Assert(pMemLnxToMap->Core.pv);
1585	pMemLnx->Core.pv = (uint8_t *)pMemLnxToMap->Core.pv + offSub;
1586	}
1587	}
1588	#endif
1589	}
1590	else
1591	{
1592	/*
1593	* MMIO / physical memory.
1594	*/
1595	Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1596	#if RTLNX_VER_MIN(2,6,25)
1597	/*
1598	* ioremap() defaults to no caching since the 2.6 kernels.
1599	* ioremap_nocache() has been removed finally in 5.6-rc1.
1600	*/
1601	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1602	? ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1603	: ioremap_cache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1604	#else /* KERNEL_VERSION < 2.6.25 */
1605	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1606	? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1607	: ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1608	#endif /* KERNEL_VERSION < 2.6.25 */
1609	if (pMemLnx->Core.pv)
1610	{
1611	/** @todo fix protection. */
1612	rc = VINF_SUCCESS;
1613	}
1614	}
1615	if (RT_SUCCESS(rc))
1616	{
1617	pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1618	*ppMem = &pMemLnx->Core;
1619	IPRT_LINUX_RESTORE_EFL_AC();
1620	return VINF_SUCCESS;
1621	}
1622	rtR0MemObjDelete(&pMemLnx->Core);
1623	}
1624
1625	IPRT_LINUX_RESTORE_EFL_AC();
1626	return rc;
1627	}
1628
1629
1630	#ifdef VBOX_USE_PAE_HACK
1631	/**
1632	* Replace the PFN of a PTE with the address of the actual page.
1633	*
1634	* The caller maps a reserved dummy page at the address with the desired access
1635	* and flags.
1636	*
1637	* This hack is required for older Linux kernels which don't provide
1638	* remap_pfn_range().
1639	*
1640	* @returns 0 on success, -ENOMEM on failure.
1641	* @param mm The memory context.
1642	* @param ulAddr The mapping address.
1643	* @param Phys The physical address of the page to map.
1644	*/
1645	static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1646	{
1647	int rc = -ENOMEM;
1648	pgd_t *pgd;
1649
1650	spin_lock(&mm->page_table_lock);
1651
1652	pgd = pgd_offset(mm, ulAddr);
1653	if (!pgd_none(pgd) && !pgd_bad(pgd))
1654	{
1655	pmd_t *pmd = pmd_offset(pgd, ulAddr);
1656	if (!pmd_none(*pmd))
1657	{
1658	pte_t *ptep = pte_offset_map(pmd, ulAddr);
1659	if (ptep)
1660	{
1661	pte_t pte = *ptep;
1662	pte.pte_high &= 0xfff00000;
1663	pte.pte_high \|= ((Phys >> 32) & 0x000fffff);
1664	pte.pte_low &= 0x00000fff;
1665	pte.pte_low \|= (Phys & 0xfffff000);
1666	set_pte(ptep, pte);
1667	pte_unmap(ptep);
1668	rc = 0;
1669	}
1670	}
1671	}
1672
1673	spin_unlock(&mm->page_table_lock);
1674	return rc;
1675	}
1676	#endif /* VBOX_USE_PAE_HACK */
1677
1678
1679	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1680	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub)
1681	{
1682	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1683	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1684	int rc = VERR_NO_MEMORY;
1685	PRTR0MEMOBJLNX pMemLnx;
1686	#ifdef VBOX_USE_PAE_HACK
1687	struct page *pDummyPage;
1688	RTHCPHYS DummyPhys;
1689	#endif
1690	IPRT_LINUX_SAVE_EFL_AC();
1691
1692	/*
1693	* Check for restrictions.
1694	*/
1695	if (!pTask)
1696	return VERR_NOT_SUPPORTED;
1697	if (uAlignment > PAGE_SIZE)
1698	return VERR_NOT_SUPPORTED;
1699
1700	#ifdef VBOX_USE_PAE_HACK
1701	/*
1702	* Allocate a dummy page for use when mapping the memory.
1703	*/
1704	pDummyPage = alloc_page(GFP_USER \| __GFP_NOWARN);
1705	if (!pDummyPage)
1706	{
1707	IPRT_LINUX_RESTORE_EFL_AC();
1708	return VERR_NO_MEMORY;
1709	}
1710	SetPageReserved(pDummyPage);
1711	DummyPhys = page_to_phys(pDummyPage);
1712	#endif
1713
1714	/*
1715	* Create the IPRT memory object.
1716	*/
1717	Assert(!offSub \|\| cbSub);
1718	if (cbSub == 0)
1719	cbSub = pMemLnxToMap->Core.cb;
1720	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1721	if (pMemLnx)
1722	{
1723	/*
1724	* Allocate user space mapping.
1725	*/
1726	void *pv;
1727	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cbSub, uAlignment, pTask, fProt);
1728	if (pv != (void *)-1)
1729	{
1730	/*
1731	* Map page by page into the mmap area.
1732	* This is generic, paranoid and not very efficient.
1733	*/
1734	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1735	unsigned long ulAddrCur = (unsigned long)pv;
1736	const size_t cPages = (offSub + cbSub) >> PAGE_SHIFT;
1737	size_t iPage;
1738
1739	LNX_MM_DOWN_WRITE(pTask->mm);
1740
1741	rc = VINF_SUCCESS;
1742	if (pMemLnxToMap->cPages)
1743	{
1744	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1745	{
1746	#if RTLNX_VER_MAX(2,6,11)
1747	RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1748	#endif
1749	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1750	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1751	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1752	#endif
1753	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1754	/* remap_page_range() limitation on x86 */
1755	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1756	#endif
1757
1758	#if defined(VBOX_USE_INSERT_PAGE) && RTLNX_VER_MIN(2,6,22)
1759	rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1760	/* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1761	* See remap_pfn_range() in mm/memory.c */
1762	#if RTLNX_VER_MIN(3,7,0)
1763	vma->vm_flags \|= VM_DONTEXPAND \| VM_DONTDUMP;
1764	#else
1765	vma->vm_flags \|= VM_RESERVED;
1766	#endif
1767	#elif RTLNX_VER_MIN(2,6,11)
1768	rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1769	#elif defined(VBOX_USE_PAE_HACK)
1770	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1771	if (!rc)
1772	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1773	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1774	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1775	#else /* 2.4 */
1776	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1777	#endif
1778	if (rc)
1779	{
1780	rc = VERR_NO_MEMORY;
1781	break;
1782	}
1783	}
1784	}
1785	else
1786	{
1787	RTHCPHYS Phys;
1788	if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
1789	Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
1790	else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
1791	Phys = pMemLnxToMap->Core.u.Cont.Phys;
1792	else
1793	{
1794	AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
1795	Phys = NIL_RTHCPHYS;
1796	}
1797	if (Phys != NIL_RTHCPHYS)
1798	{
1799	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
1800	{
1801	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1802	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1803	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1804	#endif
1805	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1806	/* remap_page_range() limitation on x86 */
1807	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1808	#endif
1809
1810	#if RTLNX_VER_MIN(2,6,11)
1811	rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1812	#elif defined(VBOX_USE_PAE_HACK)
1813	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1814	if (!rc)
1815	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1816	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1817	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1818	#else /* 2.4 */
1819	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1820	#endif
1821	if (rc)
1822	{
1823	rc = VERR_NO_MEMORY;
1824	break;
1825	}
1826	}
1827	}
1828	}
1829
1830	#ifdef CONFIG_NUMA_BALANCING
1831	# if RTLNX_VER_MAX(3,13,0) && RTLNX_RHEL_MAX(7,0)
1832	# define VBOX_NUMA_HACK_OLD
1833	# endif
1834	if (RT_SUCCESS(rc))
1835	{
1836	/** @todo Ugly hack! But right now we have no other means to
1837	* disable automatic NUMA page balancing. */
1838	# ifdef RT_OS_X86
1839	# ifdef VBOX_NUMA_HACK_OLD
1840	pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
1841	# endif
1842	pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
1843	# else
1844	# ifdef VBOX_NUMA_HACK_OLD
1845	pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
1846	# endif
1847	pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
1848	# endif
1849	}
1850	#endif /* CONFIG_NUMA_BALANCING */
1851
1852	LNX_MM_UP_WRITE(pTask->mm);
1853
1854	if (RT_SUCCESS(rc))
1855	{
1856	#ifdef VBOX_USE_PAE_HACK
1857	__free_page(pDummyPage);
1858	#endif
1859	pMemLnx->Core.pv = pv;
1860	pMemLnx->Core.u.Mapping.R0Process = R0Process;
1861	*ppMem = &pMemLnx->Core;
1862	IPRT_LINUX_RESTORE_EFL_AC();
1863	return VINF_SUCCESS;
1864	}
1865
1866	/*
1867	* Bail out.
1868	*/
1869	rtR0MemObjLinuxDoMunmap(pv, cbSub, pTask);
1870	}
1871	rtR0MemObjDelete(&pMemLnx->Core);
1872	}
1873	#ifdef VBOX_USE_PAE_HACK
1874	__free_page(pDummyPage);
1875	#endif
1876
1877	IPRT_LINUX_RESTORE_EFL_AC();
1878	return rc;
1879	}
1880
1881
1882	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1883	{
1884	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
1885	/*
1886	* Currently only supported when we've got addresses PTEs from the kernel.
1887	*/
1888	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1889	if (pMemLnx->pArea && pMemLnx->papPtesForArea)
1890	{
1891	pgprot_t const fPg = rtR0MemObjLinuxConvertProt(fProt, true /fKernel/);
1892	size_t const cPages = (offSub + cbSub) >> PAGE_SHIFT;
1893	pte_t **papPtes = pMemLnx->papPtesForArea;
1894	size_t i;
1895
1896	for (i = offSub >> PAGE_SHIFT; i < cPages; i++)
1897	{
1898	set_pte(papPtes[i], mk_pte(pMemLnx->apPages[i], fPg));
1899	}
1900	preempt_disable();
1901	__flush_tlb_all();
1902	preempt_enable();
1903	return VINF_SUCCESS;
1904	}
1905	# elif defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
1906	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1907	if ( pMemLnx->fExecutable
1908	&& pMemLnx->fMappedToRing0)
1909	{
1910	LNXAPPLYPGRANGE Args;
1911	Args.pMemLnx = pMemLnx;
1912	Args.fPg = rtR0MemObjLinuxConvertProt(fProt, true /fKernel/);
1913	int rcLnx = apply_to_page_range(current->active_mm, (unsigned long)pMemLnx->Core.pv + offSub, cbSub,
1914	rtR0MemObjLinuxApplyPageRange, (void *)&Args);
1915	if (rcLnx)
1916	return VERR_NOT_SUPPORTED;
1917
1918	return VINF_SUCCESS;
1919	}
1920	# endif
1921
1922	NOREF(pMem);
1923	NOREF(offSub);
1924	NOREF(cbSub);
1925	NOREF(fProt);
1926	return VERR_NOT_SUPPORTED;
1927	}
1928
1929
1930	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1931	{
1932	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1933
1934	if (pMemLnx->cPages)
1935	return page_to_phys(pMemLnx->apPages[iPage]);
1936
1937	switch (pMemLnx->Core.enmType)
1938	{
1939	case RTR0MEMOBJTYPE_CONT:
1940	return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
1941
1942	case RTR0MEMOBJTYPE_PHYS:
1943	return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
1944
1945	/* the parent knows */
1946	case RTR0MEMOBJTYPE_MAPPING:
1947	return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
1948
1949	/* cPages > 0 */
1950	case RTR0MEMOBJTYPE_LOW:
1951	case RTR0MEMOBJTYPE_LOCK:
1952	case RTR0MEMOBJTYPE_PHYS_NC:
1953	case RTR0MEMOBJTYPE_PAGE:
1954	default:
1955	AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
1956	/* fall thru */
1957
1958	case RTR0MEMOBJTYPE_RES_VIRT:
1959	return NIL_RTHCPHYS;
1960	}
1961	}
1962

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c@ 87074

Download in other formats: