memobj-r0drv-linux.c@ 91479

Last change on this file since 91479 was 91479, checked in by vboxsync, 3 years ago
IPRT/memobj: Passing pszTag around...
Property svn:eol-style set to `native` Property svn:keywords set to `Id Rev Revision`
File size: 66.4 KB

Line
1	/* $Id: memobj-r0drv-linux.c 91479 2021-09-29 23:43:57Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-linux-kernel.h"
32
33	#include <iprt/memobj.h>
34	#include <iprt/assert.h>
35	#include <iprt/err.h>
36	#include <iprt/log.h>
37	#include <iprt/mem.h>
38	#include <iprt/process.h>
39	#include <iprt/string.h>
40	#include "internal/memobj.h"
41	#include "internal/iprt.h"
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	/* early 2.6 kernels */
48	#ifndef PAGE_SHARED_EXEC
49	# define PAGE_SHARED_EXEC PAGE_SHARED
50	#endif
51	#ifndef PAGE_READONLY_EXEC
52	# define PAGE_READONLY_EXEC PAGE_READONLY
53	#endif
54
55	/** @def IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
56	* Whether we use alloc_vm_area (3.2+) for executable memory.
57	* This is a must for 5.8+, but we enable it all the way back to 3.2.x for
58	* better W^R compliance (fExecutable flag). */
59	#if RTLNX_VER_RANGE(3,2,0, 5,10,0) \|\| defined(DOXYGEN_RUNNING)
60	# define IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
61	#endif
62	/** @def IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
63	* alloc_vm_area was removed with 5.10 so we have to resort to a different way
64	* to allocate executable memory.
65	* It would be possible to remove IPRT_USE_ALLOC_VM_AREA_FOR_EXEC and use
66	* this path execlusively for 3.2+ but no time to test it really works on every
67	* supported kernel, so better play safe for now.
68	*/
69	#if RTLNX_VER_MIN(5,10,0) \|\| defined(DOXYGEN_RUNNING)
70	# define IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
71	#endif
72
73	/*
74	* 2.6.29+ kernels don't work with remap_pfn_range() anymore because
75	* track_pfn_vma_new() is apparently not defined for non-RAM pages.
76	* It should be safe to use vm_insert_page() older kernels as well.
77	*/
78	#if RTLNX_VER_MIN(2,6,23)
79	# define VBOX_USE_INSERT_PAGE
80	#endif
81	#if defined(CONFIG_X86_PAE) \
82	&& ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
83	\|\| RTLNX_VER_RANGE(2,6,0, 2,6,11) )
84	# define VBOX_USE_PAE_HACK
85	#endif
86
87	/* gfp_t was introduced in 2.6.14, define it for earlier. */
88	#if RTLNX_VER_MAX(2,6,14)
89	# define gfp_t unsigned
90	#endif
91
92	/*
93	* Wrappers around mmap_lock/mmap_sem difference.
94	*/
95	#if RTLNX_VER_MIN(5,8,0)
96	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_lock)
97	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_lock)
98	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_lock)
99	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_lock)
100	#else
101	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_sem)
102	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_sem)
103	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_sem)
104	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_sem)
105	#endif
106
107
108	/*********************************************************************************************************************************
109	* Structures and Typedefs *
110	*********************************************************************************************************************************/
111	/**
112	* The Linux version of the memory object structure.
113	*/
114	typedef struct RTR0MEMOBJLNX
115	{
116	/** The core structure. */
117	RTR0MEMOBJINTERNAL Core;
118	/** Set if the allocation is contiguous.
119	* This means it has to be given back as one chunk. */
120	bool fContiguous;
121	/** Set if executable allocation. */
122	bool fExecutable;
123	/** Set if we've vmap'ed the memory into ring-0. */
124	bool fMappedToRing0;
125	#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
126	/** Return from alloc_vm_area() that we now need to use for executable
127	* memory. */
128	struct vm_struct *pArea;
129	/** PTE array that goes along with pArea (must be freed). */
130	pte_t **papPtesForArea;
131	#endif
132	/** The pages in the apPages array. */
133	size_t cPages;
134	/** Array of struct page pointers. (variable size) */
135	struct page *apPages[1];
136	} RTR0MEMOBJLNX;
137	/** Pointer to the linux memory object. */
138	typedef RTR0MEMOBJLNX *PRTR0MEMOBJLNX;
139
140
141	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
142
143
144	/**
145	* Helper that converts from a RTR0PROCESS handle to a linux task.
146	*
147	* @returns The corresponding Linux task.
148	* @param R0Process IPRT ring-0 process handle.
149	*/
150	static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
151	{
152	/** @todo fix rtR0ProcessToLinuxTask!! */
153	/** @todo many (all?) callers currently assume that we return 'current'! */
154	return R0Process == RTR0ProcHandleSelf() ? current : NULL;
155	}
156
157
158	/**
159	* Compute order. Some functions allocate 2^order pages.
160	*
161	* @returns order.
162	* @param cPages Number of pages.
163	*/
164	static int rtR0MemObjLinuxOrder(size_t cPages)
165	{
166	int iOrder;
167	size_t cTmp;
168
169	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
170	;
171	if (cPages & ~((size_t)1 << iOrder))
172	++iOrder;
173
174	return iOrder;
175	}
176
177
178	/**
179	* Converts from RTMEM_PROT_* to Linux PAGE_*.
180	*
181	* @returns Linux page protection constant.
182	* @param fProt The IPRT protection mask.
183	* @param fKernel Whether it applies to kernel or user space.
184	*/
185	static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
186	{
187	switch (fProt)
188	{
189	default:
190	AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
191	case RTMEM_PROT_NONE:
192	return PAGE_NONE;
193
194	case RTMEM_PROT_READ:
195	return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
196
197	case RTMEM_PROT_WRITE:
198	case RTMEM_PROT_WRITE \| RTMEM_PROT_READ:
199	return fKernel ? PAGE_KERNEL : PAGE_SHARED;
200
201	case RTMEM_PROT_EXEC:
202	case RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
203	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
204	if (fKernel)
205	{
206	pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
207	pgprot_val(fPg) &= ~_PAGE_RW;
208	return fPg;
209	}
210	return PAGE_READONLY_EXEC;
211	#else
212	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
213	#endif
214
215	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
216	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
217	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
218	}
219	}
220
221
222	/**
223	* Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
224	* an empty user space mapping.
225	*
226	* We acquire the mmap_sem/mmap_lock of the task!
227	*
228	* @returns Pointer to the mapping.
229	* (void *)-1 on failure.
230	* @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
231	* @param cb The size of the mapping.
232	* @param uAlignment The alignment of the mapping.
233	* @param pTask The Linux task to create this mapping in.
234	* @param fProt The RTMEM_PROT_* mask.
235	*/
236	static void rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct pTask, unsigned fProt)
237	{
238	unsigned fLnxProt;
239	unsigned long ulAddr;
240
241	Assert(pTask == current); /* do_mmap */
242	RT_NOREF_PV(pTask);
243
244	/*
245	* Convert from IPRT protection to mman.h PROT_ and call do_mmap.
246	*/
247	fProt &= (RTMEM_PROT_NONE \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC);
248	if (fProt == RTMEM_PROT_NONE)
249	fLnxProt = PROT_NONE;
250	else
251	{
252	fLnxProt = 0;
253	if (fProt & RTMEM_PROT_READ)
254	fLnxProt \|= PROT_READ;
255	if (fProt & RTMEM_PROT_WRITE)
256	fLnxProt \|= PROT_WRITE;
257	if (fProt & RTMEM_PROT_EXEC)
258	fLnxProt \|= PROT_EXEC;
259	}
260
261	if (R3PtrFixed != (RTR3PTR)-1)
262	{
263	#if RTLNX_VER_MIN(3,5,0)
264	ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
265	#else
266	LNX_MM_DOWN_WRITE(pTask->mm);
267	ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
268	LNX_MM_UP_WRITE(pTask->mm);
269	#endif
270	}
271	else
272	{
273	#if RTLNX_VER_MIN(3,5,0)
274	ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
275	#else
276	LNX_MM_DOWN_WRITE(pTask->mm);
277	ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
278	LNX_MM_UP_WRITE(pTask->mm);
279	#endif
280	if ( !(ulAddr & ~PAGE_MASK)
281	&& (ulAddr & (uAlignment - 1)))
282	{
283	/** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
284	* up alignment gaps. This is of course complicated by fragmentation (which we might have cause
285	* ourselves) and further by there begin two mmap strategies (top / bottom). */
286	/* For now, just ignore uAlignment requirements... */
287	}
288	}
289
290
291	if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
292	return (void *)-1;
293	return (void *)ulAddr;
294	}
295
296
297	/**
298	* Worker that destroys a user space mapping.
299	* Undoes what rtR0MemObjLinuxDoMmap did.
300	*
301	* We acquire the mmap_sem/mmap_lock of the task!
302	*
303	* @param pv The ring-3 mapping.
304	* @param cb The size of the mapping.
305	* @param pTask The Linux task to destroy this mapping in.
306	*/
307	static void rtR0MemObjLinuxDoMunmap(void pv, size_t cb, struct task_struct pTask)
308	{
309	#if RTLNX_VER_MIN(3,5,0)
310	Assert(pTask == current); RT_NOREF_PV(pTask);
311	vm_munmap((unsigned long)pv, cb);
312	#elif defined(USE_RHEL4_MUNMAP)
313	LNX_MM_DOWN_WRITE(pTask->mm);
314	do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
315	LNX_MM_UP_WRITE(pTask->mm);
316	#else
317	LNX_MM_DOWN_WRITE(pTask->mm);
318	do_munmap(pTask->mm, (unsigned long)pv, cb);
319	LNX_MM_UP_WRITE(pTask->mm);
320	#endif
321	}
322
323
324	/**
325	* Internal worker that allocates physical pages and creates the memory object for them.
326	*
327	* @returns IPRT status code.
328	* @param ppMemLnx Where to store the memory object pointer.
329	* @param enmType The object type.
330	* @param cb The number of bytes to allocate.
331	* @param uAlignment The alignment of the physical memory.
332	* Only valid if fContiguous == true, ignored otherwise.
333	* @param fFlagsLnx The page allocation flags (GPFs).
334	* @param fContiguous Whether the allocation must be contiguous.
335	* @param fExecutable Whether the memory must be executable.
336	* @param rcNoMem What to return when we're out of pages.
337	*/
338	static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
339	size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, bool fExecutable, int rcNoMem)
340	{
341	size_t iPage;
342	size_t const cPages = cb >> PAGE_SHIFT;
343	struct page *paPages;
344
345	/*
346	* Allocate a memory object structure that's large enough to contain
347	* the page pointer array.
348	*/
349	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType,
350	NULL, cb, NULL);
351	if (!pMemLnx)
352	return VERR_NO_MEMORY;
353	pMemLnx->cPages = cPages;
354
355	if (cPages > 255)
356	{
357	# ifdef __GFP_REPEAT
358	/* Try hard to allocate the memory, but the allocation attempt might fail. */
359	fFlagsLnx \|= __GFP_REPEAT;
360	# endif
361	# ifdef __GFP_NOMEMALLOC
362	/* Introduced with Linux 2.6.12: Don't use emergency reserves */
363	fFlagsLnx \|= __GFP_NOMEMALLOC;
364	# endif
365	}
366
367	/*
368	* Allocate the pages.
369	* For small allocations we'll try contiguous first and then fall back on page by page.
370	*/
371	#if RTLNX_VER_MIN(2,4,22)
372	if ( fContiguous
373	\|\| cb <= PAGE_SIZE * 2)
374	{
375	# ifdef VBOX_USE_INSERT_PAGE
376	paPages = alloc_pages(fFlagsLnx \| __GFP_COMP \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
377	# else
378	paPages = alloc_pages(fFlagsLnx \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
379	# endif
380	if (paPages)
381	{
382	fContiguous = true;
383	for (iPage = 0; iPage < cPages; iPage++)
384	pMemLnx->apPages[iPage] = &paPages[iPage];
385	}
386	else if (fContiguous)
387	{
388	rtR0MemObjDelete(&pMemLnx->Core);
389	return rcNoMem;
390	}
391	}
392
393	if (!fContiguous)
394	{
395	/** @todo Try use alloc_pages_bulk_array when available, it should be faster
396	* than a alloc_page loop. Put it in #ifdefs similar to
397	* IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC. */
398	for (iPage = 0; iPage < cPages; iPage++)
399	{
400	pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx \| __GFP_NOWARN);
401	if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
402	{
403	while (iPage-- > 0)
404	__free_page(pMemLnx->apPages[iPage]);
405	rtR0MemObjDelete(&pMemLnx->Core);
406	return rcNoMem;
407	}
408	}
409	}
410
411	#else /* < 2.4.22 */
412	/** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
413	paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
414	if (!paPages)
415	{
416	rtR0MemObjDelete(&pMemLnx->Core);
417	return rcNoMem;
418	}
419	for (iPage = 0; iPage < cPages; iPage++)
420	{
421	pMemLnx->apPages[iPage] = &paPages[iPage];
422	if (fExecutable)
423	MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
424	if (PageHighMem(pMemLnx->apPages[iPage]))
425	BUG();
426	}
427
428	fContiguous = true;
429	#endif /* < 2.4.22 */
430	pMemLnx->fContiguous = fContiguous;
431	pMemLnx->fExecutable = fExecutable;
432
433	#if RTLNX_VER_MAX(4,5,0)
434	/*
435	* Reserve the pages.
436	*
437	* Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
438	* pages. According to Michal Hocko this shouldn't be necessary anyway because
439	* as pages which are not on the LRU list are never evictable.
440	*/
441	for (iPage = 0; iPage < cPages; iPage++)
442	SetPageReserved(pMemLnx->apPages[iPage]);
443	#endif
444
445	/*
446	* Note that the physical address of memory allocated with alloc_pages(flags, order)
447	* is always 2^(PAGE_SHIFT+order)-aligned.
448	*/
449	if ( fContiguous
450	&& uAlignment > PAGE_SIZE)
451	{
452	/*
453	* Check for alignment constraints. The physical address of memory allocated with
454	* alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
455	*/
456	if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
457	{
458	/*
459	* This should never happen!
460	*/
461	printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
462	(unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
463	rtR0MemObjLinuxFreePages(pMemLnx);
464	return rcNoMem;
465	}
466	}
467
468	*ppMemLnx = pMemLnx;
469	return VINF_SUCCESS;
470	}
471
472
473	/**
474	* Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
475	*
476	* This method does NOT free the object.
477	*
478	* @param pMemLnx The object which physical pages should be freed.
479	*/
480	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
481	{
482	size_t iPage = pMemLnx->cPages;
483	if (iPage > 0)
484	{
485	/*
486	* Restore the page flags.
487	*/
488	while (iPage-- > 0)
489	{
490	#if RTLNX_VER_MAX(4,5,0)
491	/* See SetPageReserved() in rtR0MemObjLinuxAllocPages() */
492	ClearPageReserved(pMemLnx->apPages[iPage]);
493	#endif
494	#if RTLNX_VER_MAX(2,4,22)
495	if (pMemLnx->fExecutable)
496	MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
497	#endif
498	}
499
500	/*
501	* Free the pages.
502	*/
503	#if RTLNX_VER_MIN(2,4,22)
504	if (!pMemLnx->fContiguous)
505	{
506	iPage = pMemLnx->cPages;
507	while (iPage-- > 0)
508	__free_page(pMemLnx->apPages[iPage]);
509	}
510	else
511	#endif
512	__free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
513
514	pMemLnx->cPages = 0;
515	}
516	}
517
518
519	#ifdef IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
520	/**
521	* User data passed to the apply_to_page_range() callback.
522	*/
523	typedef struct LNXAPPLYPGRANGE
524	{
525	/** Pointer to the memory object. */
526	PRTR0MEMOBJLNX pMemLnx;
527	/** The page protection flags to apply. */
528	pgprot_t fPg;
529	} LNXAPPLYPGRANGE;
530	/** Pointer to the user data. */
531	typedef LNXAPPLYPGRANGE *PLNXAPPLYPGRANGE;
532	/** Pointer to the const user data. */
533	typedef const LNXAPPLYPGRANGE *PCLNXAPPLYPGRANGE;
534
535	/**
536	* Callback called in apply_to_page_range().
537	*
538	* @returns Linux status code.
539	* @param pPte Pointer to the page table entry for the given address.
540	* @param uAddr The address to apply the new protection to.
541	* @param pvUser The opaque user data.
542	*/
543	static int rtR0MemObjLinuxApplyPageRange(pte_t pPte, unsigned long uAddr, void pvUser)
544	{
545	PCLNXAPPLYPGRANGE pArgs = (PCLNXAPPLYPGRANGE)pvUser;
546	PRTR0MEMOBJLNX pMemLnx = pArgs->pMemLnx;
547	size_t idxPg = (uAddr - (unsigned long)pMemLnx->Core.pv) >> PAGE_SHIFT;
548
549	set_pte(pPte, mk_pte(pMemLnx->apPages[idxPg], pArgs->fPg));
550	return 0;
551	}
552	#endif
553
554
555	/**
556	* Maps the allocation into ring-0.
557	*
558	* This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
559	*
560	* Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
561	* space, so we'll use that mapping if possible. If execute access is required, we'll
562	* play safe and do our own mapping.
563	*
564	* @returns IPRT status code.
565	* @param pMemLnx The linux memory object to map.
566	* @param fExecutable Whether execute access is required.
567	*/
568	static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
569	{
570	int rc = VINF_SUCCESS;
571
572	/*
573	* Choose mapping strategy.
574	*/
575	bool fMustMap = fExecutable
576	\|\| !pMemLnx->fContiguous;
577	if (!fMustMap)
578	{
579	size_t iPage = pMemLnx->cPages;
580	while (iPage-- > 0)
581	if (PageHighMem(pMemLnx->apPages[iPage]))
582	{
583	fMustMap = true;
584	break;
585	}
586	}
587
588	Assert(!pMemLnx->Core.pv);
589	Assert(!pMemLnx->fMappedToRing0);
590
591	if (fMustMap)
592	{
593	/*
594	* Use vmap - 2.4.22 and later.
595	*/
596	#if RTLNX_VER_MIN(2,4,22)
597	pgprot_t fPg;
598	pgprot_val(fPg) = _PAGE_PRESENT \| _PAGE_RW;
599	# ifdef _PAGE_NX
600	if (!fExecutable)
601	pgprot_val(fPg) \|= _PAGE_NX;
602	# endif
603
604	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
605	if (fExecutable)
606	{
607	# if RTLNX_VER_MIN(3,2,51)
608	pte_t papPtes = (pte_t )kmalloc_array(pMemLnx->cPages, sizeof(papPtes[0]), GFP_KERNEL);
609	# else
610	pte_t papPtes = (pte_t )kmalloc(pMemLnx->cPages * sizeof(papPtes[0]), GFP_KERNEL);
611	# endif
612	if (papPtes)
613	{
614	pMemLnx->pArea = alloc_vm_area(pMemLnx->Core.cb, papPtes); /* Note! pArea->nr_pages is not set. */
615	if (pMemLnx->pArea)
616	{
617	size_t i;
618	Assert(pMemLnx->pArea->size >= pMemLnx->Core.cb); /* Note! includes guard page. */
619	Assert(pMemLnx->pArea->addr);
620	# ifdef _PAGE_NX
621	pgprot_val(fPg) \|= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
622	# endif
623	pMemLnx->papPtesForArea = papPtes;
624	for (i = 0; i < pMemLnx->cPages; i++)
625	*papPtes[i] = mk_pte(pMemLnx->apPages[i], fPg);
626	pMemLnx->Core.pv = pMemLnx->pArea->addr;
627	pMemLnx->fMappedToRing0 = true;
628	}
629	else
630	{
631	kfree(papPtes);
632	rc = VERR_MAP_FAILED;
633	}
634	}
635	else
636	rc = VERR_MAP_FAILED;
637	}
638	else
639	# endif
640	{
641	# if defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
642	if (fExecutable)
643	pgprot_val(fPg) \|= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
644	# endif
645
646	# ifdef VM_MAP
647	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
648	# else
649	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
650	# endif
651	if (pMemLnx->Core.pv)
652	pMemLnx->fMappedToRing0 = true;
653	else
654	rc = VERR_MAP_FAILED;
655	}
656	#else /* < 2.4.22 */
657	rc = VERR_NOT_SUPPORTED;
658	#endif
659	}
660	else
661	{
662	/*
663	* Use the kernel RAM mapping.
664	*/
665	pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
666	Assert(pMemLnx->Core.pv);
667	}
668
669	return rc;
670	}
671
672
673	/**
674	* Undoes what rtR0MemObjLinuxVMap() did.
675	*
676	* @param pMemLnx The linux memory object.
677	*/
678	static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
679	{
680	#if RTLNX_VER_MIN(2,4,22)
681	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
682	if (pMemLnx->pArea)
683	{
684	# if 0
685	pte_t **papPtes = pMemLnx->papPtesForArea;
686	size_t i;
687	for (i = 0; i < pMemLnx->cPages; i++)
688	*papPtes[i] = 0;
689	# endif
690	free_vm_area(pMemLnx->pArea);
691	kfree(pMemLnx->papPtesForArea);
692	pMemLnx->pArea = NULL;
693	pMemLnx->papPtesForArea = NULL;
694	}
695	else
696	# endif
697	if (pMemLnx->fMappedToRing0)
698	{
699	Assert(pMemLnx->Core.pv);
700	vunmap(pMemLnx->Core.pv);
701	pMemLnx->fMappedToRing0 = false;
702	}
703	#else /* < 2.4.22 */
704	Assert(!pMemLnx->fMappedToRing0);
705	#endif
706	pMemLnx->Core.pv = NULL;
707	}
708
709
710	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
711	{
712	IPRT_LINUX_SAVE_EFL_AC();
713	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
714
715	/*
716	* Release any memory that we've allocated or locked.
717	*/
718	switch (pMemLnx->Core.enmType)
719	{
720	case RTR0MEMOBJTYPE_LOW:
721	case RTR0MEMOBJTYPE_PAGE:
722	case RTR0MEMOBJTYPE_CONT:
723	case RTR0MEMOBJTYPE_PHYS:
724	case RTR0MEMOBJTYPE_PHYS_NC:
725	rtR0MemObjLinuxVUnmap(pMemLnx);
726	rtR0MemObjLinuxFreePages(pMemLnx);
727	break;
728
729	case RTR0MEMOBJTYPE_LOCK:
730	if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
731	{
732	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
733	size_t iPage;
734	Assert(pTask);
735	if (pTask && pTask->mm)
736	LNX_MM_DOWN_READ(pTask->mm);
737
738	iPage = pMemLnx->cPages;
739	while (iPage-- > 0)
740	{
741	if (!PageReserved(pMemLnx->apPages[iPage]))
742	SetPageDirty(pMemLnx->apPages[iPage]);
743	#if RTLNX_VER_MIN(4,6,0)
744	put_page(pMemLnx->apPages[iPage]);
745	#else
746	page_cache_release(pMemLnx->apPages[iPage]);
747	#endif
748	}
749
750	if (pTask && pTask->mm)
751	LNX_MM_UP_READ(pTask->mm);
752	}
753	/* else: kernel memory - nothing to do here. */
754	break;
755
756	case RTR0MEMOBJTYPE_RES_VIRT:
757	Assert(pMemLnx->Core.pv);
758	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
759	{
760	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
761	Assert(pTask);
762	if (pTask && pTask->mm)
763	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
764	}
765	else
766	{
767	vunmap(pMemLnx->Core.pv);
768
769	Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
770	__free_page(pMemLnx->apPages[0]);
771	pMemLnx->apPages[0] = NULL;
772	pMemLnx->cPages = 0;
773	}
774	pMemLnx->Core.pv = NULL;
775	break;
776
777	case RTR0MEMOBJTYPE_MAPPING:
778	Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
779	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
780	{
781	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
782	Assert(pTask);
783	if (pTask && pTask->mm)
784	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
785	}
786	else
787	vunmap(pMemLnx->Core.pv);
788	pMemLnx->Core.pv = NULL;
789	break;
790
791	default:
792	AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
793	return VERR_INTERNAL_ERROR;
794	}
795	IPRT_LINUX_RESTORE_EFL_ONLY_AC();
796	return VINF_SUCCESS;
797	}
798
799
800	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
801	{
802	IPRT_LINUX_SAVE_EFL_AC();
803	PRTR0MEMOBJLNX pMemLnx;
804	int rc;
805
806	#if RTLNX_VER_MIN(2,4,22)
807	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
808	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
809	#else
810	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
811	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
812	#endif
813	if (RT_SUCCESS(rc))
814	{
815	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
816	if (RT_SUCCESS(rc))
817	{
818	*ppMem = &pMemLnx->Core;
819	IPRT_LINUX_RESTORE_EFL_AC();
820	return rc;
821	}
822
823	rtR0MemObjLinuxFreePages(pMemLnx);
824	rtR0MemObjDelete(&pMemLnx->Core);
825	}
826
827	IPRT_LINUX_RESTORE_EFL_AC();
828	return rc;
829	}
830
831
832	DECLHIDDEN(int) rtR0MemObjNativeAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags,
833	const char *pszTag)
834	{
835	return rtR0MemObjFallbackAllocLarge(ppMem, cb, cbLargePage, fFlags, pszTag);
836	}
837
838
839	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
840	{
841	IPRT_LINUX_SAVE_EFL_AC();
842	PRTR0MEMOBJLNX pMemLnx;
843	int rc;
844
845	/* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
846	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
847	/* ZONE_DMA32: 0-4GB */
848	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
849	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
850	if (RT_FAILURE(rc))
851	#endif
852	#ifdef RT_ARCH_AMD64
853	/* ZONE_DMA: 0-16MB */
854	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
855	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
856	#else
857	# ifdef CONFIG_X86_PAE
858	# endif
859	/* ZONE_NORMAL: 0-896MB */
860	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
861	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
862	#endif
863	if (RT_SUCCESS(rc))
864	{
865	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
866	if (RT_SUCCESS(rc))
867	{
868	*ppMem = &pMemLnx->Core;
869	IPRT_LINUX_RESTORE_EFL_AC();
870	return rc;
871	}
872
873	rtR0MemObjLinuxFreePages(pMemLnx);
874	rtR0MemObjDelete(&pMemLnx->Core);
875	}
876
877	IPRT_LINUX_RESTORE_EFL_AC();
878	return rc;
879	}
880
881
882	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
883	{
884	IPRT_LINUX_SAVE_EFL_AC();
885	PRTR0MEMOBJLNX pMemLnx;
886	int rc;
887
888	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
889	/* ZONE_DMA32: 0-4GB */
890	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
891	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
892	if (RT_FAILURE(rc))
893	#endif
894	#ifdef RT_ARCH_AMD64
895	/* ZONE_DMA: 0-16MB */
896	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
897	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
898	#else
899	/* ZONE_NORMAL (32-bit hosts): 0-896MB */
900	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
901	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
902	#endif
903	if (RT_SUCCESS(rc))
904	{
905	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
906	if (RT_SUCCESS(rc))
907	{
908	#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_HIGHMEM64G))
909	size_t iPage = pMemLnx->cPages;
910	while (iPage-- > 0)
911	Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
912	#endif
913	pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
914	*ppMem = &pMemLnx->Core;
915	IPRT_LINUX_RESTORE_EFL_AC();
916	return rc;
917	}
918
919	rtR0MemObjLinuxFreePages(pMemLnx);
920	rtR0MemObjDelete(&pMemLnx->Core);
921	}
922
923	IPRT_LINUX_RESTORE_EFL_AC();
924	return rc;
925	}
926
927
928	/**
929	* Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
930	*
931	* @returns IPRT status code.
932	* @param ppMemLnx Where to
933	* @param enmType The object type.
934	* @param cb The size of the allocation.
935	* @param uAlignment The alignment of the physical memory.
936	* Only valid for fContiguous == true, ignored otherwise.
937	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
938	* @param fGfp The Linux GFP flags to use for the allocation.
939	*/
940	static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
941	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, gfp_t fGfp)
942	{
943	PRTR0MEMOBJLNX pMemLnx;
944	int rc;
945
946	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
947	enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
948	false /fExecutable/, VERR_NO_PHYS_MEMORY);
949	if (RT_FAILURE(rc))
950	return rc;
951
952	/*
953	* Check the addresses if necessary. (Can be optimized a bit for PHYS.)
954	*/
955	if (PhysHighest != NIL_RTHCPHYS)
956	{
957	size_t iPage = pMemLnx->cPages;
958	while (iPage-- > 0)
959	if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
960	{
961	rtR0MemObjLinuxFreePages(pMemLnx);
962	rtR0MemObjDelete(&pMemLnx->Core);
963	return VERR_NO_MEMORY;
964	}
965	}
966
967	/*
968	* Complete the object.
969	*/
970	if (enmType == RTR0MEMOBJTYPE_PHYS)
971	{
972	pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
973	pMemLnx->Core.u.Phys.fAllocated = true;
974	}
975	*ppMem = &pMemLnx->Core;
976	return rc;
977	}
978
979
980	/**
981	* Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
982	*
983	* @returns IPRT status code.
984	* @param ppMem Where to store the memory object pointer on success.
985	* @param enmType The object type.
986	* @param cb The size of the allocation.
987	* @param uAlignment The alignment of the physical memory.
988	* Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
989	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
990	*/
991	static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
992	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest)
993	{
994	int rc;
995	IPRT_LINUX_SAVE_EFL_AC();
996
997	/*
998	* There are two clear cases and that's the <=16MB and anything-goes ones.
999	* When the physical address limit is somewhere in-between those two we'll
1000	* just have to try, starting with HIGHUSER and working our way thru the
1001	* different types, hoping we'll get lucky.
1002	*
1003	* We should probably move this physical address restriction logic up to
1004	* the page alloc function as it would be more efficient there. But since
1005	* we don't expect this to be a performance issue just yet it can wait.
1006	*/
1007	if (PhysHighest == NIL_RTHCPHYS)
1008	/* ZONE_HIGHMEM: the whole physical memory */
1009	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
1010	else if (PhysHighest <= _1M * 16)
1011	/* ZONE_DMA: 0-16MB */
1012	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
1013	else
1014	{
1015	rc = VERR_NO_MEMORY;
1016	if (RT_FAILURE(rc))
1017	/* ZONE_HIGHMEM: the whole physical memory */
1018	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
1019	if (RT_FAILURE(rc))
1020	/* ZONE_NORMAL: 0-896MB */
1021	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_USER);
1022	#ifdef GFP_DMA32
1023	if (RT_FAILURE(rc))
1024	/* ZONE_DMA32: 0-4GB */
1025	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA32);
1026	#endif
1027	if (RT_FAILURE(rc))
1028	/* ZONE_DMA: 0-16MB */
1029	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
1030	}
1031	IPRT_LINUX_RESTORE_EFL_AC();
1032	return rc;
1033	}
1034
1035
1036	/**
1037	* Translates a kernel virtual address to a linux page structure by walking the
1038	* page tables.
1039	*
1040	* @note We do assume that the page tables will not change as we are walking
1041	* them. This assumption is rather forced by the fact that I could not
1042	* immediately see any way of preventing this from happening. So, we
1043	* take some extra care when accessing them.
1044	*
1045	* Because of this, we don't want to use this function on memory where
1046	* attribute changes to nearby pages is likely to cause large pages to
1047	* be used or split up. So, don't use this for the linear mapping of
1048	* physical memory.
1049	*
1050	* @returns Pointer to the page structur or NULL if it could not be found.
1051	* @param pv The kernel virtual address.
1052	*/
1053	RTDECL(struct page ) rtR0MemObjLinuxVirtToPage(void pv)
1054	{
1055	unsigned long ulAddr = (unsigned long)pv;
1056	unsigned long pfn;
1057	struct page *pPage;
1058	pte_t *pEntry;
1059	union
1060	{
1061	pgd_t Global;
1062	#if RTLNX_VER_MIN(4,12,0)
1063	p4d_t Four;
1064	#endif
1065	#if RTLNX_VER_MIN(2,6,11)
1066	pud_t Upper;
1067	#endif
1068	pmd_t Middle;
1069	pte_t Entry;
1070	} u;
1071
1072	/* Should this happen in a situation this code will be called in? And if
1073	* so, can it change under our feet? See also
1074	* "Documentation/vm/active_mm.txt" in the kernel sources. */
1075	if (RT_UNLIKELY(!current->active_mm))
1076	return NULL;
1077	u.Global = *pgd_offset(current->active_mm, ulAddr);
1078	if (RT_UNLIKELY(pgd_none(u.Global)))
1079	return NULL;
1080	#if RTLNX_VER_MIN(2,6,11)
1081	# if RTLNX_VER_MIN(4,12,0)
1082	u.Four = *p4d_offset(&u.Global, ulAddr);
1083	if (RT_UNLIKELY(p4d_none(u.Four)))
1084	return NULL;
1085	if (p4d_large(u.Four))
1086	{
1087	pPage = p4d_page(u.Four);
1088	AssertReturn(pPage, NULL);
1089	pfn = page_to_pfn(pPage); /* doing the safe way... */
1090	AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
1091	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
1092	return pfn_to_page(pfn);
1093	}
1094	u.Upper = *pud_offset(&u.Four, ulAddr);
1095	# else /* < 4.12 */
1096	u.Upper = *pud_offset(&u.Global, ulAddr);
1097	# endif /* < 4.12 */
1098	if (RT_UNLIKELY(pud_none(u.Upper)))
1099	return NULL;
1100	# if RTLNX_VER_MIN(2,6,25)
1101	if (pud_large(u.Upper))
1102	{
1103	pPage = pud_page(u.Upper);
1104	AssertReturn(pPage, NULL);
1105	pfn = page_to_pfn(pPage); /* doing the safe way... */
1106	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
1107	return pfn_to_page(pfn);
1108	}
1109	# endif
1110	u.Middle = *pmd_offset(&u.Upper, ulAddr);
1111	#else /* < 2.6.11 */
1112	u.Middle = *pmd_offset(&u.Global, ulAddr);
1113	#endif /* < 2.6.11 */
1114	if (RT_UNLIKELY(pmd_none(u.Middle)))
1115	return NULL;
1116	#if RTLNX_VER_MIN(2,6,0)
1117	if (pmd_large(u.Middle))
1118	{
1119	pPage = pmd_page(u.Middle);
1120	AssertReturn(pPage, NULL);
1121	pfn = page_to_pfn(pPage); /* doing the safe way... */
1122	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
1123	return pfn_to_page(pfn);
1124	}
1125	#endif
1126
1127	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
1128	pEntry = pte_offset_map(&u.Middle, ulAddr);
1129	#else
1130	pEntry = pte_offset(&u.Middle, ulAddr);
1131	#endif
1132	if (RT_UNLIKELY(!pEntry))
1133	return NULL;
1134	u.Entry = *pEntry;
1135	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map)
1136	pte_unmap(pEntry);
1137	#endif
1138
1139	if (RT_UNLIKELY(!pte_present(u.Entry)))
1140	return NULL;
1141	return pte_page(u.Entry);
1142	}
1143	RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
1144
1145
1146	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
1147	{
1148	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest);
1149	}
1150
1151
1152	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
1153	{
1154	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest);
1155	}
1156
1157
1158	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
1159	{
1160	IPRT_LINUX_SAVE_EFL_AC();
1161
1162	/*
1163	* All we need to do here is to validate that we can use
1164	* ioremap on the specified address (32/64-bit dma_addr_t).
1165	*/
1166	PRTR0MEMOBJLNX pMemLnx;
1167	dma_addr_t PhysAddr = Phys;
1168	AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
1169
1170	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb, NULL);
1171	if (!pMemLnx)
1172	{
1173	IPRT_LINUX_RESTORE_EFL_AC();
1174	return VERR_NO_MEMORY;
1175	}
1176
1177	pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
1178	pMemLnx->Core.u.Phys.fAllocated = false;
1179	pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
1180	Assert(!pMemLnx->cPages);
1181	*ppMem = &pMemLnx->Core;
1182	IPRT_LINUX_RESTORE_EFL_AC();
1183	return VINF_SUCCESS;
1184	}
1185
1186	/* openSUSE Leap 42.3 detection :-/ */
1187	#if RTLNX_VER_RANGE(4,4,0, 4,6,0) && defined(FAULT_FLAG_REMOTE)
1188	# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
1189	#else
1190	# define GET_USER_PAGES_API LINUX_VERSION_CODE
1191	#endif
1192
1193	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
1194	{
1195	IPRT_LINUX_SAVE_EFL_AC();
1196	const int cPages = cb >> PAGE_SHIFT;
1197	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1198	struct vm_area_struct **papVMAs;
1199	PRTR0MEMOBJLNX pMemLnx;
1200	int rc = VERR_NO_MEMORY;
1201	int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
1202
1203	/*
1204	* Check for valid task and size overflows.
1205	*/
1206	if (!pTask)
1207	return VERR_NOT_SUPPORTED;
1208	if (((size_t)cPages << PAGE_SHIFT) != cb)
1209	return VERR_OUT_OF_RANGE;
1210
1211	/*
1212	* Allocate the memory object and a temporary buffer for the VMAs.
1213	*/
1214	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK,
1215	(void *)R3Ptr, cb, NULL);
1216	if (!pMemLnx)
1217	{
1218	IPRT_LINUX_RESTORE_EFL_AC();
1219	return VERR_NO_MEMORY;
1220	}
1221
1222	papVMAs = (struct vm_area_struct *)RTMemAlloc(sizeof(papVMAs) * cPages);
1223	if (papVMAs)
1224	{
1225	LNX_MM_DOWN_READ(pTask->mm);
1226
1227	/*
1228	* Get user pages.
1229	*/
1230	/** @todo r=bird: Should we not force read access too? */
1231	#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
1232	if (R0Process == RTR0ProcHandleSelf())
1233	rc = get_user_pages(R3Ptr, /* Where from. */
1234	cPages, /* How many pages. */
1235	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1236	fWrite ? FOLL_WRITE \| /* Write to memory. */
1237	FOLL_FORCE /* force write access. */
1238	: 0, /* Write to memory. */
1239	# else
1240	fWrite, /* Write to memory. */
1241	fWrite, /* force write access. */
1242	# endif
1243	&pMemLnx->apPages[0], /* Page array. */
1244	papVMAs); /* vmas */
1245	/*
1246	* Actually this should not happen at the moment as call this function
1247	* only for our own process.
1248	*/
1249	else
1250	rc = get_user_pages_remote(
1251	# if GET_USER_PAGES_API < KERNEL_VERSION(5, 9, 0)
1252	pTask, /* Task for fault accounting. */
1253	# endif
1254	pTask->mm, /* Whose pages. */
1255	R3Ptr, /* Where from. */
1256	cPages, /* How many pages. */
1257	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1258	fWrite ? FOLL_WRITE \| /* Write to memory. */
1259	FOLL_FORCE /* force write access. */
1260	: 0, /* Write to memory. */
1261	# else
1262	fWrite, /* Write to memory. */
1263	fWrite, /* force write access. */
1264	# endif
1265	&pMemLnx->apPages[0], /* Page array. */
1266	papVMAs /* vmas */
1267	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
1268	, NULL /* locked */
1269	# endif
1270	);
1271	#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1272	rc = get_user_pages(pTask, /* Task for fault accounting. */
1273	pTask->mm, /* Whose pages. */
1274	R3Ptr, /* Where from. */
1275	cPages, /* How many pages. */
1276	/* The get_user_pages API change was back-ported to 4.4.168. */
1277	# if RTLNX_VER_RANGE(4,4,168, 4,5,0)
1278	fWrite ? FOLL_WRITE \| /* Write to memory. */
1279	FOLL_FORCE /* force write access. */
1280	: 0, /* Write to memory. */
1281	# else
1282	fWrite, /* Write to memory. */
1283	fWrite, /* force write access. */
1284	# endif
1285	&pMemLnx->apPages[0], /* Page array. */
1286	papVMAs); /* vmas */
1287	#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1288	if (rc == cPages)
1289	{
1290	/*
1291	* Flush dcache (required?), protect against fork and _really_ pin the page
1292	* table entries. get_user_pages() will protect against swapping out the
1293	* pages but it will NOT protect against removing page table entries. This
1294	* can be achieved with
1295	* - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1296	* an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1297	* Usual Linux distributions support only a limited size of locked pages
1298	* (e.g. 32KB).
1299	* - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1300	* or by
1301	* - setting the VM_LOCKED flag. This is the same as doing mlock() without
1302	* a range check.
1303	*/
1304	/** @todo The Linux fork() protection will require more work if this API
1305	* is to be used for anything but locking VM pages. */
1306	while (rc-- > 0)
1307	{
1308	flush_dcache_page(pMemLnx->apPages[rc]);
1309	papVMAs[rc]->vm_flags \|= VM_DONTCOPY \| VM_LOCKED;
1310	}
1311
1312	LNX_MM_UP_READ(pTask->mm);
1313
1314	RTMemFree(papVMAs);
1315
1316	pMemLnx->Core.u.Lock.R0Process = R0Process;
1317	pMemLnx->cPages = cPages;
1318	Assert(!pMemLnx->fMappedToRing0);
1319	*ppMem = &pMemLnx->Core;
1320
1321	IPRT_LINUX_RESTORE_EFL_AC();
1322	return VINF_SUCCESS;
1323	}
1324
1325	/*
1326	* Failed - we need to unlock any pages that we succeeded to lock.
1327	*/
1328	while (rc-- > 0)
1329	{
1330	if (!PageReserved(pMemLnx->apPages[rc]))
1331	SetPageDirty(pMemLnx->apPages[rc]);
1332	#if RTLNX_VER_MIN(4,6,0)
1333	put_page(pMemLnx->apPages[rc]);
1334	#else
1335	page_cache_release(pMemLnx->apPages[rc]);
1336	#endif
1337	}
1338
1339	LNX_MM_UP_READ(pTask->mm);
1340
1341	RTMemFree(papVMAs);
1342	rc = VERR_LOCK_FAILED;
1343	}
1344
1345	rtR0MemObjDelete(&pMemLnx->Core);
1346	IPRT_LINUX_RESTORE_EFL_AC();
1347	return rc;
1348	}
1349
1350
1351	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1352	{
1353	IPRT_LINUX_SAVE_EFL_AC();
1354	void pvLast = (uint8_t )pv + cb - 1;
1355	size_t const cPages = cb >> PAGE_SHIFT;
1356	PRTR0MEMOBJLNX pMemLnx;
1357	bool fLinearMapping;
1358	int rc;
1359	uint8_t *pbPage;
1360	size_t iPage;
1361	NOREF(fAccess);
1362
1363	if ( !RTR0MemKernelIsValidAddr(pv)
1364	\|\| !RTR0MemKernelIsValidAddr(pv + cb))
1365	return VERR_INVALID_PARAMETER;
1366
1367	/*
1368	* The lower part of the kernel memory has a linear mapping between
1369	* physical and virtual addresses. So we take a short cut here. This is
1370	* assumed to be the cleanest way to handle those addresses (and the code
1371	* is well tested, though the test for determining it is not very nice).
1372	* If we ever decide it isn't we can still remove it.
1373	*/
1374	#if 0
1375	fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1376	#else
1377	fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1378	&& (unsigned long)pvLast < (unsigned long)high_memory;
1379	#endif
1380
1381	/*
1382	* Allocate the memory object.
1383	*/
1384	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, pv, cb, NULL);
1385	if (!pMemLnx)
1386	{
1387	IPRT_LINUX_RESTORE_EFL_AC();
1388	return VERR_NO_MEMORY;
1389	}
1390
1391	/*
1392	* Gather the pages.
1393	* We ASSUME all kernel pages are non-swappable and non-movable.
1394	*/
1395	rc = VINF_SUCCESS;
1396	pbPage = (uint8_t *)pvLast;
1397	iPage = cPages;
1398	if (!fLinearMapping)
1399	{
1400	while (iPage-- > 0)
1401	{
1402	struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1403	if (RT_UNLIKELY(!pPage))
1404	{
1405	rc = VERR_LOCK_FAILED;
1406	break;
1407	}
1408	pMemLnx->apPages[iPage] = pPage;
1409	pbPage -= PAGE_SIZE;
1410	}
1411	}
1412	else
1413	{
1414	while (iPage-- > 0)
1415	{
1416	pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1417	pbPage -= PAGE_SIZE;
1418	}
1419	}
1420	if (RT_SUCCESS(rc))
1421	{
1422	/*
1423	* Complete the memory object and return.
1424	*/
1425	pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1426	pMemLnx->cPages = cPages;
1427	Assert(!pMemLnx->fMappedToRing0);
1428	*ppMem = &pMemLnx->Core;
1429
1430	IPRT_LINUX_RESTORE_EFL_AC();
1431	return VINF_SUCCESS;
1432	}
1433
1434	rtR0MemObjDelete(&pMemLnx->Core);
1435	IPRT_LINUX_RESTORE_EFL_AC();
1436	return rc;
1437	}
1438
1439
1440	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1441	{
1442	#if RTLNX_VER_MIN(2,4,22)
1443	IPRT_LINUX_SAVE_EFL_AC();
1444	const size_t cPages = cb >> PAGE_SHIFT;
1445	struct page *pDummyPage;
1446	struct page **papPages;
1447
1448	/* check for unsupported stuff. */
1449	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1450	if (uAlignment > PAGE_SIZE)
1451	return VERR_NOT_SUPPORTED;
1452
1453	/*
1454	* Allocate a dummy page and create a page pointer array for vmap such that
1455	* the dummy page is mapped all over the reserved area.
1456	*/
1457	pDummyPage = alloc_page(GFP_HIGHUSER \| __GFP_NOWARN);
1458	if (pDummyPage)
1459	{
1460	papPages = RTMemAlloc(sizeof(papPages) cPages);
1461	if (papPages)
1462	{
1463	void *pv;
1464	size_t iPage = cPages;
1465	while (iPage-- > 0)
1466	papPages[iPage] = pDummyPage;
1467	# ifdef VM_MAP
1468	pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1469	# else
1470	pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1471	# endif
1472	RTMemFree(papPages);
1473	if (pv)
1474	{
1475	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb, NULL);
1476	if (pMemLnx)
1477	{
1478	pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1479	pMemLnx->cPages = 1;
1480	pMemLnx->apPages[0] = pDummyPage;
1481	*ppMem = &pMemLnx->Core;
1482	IPRT_LINUX_RESTORE_EFL_AC();
1483	return VINF_SUCCESS;
1484	}
1485	vunmap(pv);
1486	}
1487	}
1488	__free_page(pDummyPage);
1489	}
1490	IPRT_LINUX_RESTORE_EFL_AC();
1491	return VERR_NO_MEMORY;
1492
1493	#else /* < 2.4.22 */
1494	/*
1495	* Could probably use ioremap here, but the caller is in a better position than us
1496	* to select some safe physical memory.
1497	*/
1498	return VERR_NOT_SUPPORTED;
1499	#endif
1500	}
1501
1502
1503	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1504	{
1505	IPRT_LINUX_SAVE_EFL_AC();
1506	PRTR0MEMOBJLNX pMemLnx;
1507	void *pv;
1508	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1509	if (!pTask)
1510	return VERR_NOT_SUPPORTED;
1511
1512	/*
1513	* Check that the specified alignment is supported.
1514	*/
1515	if (uAlignment > PAGE_SIZE)
1516	return VERR_NOT_SUPPORTED;
1517
1518	/*
1519	* Let rtR0MemObjLinuxDoMmap do the difficult bits.
1520	*/
1521	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1522	if (pv == (void *)-1)
1523	{
1524	IPRT_LINUX_RESTORE_EFL_AC();
1525	return VERR_NO_MEMORY;
1526	}
1527
1528	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb, NULL);
1529	if (!pMemLnx)
1530	{
1531	rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1532	IPRT_LINUX_RESTORE_EFL_AC();
1533	return VERR_NO_MEMORY;
1534	}
1535
1536	pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1537	*ppMem = &pMemLnx->Core;
1538	IPRT_LINUX_RESTORE_EFL_AC();
1539	return VINF_SUCCESS;
1540	}
1541
1542
1543	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
1544	unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag)
1545	{
1546	int rc = VERR_NO_MEMORY;
1547	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1548	PRTR0MEMOBJLNX pMemLnx;
1549	IPRT_LINUX_SAVE_EFL_AC();
1550
1551	/* Fail if requested to do something we can't. */
1552	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1553	if (uAlignment > PAGE_SIZE)
1554	return VERR_NOT_SUPPORTED;
1555
1556	/*
1557	* Create the IPRT memory object.
1558	*/
1559	if (!cbSub)
1560	cbSub = pMemLnxToMap->Core.cb - offSub;
1561	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub, pszTag);
1562	if (pMemLnx)
1563	{
1564	if (pMemLnxToMap->cPages)
1565	{
1566	#if RTLNX_VER_MIN(2,4,22)
1567	/*
1568	* Use vmap - 2.4.22 and later.
1569	*/
1570	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1571	/** @todo We don't really care too much for EXEC here... 5.8 always adds NX. */
1572	Assert(((offSub + cbSub) >> PAGE_SHIFT) <= pMemLnxToMap->cPages);
1573	# ifdef VM_MAP
1574	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_MAP, fPg);
1575	# else
1576	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_ALLOC, fPg);
1577	# endif
1578	if (pMemLnx->Core.pv)
1579	{
1580	pMemLnx->fMappedToRing0 = true;
1581	rc = VINF_SUCCESS;
1582	}
1583	else
1584	rc = VERR_MAP_FAILED;
1585
1586	#else /* < 2.4.22 */
1587	/*
1588	* Only option here is to share mappings if possible and forget about fProt.
1589	*/
1590	if (rtR0MemObjIsRing3(pMemToMap))
1591	rc = VERR_NOT_SUPPORTED;
1592	else
1593	{
1594	rc = VINF_SUCCESS;
1595	if (!pMemLnxToMap->Core.pv)
1596	rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1597	if (RT_SUCCESS(rc))
1598	{
1599	Assert(pMemLnxToMap->Core.pv);
1600	pMemLnx->Core.pv = (uint8_t *)pMemLnxToMap->Core.pv + offSub;
1601	}
1602	}
1603	#endif
1604	}
1605	else
1606	{
1607	/*
1608	* MMIO / physical memory.
1609	*/
1610	Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1611	#if RTLNX_VER_MIN(2,6,25)
1612	/*
1613	* ioremap() defaults to no caching since the 2.6 kernels.
1614	* ioremap_nocache() has been removed finally in 5.6-rc1.
1615	*/
1616	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1617	? ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1618	: ioremap_cache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1619	#else /* KERNEL_VERSION < 2.6.25 */
1620	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1621	? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1622	: ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1623	#endif /* KERNEL_VERSION < 2.6.25 */
1624	if (pMemLnx->Core.pv)
1625	{
1626	/** @todo fix protection. */
1627	rc = VINF_SUCCESS;
1628	}
1629	}
1630	if (RT_SUCCESS(rc))
1631	{
1632	pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1633	*ppMem = &pMemLnx->Core;
1634	IPRT_LINUX_RESTORE_EFL_AC();
1635	return VINF_SUCCESS;
1636	}
1637	rtR0MemObjDelete(&pMemLnx->Core);
1638	}
1639
1640	IPRT_LINUX_RESTORE_EFL_AC();
1641	return rc;
1642	}
1643
1644
1645	#ifdef VBOX_USE_PAE_HACK
1646	/**
1647	* Replace the PFN of a PTE with the address of the actual page.
1648	*
1649	* The caller maps a reserved dummy page at the address with the desired access
1650	* and flags.
1651	*
1652	* This hack is required for older Linux kernels which don't provide
1653	* remap_pfn_range().
1654	*
1655	* @returns 0 on success, -ENOMEM on failure.
1656	* @param mm The memory context.
1657	* @param ulAddr The mapping address.
1658	* @param Phys The physical address of the page to map.
1659	*/
1660	static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1661	{
1662	int rc = -ENOMEM;
1663	pgd_t *pgd;
1664
1665	spin_lock(&mm->page_table_lock);
1666
1667	pgd = pgd_offset(mm, ulAddr);
1668	if (!pgd_none(pgd) && !pgd_bad(pgd))
1669	{
1670	pmd_t *pmd = pmd_offset(pgd, ulAddr);
1671	if (!pmd_none(*pmd))
1672	{
1673	pte_t *ptep = pte_offset_map(pmd, ulAddr);
1674	if (ptep)
1675	{
1676	pte_t pte = *ptep;
1677	pte.pte_high &= 0xfff00000;
1678	pte.pte_high \|= ((Phys >> 32) & 0x000fffff);
1679	pte.pte_low &= 0x00000fff;
1680	pte.pte_low \|= (Phys & 0xfffff000);
1681	set_pte(ptep, pte);
1682	pte_unmap(ptep);
1683	rc = 0;
1684	}
1685	}
1686	}
1687
1688	spin_unlock(&mm->page_table_lock);
1689	return rc;
1690	}
1691	#endif /* VBOX_USE_PAE_HACK */
1692
1693
1694	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1695	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag)
1696	{
1697	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1698	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1699	int rc = VERR_NO_MEMORY;
1700	PRTR0MEMOBJLNX pMemLnx;
1701	#ifdef VBOX_USE_PAE_HACK
1702	struct page *pDummyPage;
1703	RTHCPHYS DummyPhys;
1704	#endif
1705	IPRT_LINUX_SAVE_EFL_AC();
1706
1707	/*
1708	* Check for restrictions.
1709	*/
1710	if (!pTask)
1711	return VERR_NOT_SUPPORTED;
1712	if (uAlignment > PAGE_SIZE)
1713	return VERR_NOT_SUPPORTED;
1714
1715	#ifdef VBOX_USE_PAE_HACK
1716	/*
1717	* Allocate a dummy page for use when mapping the memory.
1718	*/
1719	pDummyPage = alloc_page(GFP_USER \| __GFP_NOWARN);
1720	if (!pDummyPage)
1721	{
1722	IPRT_LINUX_RESTORE_EFL_AC();
1723	return VERR_NO_MEMORY;
1724	}
1725	SetPageReserved(pDummyPage);
1726	DummyPhys = page_to_phys(pDummyPage);
1727	#endif
1728
1729	/*
1730	* Create the IPRT memory object.
1731	*/
1732	Assert(!offSub \|\| cbSub);
1733	if (cbSub == 0)
1734	cbSub = pMemLnxToMap->Core.cb;
1735	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub, pszTag);
1736	if (pMemLnx)
1737	{
1738	/*
1739	* Allocate user space mapping.
1740	*/
1741	void *pv;
1742	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cbSub, uAlignment, pTask, fProt);
1743	if (pv != (void *)-1)
1744	{
1745	/*
1746	* Map page by page into the mmap area.
1747	* This is generic, paranoid and not very efficient.
1748	*/
1749	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1750	unsigned long ulAddrCur = (unsigned long)pv;
1751	const size_t cPages = (offSub + cbSub) >> PAGE_SHIFT;
1752	size_t iPage;
1753
1754	LNX_MM_DOWN_WRITE(pTask->mm);
1755
1756	rc = VINF_SUCCESS;
1757	if (pMemLnxToMap->cPages)
1758	{
1759	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1760	{
1761	#if RTLNX_VER_MAX(2,6,11)
1762	RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1763	#endif
1764	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1765	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1766	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1767	#endif
1768	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1769	/* remap_page_range() limitation on x86 */
1770	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1771	#endif
1772
1773	#if defined(VBOX_USE_INSERT_PAGE) && RTLNX_VER_MIN(2,6,22)
1774	rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1775	/* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1776	* See remap_pfn_range() in mm/memory.c */
1777	#if RTLNX_VER_MIN(3,7,0)
1778	vma->vm_flags \|= VM_DONTEXPAND \| VM_DONTDUMP;
1779	#else
1780	vma->vm_flags \|= VM_RESERVED;
1781	#endif
1782	#elif RTLNX_VER_MIN(2,6,11)
1783	rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1784	#elif defined(VBOX_USE_PAE_HACK)
1785	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1786	if (!rc)
1787	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1788	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1789	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1790	#else /* 2.4 */
1791	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1792	#endif
1793	if (rc)
1794	{
1795	rc = VERR_NO_MEMORY;
1796	break;
1797	}
1798	}
1799	}
1800	else
1801	{
1802	RTHCPHYS Phys;
1803	if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
1804	Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
1805	else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
1806	Phys = pMemLnxToMap->Core.u.Cont.Phys;
1807	else
1808	{
1809	AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
1810	Phys = NIL_RTHCPHYS;
1811	}
1812	if (Phys != NIL_RTHCPHYS)
1813	{
1814	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
1815	{
1816	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1817	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1818	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1819	#endif
1820	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1821	/* remap_page_range() limitation on x86 */
1822	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1823	#endif
1824
1825	#if RTLNX_VER_MIN(2,6,11)
1826	rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1827	#elif defined(VBOX_USE_PAE_HACK)
1828	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1829	if (!rc)
1830	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1831	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1832	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1833	#else /* 2.4 */
1834	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1835	#endif
1836	if (rc)
1837	{
1838	rc = VERR_NO_MEMORY;
1839	break;
1840	}
1841	}
1842	}
1843	}
1844
1845	#ifdef CONFIG_NUMA_BALANCING
1846	# if RTLNX_VER_MAX(3,13,0) && RTLNX_RHEL_MAX(7,0)
1847	# define VBOX_NUMA_HACK_OLD
1848	# endif
1849	if (RT_SUCCESS(rc))
1850	{
1851	/** @todo Ugly hack! But right now we have no other means to
1852	* disable automatic NUMA page balancing. */
1853	# ifdef RT_OS_X86
1854	# ifdef VBOX_NUMA_HACK_OLD
1855	pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
1856	# endif
1857	pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
1858	# else
1859	# ifdef VBOX_NUMA_HACK_OLD
1860	pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
1861	# endif
1862	pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
1863	# endif
1864	}
1865	#endif /* CONFIG_NUMA_BALANCING */
1866
1867	LNX_MM_UP_WRITE(pTask->mm);
1868
1869	if (RT_SUCCESS(rc))
1870	{
1871	#ifdef VBOX_USE_PAE_HACK
1872	__free_page(pDummyPage);
1873	#endif
1874	pMemLnx->Core.pv = pv;
1875	pMemLnx->Core.u.Mapping.R0Process = R0Process;
1876	*ppMem = &pMemLnx->Core;
1877	IPRT_LINUX_RESTORE_EFL_AC();
1878	return VINF_SUCCESS;
1879	}
1880
1881	/*
1882	* Bail out.
1883	*/
1884	rtR0MemObjLinuxDoMunmap(pv, cbSub, pTask);
1885	}
1886	rtR0MemObjDelete(&pMemLnx->Core);
1887	}
1888	#ifdef VBOX_USE_PAE_HACK
1889	__free_page(pDummyPage);
1890	#endif
1891
1892	IPRT_LINUX_RESTORE_EFL_AC();
1893	return rc;
1894	}
1895
1896
1897	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1898	{
1899	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
1900	/*
1901	* Currently only supported when we've got addresses PTEs from the kernel.
1902	*/
1903	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1904	if (pMemLnx->pArea && pMemLnx->papPtesForArea)
1905	{
1906	pgprot_t const fPg = rtR0MemObjLinuxConvertProt(fProt, true /fKernel/);
1907	size_t const cPages = (offSub + cbSub) >> PAGE_SHIFT;
1908	pte_t **papPtes = pMemLnx->papPtesForArea;
1909	size_t i;
1910
1911	for (i = offSub >> PAGE_SHIFT; i < cPages; i++)
1912	{
1913	set_pte(papPtes[i], mk_pte(pMemLnx->apPages[i], fPg));
1914	}
1915	preempt_disable();
1916	__flush_tlb_all();
1917	preempt_enable();
1918	return VINF_SUCCESS;
1919	}
1920	# elif defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
1921	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1922	if ( pMemLnx->fExecutable
1923	&& pMemLnx->fMappedToRing0)
1924	{
1925	LNXAPPLYPGRANGE Args;
1926	Args.pMemLnx = pMemLnx;
1927	Args.fPg = rtR0MemObjLinuxConvertProt(fProt, true /fKernel/);
1928	int rcLnx = apply_to_page_range(current->active_mm, (unsigned long)pMemLnx->Core.pv + offSub, cbSub,
1929	rtR0MemObjLinuxApplyPageRange, (void *)&Args);
1930	if (rcLnx)
1931	return VERR_NOT_SUPPORTED;
1932
1933	return VINF_SUCCESS;
1934	}
1935	# endif
1936
1937	NOREF(pMem);
1938	NOREF(offSub);
1939	NOREF(cbSub);
1940	NOREF(fProt);
1941	return VERR_NOT_SUPPORTED;
1942	}
1943
1944
1945	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1946	{
1947	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1948
1949	if (pMemLnx->cPages)
1950	return page_to_phys(pMemLnx->apPages[iPage]);
1951
1952	switch (pMemLnx->Core.enmType)
1953	{
1954	case RTR0MEMOBJTYPE_CONT:
1955	return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
1956
1957	case RTR0MEMOBJTYPE_PHYS:
1958	return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
1959
1960	/* the parent knows */
1961	case RTR0MEMOBJTYPE_MAPPING:
1962	return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
1963
1964	/* cPages > 0 */
1965	case RTR0MEMOBJTYPE_LOW:
1966	case RTR0MEMOBJTYPE_LOCK:
1967	case RTR0MEMOBJTYPE_PHYS_NC:
1968	case RTR0MEMOBJTYPE_PAGE:
1969	default:
1970	AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
1971	RT_FALL_THROUGH();
1972
1973	case RTR0MEMOBJTYPE_RES_VIRT:
1974	return NIL_RTHCPHYS;
1975	}
1976	}
1977

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c@ 91479

Download in other formats: