memobj-r0drv-linux.c@ 78381

Last change on this file since 78381 was 78120, checked in by vboxsync, 6 years ago
IPRT: Started adding a RTR0MemObjMapUserEx function that takes offSub and cbSub. bugref:9217
Property svn:eol-style set to `native` Property svn:keywords set to `Id Rev Revision`
File size: 59.7 KB

Line
1	/* $Id: memobj-r0drv-linux.c 78120 2019-04-12 13:20:50Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-linux-kernel.h"
32
33	#include <iprt/memobj.h>
34	#include <iprt/assert.h>
35	#include <iprt/err.h>
36	#include <iprt/log.h>
37	#include <iprt/mem.h>
38	#include <iprt/process.h>
39	#include <iprt/string.h>
40	#include "internal/memobj.h"
41	#include "internal/iprt.h"
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	/* early 2.6 kernels */
48	#ifndef PAGE_SHARED_EXEC
49	# define PAGE_SHARED_EXEC PAGE_SHARED
50	#endif
51	#ifndef PAGE_READONLY_EXEC
52	# define PAGE_READONLY_EXEC PAGE_READONLY
53	#endif
54
55	/*
56	* 2.6.29+ kernels don't work with remap_pfn_range() anymore because
57	* track_pfn_vma_new() is apparently not defined for non-RAM pages.
58	* It should be safe to use vm_insert_page() older kernels as well.
59	*/
60	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
61	# define VBOX_USE_INSERT_PAGE
62	#endif
63	#if defined(CONFIG_X86_PAE) \
64	&& ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
65	\|\| ( LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \
66	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)))
67	# define VBOX_USE_PAE_HACK
68	#endif
69
70	/* gfp_t was introduced in 2.6.14, define it for earlier. */
71	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14)
72	# define gfp_t unsigned
73	#endif
74
75
76	/*********************************************************************************************************************************
77	* Structures and Typedefs *
78	*********************************************************************************************************************************/
79	/**
80	* The Darwin version of the memory object structure.
81	*/
82	typedef struct RTR0MEMOBJLNX
83	{
84	/** The core structure. */
85	RTR0MEMOBJINTERNAL Core;
86	/** Set if the allocation is contiguous.
87	* This means it has to be given back as one chunk. */
88	bool fContiguous;
89	/** Set if we've vmap'ed the memory into ring-0. */
90	bool fMappedToRing0;
91	/** The pages in the apPages array. */
92	size_t cPages;
93	/** Array of struct page pointers. (variable size) */
94	struct page *apPages[1];
95	} RTR0MEMOBJLNX, *PRTR0MEMOBJLNX;
96
97
98	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
99
100
101	/**
102	* Helper that converts from a RTR0PROCESS handle to a linux task.
103	*
104	* @returns The corresponding Linux task.
105	* @param R0Process IPRT ring-0 process handle.
106	*/
107	static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
108	{
109	/** @todo fix rtR0ProcessToLinuxTask!! */
110	/** @todo many (all?) callers currently assume that we return 'current'! */
111	return R0Process == RTR0ProcHandleSelf() ? current : NULL;
112	}
113
114
115	/**
116	* Compute order. Some functions allocate 2^order pages.
117	*
118	* @returns order.
119	* @param cPages Number of pages.
120	*/
121	static int rtR0MemObjLinuxOrder(size_t cPages)
122	{
123	int iOrder;
124	size_t cTmp;
125
126	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
127	;
128	if (cPages & ~((size_t)1 << iOrder))
129	++iOrder;
130
131	return iOrder;
132	}
133
134
135	/**
136	* Converts from RTMEM_PROT_* to Linux PAGE_*.
137	*
138	* @returns Linux page protection constant.
139	* @param fProt The IPRT protection mask.
140	* @param fKernel Whether it applies to kernel or user space.
141	*/
142	static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
143	{
144	switch (fProt)
145	{
146	default:
147	AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
148	case RTMEM_PROT_NONE:
149	return PAGE_NONE;
150
151	case RTMEM_PROT_READ:
152	return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
153
154	case RTMEM_PROT_WRITE:
155	case RTMEM_PROT_WRITE \| RTMEM_PROT_READ:
156	return fKernel ? PAGE_KERNEL : PAGE_SHARED;
157
158	case RTMEM_PROT_EXEC:
159	case RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
160	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
161	if (fKernel)
162	{
163	pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
164	pgprot_val(fPg) &= ~_PAGE_RW;
165	return fPg;
166	}
167	return PAGE_READONLY_EXEC;
168	#else
169	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
170	#endif
171
172	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
173	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
174	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
175	}
176	}
177
178
179	/**
180	* Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
181	* an empty user space mapping.
182	*
183	* We acquire the mmap_sem of the task!
184	*
185	* @returns Pointer to the mapping.
186	* (void *)-1 on failure.
187	* @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
188	* @param cb The size of the mapping.
189	* @param uAlignment The alignment of the mapping.
190	* @param pTask The Linux task to create this mapping in.
191	* @param fProt The RTMEM_PROT_* mask.
192	*/
193	static void rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct pTask, unsigned fProt)
194	{
195	unsigned fLnxProt;
196	unsigned long ulAddr;
197
198	Assert(pTask == current); /* do_mmap */
199	RT_NOREF_PV(pTask);
200
201	/*
202	* Convert from IPRT protection to mman.h PROT_ and call do_mmap.
203	*/
204	fProt &= (RTMEM_PROT_NONE \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC);
205	if (fProt == RTMEM_PROT_NONE)
206	fLnxProt = PROT_NONE;
207	else
208	{
209	fLnxProt = 0;
210	if (fProt & RTMEM_PROT_READ)
211	fLnxProt \|= PROT_READ;
212	if (fProt & RTMEM_PROT_WRITE)
213	fLnxProt \|= PROT_WRITE;
214	if (fProt & RTMEM_PROT_EXEC)
215	fLnxProt \|= PROT_EXEC;
216	}
217
218	if (R3PtrFixed != (RTR3PTR)-1)
219	{
220	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
221	ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
222	#else
223	down_write(&pTask->mm->mmap_sem);
224	ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
225	up_write(&pTask->mm->mmap_sem);
226	#endif
227	}
228	else
229	{
230	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
231	ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
232	#else
233	down_write(&pTask->mm->mmap_sem);
234	ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
235	up_write(&pTask->mm->mmap_sem);
236	#endif
237	if ( !(ulAddr & ~PAGE_MASK)
238	&& (ulAddr & (uAlignment - 1)))
239	{
240	/** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
241	* up alignment gaps. This is of course complicated by fragmentation (which we might have cause
242	* ourselves) and further by there begin two mmap strategies (top / bottom). */
243	/* For now, just ignore uAlignment requirements... */
244	}
245	}
246
247
248	if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
249	return (void *)-1;
250	return (void *)ulAddr;
251	}
252
253
254	/**
255	* Worker that destroys a user space mapping.
256	* Undoes what rtR0MemObjLinuxDoMmap did.
257	*
258	* We acquire the mmap_sem of the task!
259	*
260	* @param pv The ring-3 mapping.
261	* @param cb The size of the mapping.
262	* @param pTask The Linux task to destroy this mapping in.
263	*/
264	static void rtR0MemObjLinuxDoMunmap(void pv, size_t cb, struct task_struct pTask)
265	{
266	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
267	Assert(pTask == current); RT_NOREF_PV(pTask);
268	vm_munmap((unsigned long)pv, cb);
269	#elif defined(USE_RHEL4_MUNMAP)
270	down_write(&pTask->mm->mmap_sem);
271	do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
272	up_write(&pTask->mm->mmap_sem);
273	#else
274	down_write(&pTask->mm->mmap_sem);
275	do_munmap(pTask->mm, (unsigned long)pv, cb);
276	up_write(&pTask->mm->mmap_sem);
277	#endif
278	}
279
280
281	/**
282	* Internal worker that allocates physical pages and creates the memory object for them.
283	*
284	* @returns IPRT status code.
285	* @param ppMemLnx Where to store the memory object pointer.
286	* @param enmType The object type.
287	* @param cb The number of bytes to allocate.
288	* @param uAlignment The alignment of the physical memory.
289	* Only valid if fContiguous == true, ignored otherwise.
290	* @param fFlagsLnx The page allocation flags (GPFs).
291	* @param fContiguous Whether the allocation must be contiguous.
292	* @param rcNoMem What to return when we're out of pages.
293	*/
294	static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
295	size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, int rcNoMem)
296	{
297	size_t iPage;
298	size_t const cPages = cb >> PAGE_SHIFT;
299	struct page *paPages;
300
301	/*
302	* Allocate a memory object structure that's large enough to contain
303	* the page pointer array.
304	*/
305	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType, NULL, cb);
306	if (!pMemLnx)
307	return VERR_NO_MEMORY;
308	pMemLnx->cPages = cPages;
309
310	if (cPages > 255)
311	{
312	# ifdef __GFP_REPEAT
313	/* Try hard to allocate the memory, but the allocation attempt might fail. */
314	fFlagsLnx \|= __GFP_REPEAT;
315	# endif
316	# ifdef __GFP_NOMEMALLOC
317	/* Introduced with Linux 2.6.12: Don't use emergency reserves */
318	fFlagsLnx \|= __GFP_NOMEMALLOC;
319	# endif
320	}
321
322	/*
323	* Allocate the pages.
324	* For small allocations we'll try contiguous first and then fall back on page by page.
325	*/
326	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
327	if ( fContiguous
328	\|\| cb <= PAGE_SIZE * 2)
329	{
330	# ifdef VBOX_USE_INSERT_PAGE
331	paPages = alloc_pages(fFlagsLnx \| __GFP_COMP \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
332	# else
333	paPages = alloc_pages(fFlagsLnx \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
334	# endif
335	if (paPages)
336	{
337	fContiguous = true;
338	for (iPage = 0; iPage < cPages; iPage++)
339	pMemLnx->apPages[iPage] = &paPages[iPage];
340	}
341	else if (fContiguous)
342	{
343	rtR0MemObjDelete(&pMemLnx->Core);
344	return rcNoMem;
345	}
346	}
347
348	if (!fContiguous)
349	{
350	for (iPage = 0; iPage < cPages; iPage++)
351	{
352	pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx \| __GFP_NOWARN);
353	if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
354	{
355	while (iPage-- > 0)
356	__free_page(pMemLnx->apPages[iPage]);
357	rtR0MemObjDelete(&pMemLnx->Core);
358	return rcNoMem;
359	}
360	}
361	}
362
363	#else /* < 2.4.22 */
364	/** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
365	paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
366	if (!paPages)
367	{
368	rtR0MemObjDelete(&pMemLnx->Core);
369	return rcNoMem;
370	}
371	for (iPage = 0; iPage < cPages; iPage++)
372	{
373	pMemLnx->apPages[iPage] = &paPages[iPage];
374	MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
375	if (PageHighMem(pMemLnx->apPages[iPage]))
376	BUG();
377	}
378
379	fContiguous = true;
380	#endif /* < 2.4.22 */
381	pMemLnx->fContiguous = fContiguous;
382
383	#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
384	/*
385	* Reserve the pages.
386	*
387	* Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
388	* pages. According to Michal Hocko this shouldn't be necessary anyway because
389	* as pages which are not on the LRU list are never evictable.
390	*/
391	for (iPage = 0; iPage < cPages; iPage++)
392	SetPageReserved(pMemLnx->apPages[iPage]);
393	#endif
394
395	/*
396	* Note that the physical address of memory allocated with alloc_pages(flags, order)
397	* is always 2^(PAGE_SHIFT+order)-aligned.
398	*/
399	if ( fContiguous
400	&& uAlignment > PAGE_SIZE)
401	{
402	/*
403	* Check for alignment constraints. The physical address of memory allocated with
404	* alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
405	*/
406	if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
407	{
408	/*
409	* This should never happen!
410	*/
411	printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
412	(unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
413	rtR0MemObjLinuxFreePages(pMemLnx);
414	return rcNoMem;
415	}
416	}
417
418	*ppMemLnx = pMemLnx;
419	return VINF_SUCCESS;
420	}
421
422
423	/**
424	* Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
425	*
426	* This method does NOT free the object.
427	*
428	* @param pMemLnx The object which physical pages should be freed.
429	*/
430	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
431	{
432	size_t iPage = pMemLnx->cPages;
433	if (iPage > 0)
434	{
435	/*
436	* Restore the page flags.
437	*/
438	while (iPage-- > 0)
439	{
440	#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
441	/*
442	* See SetPageReserved() in rtR0MemObjLinuxAllocPages()
443	*/
444	ClearPageReserved(pMemLnx->apPages[iPage]);
445	#endif
446	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
447	#else
448	MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
449	#endif
450	}
451
452	/*
453	* Free the pages.
454	*/
455	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
456	if (!pMemLnx->fContiguous)
457	{
458	iPage = pMemLnx->cPages;
459	while (iPage-- > 0)
460	__free_page(pMemLnx->apPages[iPage]);
461	}
462	else
463	#endif
464	__free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
465
466	pMemLnx->cPages = 0;
467	}
468	}
469
470
471	/**
472	* Maps the allocation into ring-0.
473	*
474	* This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
475	*
476	* Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
477	* space, so we'll use that mapping if possible. If execute access is required, we'll
478	* play safe and do our own mapping.
479	*
480	* @returns IPRT status code.
481	* @param pMemLnx The linux memory object to map.
482	* @param fExecutable Whether execute access is required.
483	*/
484	static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
485	{
486	int rc = VINF_SUCCESS;
487
488	/*
489	* Choose mapping strategy.
490	*/
491	bool fMustMap = fExecutable
492	\|\| !pMemLnx->fContiguous;
493	if (!fMustMap)
494	{
495	size_t iPage = pMemLnx->cPages;
496	while (iPage-- > 0)
497	if (PageHighMem(pMemLnx->apPages[iPage]))
498	{
499	fMustMap = true;
500	break;
501	}
502	}
503
504	Assert(!pMemLnx->Core.pv);
505	Assert(!pMemLnx->fMappedToRing0);
506
507	if (fMustMap)
508	{
509	/*
510	* Use vmap - 2.4.22 and later.
511	*/
512	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
513	pgprot_t fPg;
514	pgprot_val(fPg) = _PAGE_PRESENT \| _PAGE_RW;
515	# ifdef _PAGE_NX
516	if (!fExecutable)
517	pgprot_val(fPg) \|= _PAGE_NX;
518	# endif
519
520	# ifdef VM_MAP
521	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
522	# else
523	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
524	# endif
525	if (pMemLnx->Core.pv)
526	pMemLnx->fMappedToRing0 = true;
527	else
528	rc = VERR_MAP_FAILED;
529	#else /* < 2.4.22 */
530	rc = VERR_NOT_SUPPORTED;
531	#endif
532	}
533	else
534	{
535	/*
536	* Use the kernel RAM mapping.
537	*/
538	pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
539	Assert(pMemLnx->Core.pv);
540	}
541
542	return rc;
543	}
544
545
546	/**
547	* Undoes what rtR0MemObjLinuxVMap() did.
548	*
549	* @param pMemLnx The linux memory object.
550	*/
551	static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
552	{
553	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
554	if (pMemLnx->fMappedToRing0)
555	{
556	Assert(pMemLnx->Core.pv);
557	vunmap(pMemLnx->Core.pv);
558	pMemLnx->fMappedToRing0 = false;
559	}
560	#else /* < 2.4.22 */
561	Assert(!pMemLnx->fMappedToRing0);
562	#endif
563	pMemLnx->Core.pv = NULL;
564	}
565
566
567	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
568	{
569	IPRT_LINUX_SAVE_EFL_AC();
570	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
571
572	/*
573	* Release any memory that we've allocated or locked.
574	*/
575	switch (pMemLnx->Core.enmType)
576	{
577	case RTR0MEMOBJTYPE_LOW:
578	case RTR0MEMOBJTYPE_PAGE:
579	case RTR0MEMOBJTYPE_CONT:
580	case RTR0MEMOBJTYPE_PHYS:
581	case RTR0MEMOBJTYPE_PHYS_NC:
582	rtR0MemObjLinuxVUnmap(pMemLnx);
583	rtR0MemObjLinuxFreePages(pMemLnx);
584	break;
585
586	case RTR0MEMOBJTYPE_LOCK:
587	if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
588	{
589	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
590	size_t iPage;
591	Assert(pTask);
592	if (pTask && pTask->mm)
593	down_read(&pTask->mm->mmap_sem);
594
595	iPage = pMemLnx->cPages;
596	while (iPage-- > 0)
597	{
598	if (!PageReserved(pMemLnx->apPages[iPage]))
599	SetPageDirty(pMemLnx->apPages[iPage]);
600	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
601	put_page(pMemLnx->apPages[iPage]);
602	#else
603	page_cache_release(pMemLnx->apPages[iPage]);
604	#endif
605	}
606
607	if (pTask && pTask->mm)
608	up_read(&pTask->mm->mmap_sem);
609	}
610	/* else: kernel memory - nothing to do here. */
611	break;
612
613	case RTR0MEMOBJTYPE_RES_VIRT:
614	Assert(pMemLnx->Core.pv);
615	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
616	{
617	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
618	Assert(pTask);
619	if (pTask && pTask->mm)
620	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
621	}
622	else
623	{
624	vunmap(pMemLnx->Core.pv);
625
626	Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
627	__free_page(pMemLnx->apPages[0]);
628	pMemLnx->apPages[0] = NULL;
629	pMemLnx->cPages = 0;
630	}
631	pMemLnx->Core.pv = NULL;
632	break;
633
634	case RTR0MEMOBJTYPE_MAPPING:
635	Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
636	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
637	{
638	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
639	Assert(pTask);
640	if (pTask && pTask->mm)
641	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
642	}
643	else
644	vunmap(pMemLnx->Core.pv);
645	pMemLnx->Core.pv = NULL;
646	break;
647
648	default:
649	AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
650	return VERR_INTERNAL_ERROR;
651	}
652	IPRT_LINUX_RESTORE_EFL_ONLY_AC();
653	return VINF_SUCCESS;
654	}
655
656
657	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
658	{
659	IPRT_LINUX_SAVE_EFL_AC();
660	PRTR0MEMOBJLNX pMemLnx;
661	int rc;
662
663	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
664	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
665	false /* non-contiguous */, VERR_NO_MEMORY);
666	#else
667	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
668	false /* non-contiguous */, VERR_NO_MEMORY);
669	#endif
670	if (RT_SUCCESS(rc))
671	{
672	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
673	if (RT_SUCCESS(rc))
674	{
675	*ppMem = &pMemLnx->Core;
676	IPRT_LINUX_RESTORE_EFL_AC();
677	return rc;
678	}
679
680	rtR0MemObjLinuxFreePages(pMemLnx);
681	rtR0MemObjDelete(&pMemLnx->Core);
682	}
683
684	IPRT_LINUX_RESTORE_EFL_AC();
685	return rc;
686	}
687
688
689	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
690	{
691	IPRT_LINUX_SAVE_EFL_AC();
692	PRTR0MEMOBJLNX pMemLnx;
693	int rc;
694
695	/* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
696	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
697	/* ZONE_DMA32: 0-4GB */
698	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
699	false /* non-contiguous */, VERR_NO_LOW_MEMORY);
700	if (RT_FAILURE(rc))
701	#endif
702	#ifdef RT_ARCH_AMD64
703	/* ZONE_DMA: 0-16MB */
704	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
705	false /* non-contiguous */, VERR_NO_LOW_MEMORY);
706	#else
707	# ifdef CONFIG_X86_PAE
708	# endif
709	/* ZONE_NORMAL: 0-896MB */
710	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
711	false /* non-contiguous */, VERR_NO_LOW_MEMORY);
712	#endif
713	if (RT_SUCCESS(rc))
714	{
715	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
716	if (RT_SUCCESS(rc))
717	{
718	*ppMem = &pMemLnx->Core;
719	IPRT_LINUX_RESTORE_EFL_AC();
720	return rc;
721	}
722
723	rtR0MemObjLinuxFreePages(pMemLnx);
724	rtR0MemObjDelete(&pMemLnx->Core);
725	}
726
727	IPRT_LINUX_RESTORE_EFL_AC();
728	return rc;
729	}
730
731
732	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
733	{
734	IPRT_LINUX_SAVE_EFL_AC();
735	PRTR0MEMOBJLNX pMemLnx;
736	int rc;
737
738	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
739	/* ZONE_DMA32: 0-4GB */
740	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
741	true /* contiguous */, VERR_NO_CONT_MEMORY);
742	if (RT_FAILURE(rc))
743	#endif
744	#ifdef RT_ARCH_AMD64
745	/* ZONE_DMA: 0-16MB */
746	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
747	true /* contiguous */, VERR_NO_CONT_MEMORY);
748	#else
749	/* ZONE_NORMAL (32-bit hosts): 0-896MB */
750	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
751	true /* contiguous */, VERR_NO_CONT_MEMORY);
752	#endif
753	if (RT_SUCCESS(rc))
754	{
755	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
756	if (RT_SUCCESS(rc))
757	{
758	#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_HIGHMEM64G))
759	size_t iPage = pMemLnx->cPages;
760	while (iPage-- > 0)
761	Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
762	#endif
763	pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
764	*ppMem = &pMemLnx->Core;
765	IPRT_LINUX_RESTORE_EFL_AC();
766	return rc;
767	}
768
769	rtR0MemObjLinuxFreePages(pMemLnx);
770	rtR0MemObjDelete(&pMemLnx->Core);
771	}
772
773	IPRT_LINUX_RESTORE_EFL_AC();
774	return rc;
775	}
776
777
778	/**
779	* Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
780	*
781	* @returns IPRT status code.
782	* @param ppMemLnx Where to
783	* @param enmType The object type.
784	* @param cb The size of the allocation.
785	* @param uAlignment The alignment of the physical memory.
786	* Only valid for fContiguous == true, ignored otherwise.
787	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
788	* @param fGfp The Linux GFP flags to use for the allocation.
789	*/
790	static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
791	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, gfp_t fGfp)
792	{
793	PRTR0MEMOBJLNX pMemLnx;
794	int rc;
795
796	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
797	enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
798	VERR_NO_PHYS_MEMORY);
799	if (RT_FAILURE(rc))
800	return rc;
801
802	/*
803	* Check the addresses if necessary. (Can be optimized a bit for PHYS.)
804	*/
805	if (PhysHighest != NIL_RTHCPHYS)
806	{
807	size_t iPage = pMemLnx->cPages;
808	while (iPage-- > 0)
809	if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
810	{
811	rtR0MemObjLinuxFreePages(pMemLnx);
812	rtR0MemObjDelete(&pMemLnx->Core);
813	return VERR_NO_MEMORY;
814	}
815	}
816
817	/*
818	* Complete the object.
819	*/
820	if (enmType == RTR0MEMOBJTYPE_PHYS)
821	{
822	pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
823	pMemLnx->Core.u.Phys.fAllocated = true;
824	}
825	*ppMem = &pMemLnx->Core;
826	return rc;
827	}
828
829
830	/**
831	* Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
832	*
833	* @returns IPRT status code.
834	* @param ppMem Where to store the memory object pointer on success.
835	* @param enmType The object type.
836	* @param cb The size of the allocation.
837	* @param uAlignment The alignment of the physical memory.
838	* Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
839	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
840	*/
841	static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
842	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest)
843	{
844	int rc;
845	IPRT_LINUX_SAVE_EFL_AC();
846
847	/*
848	* There are two clear cases and that's the <=16MB and anything-goes ones.
849	* When the physical address limit is somewhere in-between those two we'll
850	* just have to try, starting with HIGHUSER and working our way thru the
851	* different types, hoping we'll get lucky.
852	*
853	* We should probably move this physical address restriction logic up to
854	* the page alloc function as it would be more efficient there. But since
855	* we don't expect this to be a performance issue just yet it can wait.
856	*/
857	if (PhysHighest == NIL_RTHCPHYS)
858	/* ZONE_HIGHMEM: the whole physical memory */
859	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
860	else if (PhysHighest <= _1M * 16)
861	/* ZONE_DMA: 0-16MB */
862	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
863	else
864	{
865	rc = VERR_NO_MEMORY;
866	if (RT_FAILURE(rc))
867	/* ZONE_HIGHMEM: the whole physical memory */
868	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
869	if (RT_FAILURE(rc))
870	/* ZONE_NORMAL: 0-896MB */
871	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_USER);
872	#ifdef GFP_DMA32
873	if (RT_FAILURE(rc))
874	/* ZONE_DMA32: 0-4GB */
875	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA32);
876	#endif
877	if (RT_FAILURE(rc))
878	/* ZONE_DMA: 0-16MB */
879	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
880	}
881	IPRT_LINUX_RESTORE_EFL_AC();
882	return rc;
883	}
884
885
886	/**
887	* Translates a kernel virtual address to a linux page structure by walking the
888	* page tables.
889	*
890	* @note We do assume that the page tables will not change as we are walking
891	* them. This assumption is rather forced by the fact that I could not
892	* immediately see any way of preventing this from happening. So, we
893	* take some extra care when accessing them.
894	*
895	* Because of this, we don't want to use this function on memory where
896	* attribute changes to nearby pages is likely to cause large pages to
897	* be used or split up. So, don't use this for the linear mapping of
898	* physical memory.
899	*
900	* @returns Pointer to the page structur or NULL if it could not be found.
901	* @param pv The kernel virtual address.
902	*/
903	RTDECL(struct page ) rtR0MemObjLinuxVirtToPage(void pv)
904	{
905	unsigned long ulAddr = (unsigned long)pv;
906	unsigned long pfn;
907	struct page *pPage;
908	pte_t *pEntry;
909	union
910	{
911	pgd_t Global;
912	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
913	p4d_t Four;
914	#endif
915	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
916	pud_t Upper;
917	#endif
918	pmd_t Middle;
919	pte_t Entry;
920	} u;
921
922	/* Should this happen in a situation this code will be called in? And if
923	* so, can it change under our feet? See also
924	* "Documentation/vm/active_mm.txt" in the kernel sources. */
925	if (RT_UNLIKELY(!current->active_mm))
926	return NULL;
927	u.Global = *pgd_offset(current->active_mm, ulAddr);
928	if (RT_UNLIKELY(pgd_none(u.Global)))
929	return NULL;
930	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
931	# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
932	u.Four = *p4d_offset(&u.Global, ulAddr);
933	if (RT_UNLIKELY(p4d_none(u.Four)))
934	return NULL;
935	if (p4d_large(u.Four))
936	{
937	pPage = p4d_page(u.Four);
938	AssertReturn(pPage, NULL);
939	pfn = page_to_pfn(pPage); /* doing the safe way... */
940	AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
941	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
942	return pfn_to_page(pfn);
943	}
944	u.Upper = *pud_offset(&u.Four, ulAddr);
945	# else /* < 4.12 */
946	u.Upper = *pud_offset(&u.Global, ulAddr);
947	# endif /* < 4.12 */
948	if (RT_UNLIKELY(pud_none(u.Upper)))
949	return NULL;
950	# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
951	if (pud_large(u.Upper))
952	{
953	pPage = pud_page(u.Upper);
954	AssertReturn(pPage, NULL);
955	pfn = page_to_pfn(pPage); /* doing the safe way... */
956	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
957	return pfn_to_page(pfn);
958	}
959	# endif
960	u.Middle = *pmd_offset(&u.Upper, ulAddr);
961	#else /* < 2.6.11 */
962	u.Middle = *pmd_offset(&u.Global, ulAddr);
963	#endif /* < 2.6.11 */
964	if (RT_UNLIKELY(pmd_none(u.Middle)))
965	return NULL;
966	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0)
967	if (pmd_large(u.Middle))
968	{
969	pPage = pmd_page(u.Middle);
970	AssertReturn(pPage, NULL);
971	pfn = page_to_pfn(pPage); /* doing the safe way... */
972	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
973	return pfn_to_page(pfn);
974	}
975	#endif
976
977	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) \|\| defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
978	pEntry = pte_offset_map(&u.Middle, ulAddr);
979	#else
980	pEntry = pte_offset(&u.Middle, ulAddr);
981	#endif
982	if (RT_UNLIKELY(!pEntry))
983	return NULL;
984	u.Entry = *pEntry;
985	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) \|\| defined(pte_offset_map)
986	pte_unmap(pEntry);
987	#endif
988
989	if (RT_UNLIKELY(!pte_present(u.Entry)))
990	return NULL;
991	return pte_page(u.Entry);
992	}
993	RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
994
995
996	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
997	{
998	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest);
999	}
1000
1001
1002	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
1003	{
1004	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest);
1005	}
1006
1007
1008	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
1009	{
1010	IPRT_LINUX_SAVE_EFL_AC();
1011
1012	/*
1013	* All we need to do here is to validate that we can use
1014	* ioremap on the specified address (32/64-bit dma_addr_t).
1015	*/
1016	PRTR0MEMOBJLNX pMemLnx;
1017	dma_addr_t PhysAddr = Phys;
1018	AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
1019
1020	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb);
1021	if (!pMemLnx)
1022	{
1023	IPRT_LINUX_RESTORE_EFL_AC();
1024	return VERR_NO_MEMORY;
1025	}
1026
1027	pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
1028	pMemLnx->Core.u.Phys.fAllocated = false;
1029	pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
1030	Assert(!pMemLnx->cPages);
1031	*ppMem = &pMemLnx->Core;
1032	IPRT_LINUX_RESTORE_EFL_AC();
1033	return VINF_SUCCESS;
1034	}
1035
1036	/* openSUSE Leap 42.3 detection :-/ */
1037	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) \
1038	&& LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) \
1039	&& defined(FAULT_FLAG_REMOTE)
1040	# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
1041	#else
1042	# define GET_USER_PAGES_API LINUX_VERSION_CODE
1043	#endif
1044
1045	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
1046	{
1047	IPRT_LINUX_SAVE_EFL_AC();
1048	const int cPages = cb >> PAGE_SHIFT;
1049	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1050	struct vm_area_struct **papVMAs;
1051	PRTR0MEMOBJLNX pMemLnx;
1052	int rc = VERR_NO_MEMORY;
1053	int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
1054
1055	/*
1056	* Check for valid task and size overflows.
1057	*/
1058	if (!pTask)
1059	return VERR_NOT_SUPPORTED;
1060	if (((size_t)cPages << PAGE_SHIFT) != cb)
1061	return VERR_OUT_OF_RANGE;
1062
1063	/*
1064	* Allocate the memory object and a temporary buffer for the VMAs.
1065	*/
1066	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, (void *)R3Ptr, cb);
1067	if (!pMemLnx)
1068	{
1069	IPRT_LINUX_RESTORE_EFL_AC();
1070	return VERR_NO_MEMORY;
1071	}
1072
1073	papVMAs = (struct vm_area_struct *)RTMemAlloc(sizeof(papVMAs) * cPages);
1074	if (papVMAs)
1075	{
1076	down_read(&pTask->mm->mmap_sem);
1077
1078	/*
1079	* Get user pages.
1080	*/
1081	/** @todo r=bird: Should we not force read access too? */
1082	#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
1083	if (R0Process == RTR0ProcHandleSelf())
1084	rc = get_user_pages(R3Ptr, /* Where from. */
1085	cPages, /* How many pages. */
1086	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1087	fWrite ? FOLL_WRITE \| /* Write to memory. */
1088	FOLL_FORCE /* force write access. */
1089	: 0, /* Write to memory. */
1090	# else
1091	fWrite, /* Write to memory. */
1092	fWrite, /* force write access. */
1093	# endif
1094	&pMemLnx->apPages[0], /* Page array. */
1095	papVMAs); /* vmas */
1096	/*
1097	* Actually this should not happen at the moment as call this function
1098	* only for our own process.
1099	*/
1100	else
1101	rc = get_user_pages_remote(
1102	pTask, /* Task for fault accounting. */
1103	pTask->mm, /* Whose pages. */
1104	R3Ptr, /* Where from. */
1105	cPages, /* How many pages. */
1106	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1107	fWrite ? FOLL_WRITE \| /* Write to memory. */
1108	FOLL_FORCE /* force write access. */
1109	: 0, /* Write to memory. */
1110	# else
1111	fWrite, /* Write to memory. */
1112	fWrite, /* force write access. */
1113	# endif
1114	&pMemLnx->apPages[0], /* Page array. */
1115	papVMAs /* vmas */
1116	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
1117	, NULL /* locked */
1118	# endif
1119	);
1120	#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1121	rc = get_user_pages(pTask, /* Task for fault accounting. */
1122	pTask->mm, /* Whose pages. */
1123	R3Ptr, /* Where from. */
1124	cPages, /* How many pages. */
1125	/* The get_user_pages API change was back-ported to 4.4.168. */
1126	# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 168) \
1127	&& LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
1128	fWrite ? FOLL_WRITE \| /* Write to memory. */
1129	FOLL_FORCE /* force write access. */
1130	: 0, /* Write to memory. */
1131	# else
1132	fWrite, /* Write to memory. */
1133	fWrite, /* force write access. */
1134	# endif
1135	&pMemLnx->apPages[0], /* Page array. */
1136	papVMAs); /* vmas */
1137	#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1138	if (rc == cPages)
1139	{
1140	/*
1141	* Flush dcache (required?), protect against fork and _really_ pin the page
1142	* table entries. get_user_pages() will protect against swapping out the
1143	* pages but it will NOT protect against removing page table entries. This
1144	* can be achieved with
1145	* - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1146	* an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1147	* Usual Linux distributions support only a limited size of locked pages
1148	* (e.g. 32KB).
1149	* - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1150	* or by
1151	* - setting the VM_LOCKED flag. This is the same as doing mlock() without
1152	* a range check.
1153	*/
1154	/** @todo The Linux fork() protection will require more work if this API
1155	* is to be used for anything but locking VM pages. */
1156	while (rc-- > 0)
1157	{
1158	flush_dcache_page(pMemLnx->apPages[rc]);
1159	papVMAs[rc]->vm_flags \|= VM_DONTCOPY \| VM_LOCKED;
1160	}
1161
1162	up_read(&pTask->mm->mmap_sem);
1163
1164	RTMemFree(papVMAs);
1165
1166	pMemLnx->Core.u.Lock.R0Process = R0Process;
1167	pMemLnx->cPages = cPages;
1168	Assert(!pMemLnx->fMappedToRing0);
1169	*ppMem = &pMemLnx->Core;
1170
1171	IPRT_LINUX_RESTORE_EFL_AC();
1172	return VINF_SUCCESS;
1173	}
1174
1175	/*
1176	* Failed - we need to unlock any pages that we succeeded to lock.
1177	*/
1178	while (rc-- > 0)
1179	{
1180	if (!PageReserved(pMemLnx->apPages[rc]))
1181	SetPageDirty(pMemLnx->apPages[rc]);
1182	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
1183	put_page(pMemLnx->apPages[rc]);
1184	#else
1185	page_cache_release(pMemLnx->apPages[rc]);
1186	#endif
1187	}
1188
1189	up_read(&pTask->mm->mmap_sem);
1190
1191	RTMemFree(papVMAs);
1192	rc = VERR_LOCK_FAILED;
1193	}
1194
1195	rtR0MemObjDelete(&pMemLnx->Core);
1196	IPRT_LINUX_RESTORE_EFL_AC();
1197	return rc;
1198	}
1199
1200
1201	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1202	{
1203	IPRT_LINUX_SAVE_EFL_AC();
1204	void pvLast = (uint8_t )pv + cb - 1;
1205	size_t const cPages = cb >> PAGE_SHIFT;
1206	PRTR0MEMOBJLNX pMemLnx;
1207	bool fLinearMapping;
1208	int rc;
1209	uint8_t *pbPage;
1210	size_t iPage;
1211	NOREF(fAccess);
1212
1213	if ( !RTR0MemKernelIsValidAddr(pv)
1214	\|\| !RTR0MemKernelIsValidAddr(pv + cb))
1215	return VERR_INVALID_PARAMETER;
1216
1217	/*
1218	* The lower part of the kernel memory has a linear mapping between
1219	* physical and virtual addresses. So we take a short cut here. This is
1220	* assumed to be the cleanest way to handle those addresses (and the code
1221	* is well tested, though the test for determining it is not very nice).
1222	* If we ever decide it isn't we can still remove it.
1223	*/
1224	#if 0
1225	fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1226	#else
1227	fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1228	&& (unsigned long)pvLast < (unsigned long)high_memory;
1229	#endif
1230
1231	/*
1232	* Allocate the memory object.
1233	*/
1234	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, pv, cb);
1235	if (!pMemLnx)
1236	{
1237	IPRT_LINUX_RESTORE_EFL_AC();
1238	return VERR_NO_MEMORY;
1239	}
1240
1241	/*
1242	* Gather the pages.
1243	* We ASSUME all kernel pages are non-swappable and non-movable.
1244	*/
1245	rc = VINF_SUCCESS;
1246	pbPage = (uint8_t *)pvLast;
1247	iPage = cPages;
1248	if (!fLinearMapping)
1249	{
1250	while (iPage-- > 0)
1251	{
1252	struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1253	if (RT_UNLIKELY(!pPage))
1254	{
1255	rc = VERR_LOCK_FAILED;
1256	break;
1257	}
1258	pMemLnx->apPages[iPage] = pPage;
1259	pbPage -= PAGE_SIZE;
1260	}
1261	}
1262	else
1263	{
1264	while (iPage-- > 0)
1265	{
1266	pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1267	pbPage -= PAGE_SIZE;
1268	}
1269	}
1270	if (RT_SUCCESS(rc))
1271	{
1272	/*
1273	* Complete the memory object and return.
1274	*/
1275	pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1276	pMemLnx->cPages = cPages;
1277	Assert(!pMemLnx->fMappedToRing0);
1278	*ppMem = &pMemLnx->Core;
1279
1280	IPRT_LINUX_RESTORE_EFL_AC();
1281	return VINF_SUCCESS;
1282	}
1283
1284	rtR0MemObjDelete(&pMemLnx->Core);
1285	IPRT_LINUX_RESTORE_EFL_AC();
1286	return rc;
1287	}
1288
1289
1290	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1291	{
1292	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1293	IPRT_LINUX_SAVE_EFL_AC();
1294	const size_t cPages = cb >> PAGE_SHIFT;
1295	struct page *pDummyPage;
1296	struct page **papPages;
1297
1298	/* check for unsupported stuff. */
1299	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1300	if (uAlignment > PAGE_SIZE)
1301	return VERR_NOT_SUPPORTED;
1302
1303	/*
1304	* Allocate a dummy page and create a page pointer array for vmap such that
1305	* the dummy page is mapped all over the reserved area.
1306	*/
1307	pDummyPage = alloc_page(GFP_HIGHUSER \| __GFP_NOWARN);
1308	if (pDummyPage)
1309	{
1310	papPages = RTMemAlloc(sizeof(papPages) cPages);
1311	if (papPages)
1312	{
1313	void *pv;
1314	size_t iPage = cPages;
1315	while (iPage-- > 0)
1316	papPages[iPage] = pDummyPage;
1317	# ifdef VM_MAP
1318	pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1319	# else
1320	pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1321	# endif
1322	RTMemFree(papPages);
1323	if (pv)
1324	{
1325	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1326	if (pMemLnx)
1327	{
1328	pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1329	pMemLnx->cPages = 1;
1330	pMemLnx->apPages[0] = pDummyPage;
1331	*ppMem = &pMemLnx->Core;
1332	IPRT_LINUX_RESTORE_EFL_AC();
1333	return VINF_SUCCESS;
1334	}
1335	vunmap(pv);
1336	}
1337	}
1338	__free_page(pDummyPage);
1339	}
1340	IPRT_LINUX_RESTORE_EFL_AC();
1341	return VERR_NO_MEMORY;
1342
1343	#else /* < 2.4.22 */
1344	/*
1345	* Could probably use ioremap here, but the caller is in a better position than us
1346	* to select some safe physical memory.
1347	*/
1348	return VERR_NOT_SUPPORTED;
1349	#endif
1350	}
1351
1352
1353	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1354	{
1355	IPRT_LINUX_SAVE_EFL_AC();
1356	PRTR0MEMOBJLNX pMemLnx;
1357	void *pv;
1358	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1359	if (!pTask)
1360	return VERR_NOT_SUPPORTED;
1361
1362	/*
1363	* Check that the specified alignment is supported.
1364	*/
1365	if (uAlignment > PAGE_SIZE)
1366	return VERR_NOT_SUPPORTED;
1367
1368	/*
1369	* Let rtR0MemObjLinuxDoMmap do the difficult bits.
1370	*/
1371	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1372	if (pv == (void *)-1)
1373	{
1374	IPRT_LINUX_RESTORE_EFL_AC();
1375	return VERR_NO_MEMORY;
1376	}
1377
1378	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1379	if (!pMemLnx)
1380	{
1381	rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1382	IPRT_LINUX_RESTORE_EFL_AC();
1383	return VERR_NO_MEMORY;
1384	}
1385
1386	pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1387	*ppMem = &pMemLnx->Core;
1388	IPRT_LINUX_RESTORE_EFL_AC();
1389	return VINF_SUCCESS;
1390	}
1391
1392
1393	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap,
1394	void *pvFixed, size_t uAlignment,
1395	unsigned fProt, size_t offSub, size_t cbSub)
1396	{
1397	int rc = VERR_NO_MEMORY;
1398	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1399	PRTR0MEMOBJLNX pMemLnx;
1400	IPRT_LINUX_SAVE_EFL_AC();
1401
1402	/* Fail if requested to do something we can't. */
1403	AssertMsgReturn(!offSub && !cbSub, ("%#x %#x\n", offSub, cbSub), VERR_NOT_SUPPORTED);
1404	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1405	if (uAlignment > PAGE_SIZE)
1406	return VERR_NOT_SUPPORTED;
1407
1408	/*
1409	* Create the IPRT memory object.
1410	*/
1411	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, pMemLnxToMap->Core.cb);
1412	if (pMemLnx)
1413	{
1414	if (pMemLnxToMap->cPages)
1415	{
1416	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1417	/*
1418	* Use vmap - 2.4.22 and later.
1419	*/
1420	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1421	# ifdef VM_MAP
1422	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[0], pMemLnxToMap->cPages, VM_MAP, fPg);
1423	# else
1424	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[0], pMemLnxToMap->cPages, VM_ALLOC, fPg);
1425	# endif
1426	if (pMemLnx->Core.pv)
1427	{
1428	pMemLnx->fMappedToRing0 = true;
1429	rc = VINF_SUCCESS;
1430	}
1431	else
1432	rc = VERR_MAP_FAILED;
1433
1434	#else /* < 2.4.22 */
1435	/*
1436	* Only option here is to share mappings if possible and forget about fProt.
1437	*/
1438	if (rtR0MemObjIsRing3(pMemToMap))
1439	rc = VERR_NOT_SUPPORTED;
1440	else
1441	{
1442	rc = VINF_SUCCESS;
1443	if (!pMemLnxToMap->Core.pv)
1444	rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1445	if (RT_SUCCESS(rc))
1446	{
1447	Assert(pMemLnxToMap->Core.pv);
1448	pMemLnx->Core.pv = pMemLnxToMap->Core.pv;
1449	}
1450	}
1451	#endif
1452	}
1453	else
1454	{
1455	/*
1456	* MMIO / physical memory.
1457	*/
1458	Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1459	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1460	? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase, pMemLnxToMap->Core.cb)
1461	: ioremap(pMemLnxToMap->Core.u.Phys.PhysBase, pMemLnxToMap->Core.cb);
1462	if (pMemLnx->Core.pv)
1463	{
1464	/** @todo fix protection. */
1465	rc = VINF_SUCCESS;
1466	}
1467	}
1468	if (RT_SUCCESS(rc))
1469	{
1470	pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1471	*ppMem = &pMemLnx->Core;
1472	IPRT_LINUX_RESTORE_EFL_AC();
1473	return VINF_SUCCESS;
1474	}
1475	rtR0MemObjDelete(&pMemLnx->Core);
1476	}
1477
1478	IPRT_LINUX_RESTORE_EFL_AC();
1479	return rc;
1480	}
1481
1482
1483	#ifdef VBOX_USE_PAE_HACK
1484	/**
1485	* Replace the PFN of a PTE with the address of the actual page.
1486	*
1487	* The caller maps a reserved dummy page at the address with the desired access
1488	* and flags.
1489	*
1490	* This hack is required for older Linux kernels which don't provide
1491	* remap_pfn_range().
1492	*
1493	* @returns 0 on success, -ENOMEM on failure.
1494	* @param mm The memory context.
1495	* @param ulAddr The mapping address.
1496	* @param Phys The physical address of the page to map.
1497	*/
1498	static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1499	{
1500	int rc = -ENOMEM;
1501	pgd_t *pgd;
1502
1503	spin_lock(&mm->page_table_lock);
1504
1505	pgd = pgd_offset(mm, ulAddr);
1506	if (!pgd_none(pgd) && !pgd_bad(pgd))
1507	{
1508	pmd_t *pmd = pmd_offset(pgd, ulAddr);
1509	if (!pmd_none(*pmd))
1510	{
1511	pte_t *ptep = pte_offset_map(pmd, ulAddr);
1512	if (ptep)
1513	{
1514	pte_t pte = *ptep;
1515	pte.pte_high &= 0xfff00000;
1516	pte.pte_high \|= ((Phys >> 32) & 0x000fffff);
1517	pte.pte_low &= 0x00000fff;
1518	pte.pte_low \|= (Phys & 0xfffff000);
1519	set_pte(ptep, pte);
1520	pte_unmap(ptep);
1521	rc = 0;
1522	}
1523	}
1524	}
1525
1526	spin_unlock(&mm->page_table_lock);
1527	return rc;
1528	}
1529	#endif /* VBOX_USE_PAE_HACK */
1530
1531
1532	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1533	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub)
1534	{
1535	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1536	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1537	int rc = VERR_NO_MEMORY;
1538	PRTR0MEMOBJLNX pMemLnx;
1539	#ifdef VBOX_USE_PAE_HACK
1540	struct page *pDummyPage;
1541	RTHCPHYS DummyPhys;
1542	#endif
1543	IPRT_LINUX_SAVE_EFL_AC();
1544
1545	/*
1546	* Check for restrictions.
1547	*/
1548	if (!pTask)
1549	return VERR_NOT_SUPPORTED;
1550	if (uAlignment > PAGE_SIZE)
1551	return VERR_NOT_SUPPORTED;
1552	AssertMsgReturn(!offSub && !cbSub, ("%#zx %#zx\n", offSub, cbSub), VERR_NOT_SUPPORTED); /** @todo implement sub maps */
1553
1554	#ifdef VBOX_USE_PAE_HACK
1555	/*
1556	* Allocate a dummy page for use when mapping the memory.
1557	*/
1558	pDummyPage = alloc_page(GFP_USER \| __GFP_NOWARN);
1559	if (!pDummyPage)
1560	{
1561	IPRT_LINUX_RESTORE_EFL_AC();
1562	return VERR_NO_MEMORY;
1563	}
1564	SetPageReserved(pDummyPage);
1565	DummyPhys = page_to_phys(pDummyPage);
1566	#endif
1567
1568	/*
1569	* Create the IPRT memory object.
1570	*/
1571	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, pMemLnxToMap->Core.cb);
1572	if (pMemLnx)
1573	{
1574	/*
1575	* Allocate user space mapping.
1576	*/
1577	void *pv;
1578	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, pMemLnxToMap->Core.cb, uAlignment, pTask, fProt);
1579	if (pv != (void *)-1)
1580	{
1581	/*
1582	* Map page by page into the mmap area.
1583	* This is generic, paranoid and not very efficient.
1584	*/
1585	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1586	unsigned long ulAddrCur = (unsigned long)pv;
1587	const size_t cPages = pMemLnxToMap->Core.cb >> PAGE_SHIFT;
1588	size_t iPage;
1589
1590	down_write(&pTask->mm->mmap_sem);
1591
1592	rc = VINF_SUCCESS;
1593	if (pMemLnxToMap->cPages)
1594	{
1595	for (iPage = 0; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1596	{
1597	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)
1598	RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1599	#endif
1600	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1601	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1602	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1603	#endif
1604	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1605	/* remap_page_range() limitation on x86 */
1606	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1607	#endif
1608
1609	#if defined(VBOX_USE_INSERT_PAGE) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
1610	rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1611	/* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1612	* See remap_pfn_range() in mm/memory.c */
1613	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
1614	vma->vm_flags \|= VM_DONTEXPAND \| VM_DONTDUMP;
1615	#else
1616	vma->vm_flags \|= VM_RESERVED;
1617	#endif
1618	#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1619	rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1620	#elif defined(VBOX_USE_PAE_HACK)
1621	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1622	if (!rc)
1623	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1624	#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1625	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1626	#else /* 2.4 */
1627	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1628	#endif
1629	if (rc)
1630	{
1631	rc = VERR_NO_MEMORY;
1632	break;
1633	}
1634	}
1635	}
1636	else
1637	{
1638	RTHCPHYS Phys;
1639	if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
1640	Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
1641	else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
1642	Phys = pMemLnxToMap->Core.u.Cont.Phys;
1643	else
1644	{
1645	AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
1646	Phys = NIL_RTHCPHYS;
1647	}
1648	if (Phys != NIL_RTHCPHYS)
1649	{
1650	for (iPage = 0; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
1651	{
1652	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1653	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1654	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1655	#endif
1656	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1657	/* remap_page_range() limitation on x86 */
1658	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1659	#endif
1660
1661	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1662	rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1663	#elif defined(VBOX_USE_PAE_HACK)
1664	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1665	if (!rc)
1666	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1667	#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1668	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1669	#else /* 2.4 */
1670	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1671	#endif
1672	if (rc)
1673	{
1674	rc = VERR_NO_MEMORY;
1675	break;
1676	}
1677	}
1678	}
1679	}
1680
1681	#ifdef CONFIG_NUMA_BALANCING
1682	# if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0)
1683	# ifdef RHEL_RELEASE_CODE
1684	# if RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7, 0)
1685	# define VBOX_NUMA_HACK_OLD
1686	# endif
1687	# endif
1688	# endif
1689	if (RT_SUCCESS(rc))
1690	{
1691	/** @todo Ugly hack! But right now we have no other means to
1692	* disable automatic NUMA page balancing. */
1693	# ifdef RT_OS_X86
1694	# ifdef VBOX_NUMA_HACK_OLD
1695	pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
1696	# endif
1697	pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
1698	# else
1699	# ifdef VBOX_NUMA_HACK_OLD
1700	pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
1701	# endif
1702	pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
1703	# endif
1704	}
1705	#endif /* CONFIG_NUMA_BALANCING */
1706
1707	up_write(&pTask->mm->mmap_sem);
1708
1709	if (RT_SUCCESS(rc))
1710	{
1711	#ifdef VBOX_USE_PAE_HACK
1712	__free_page(pDummyPage);
1713	#endif
1714	pMemLnx->Core.pv = pv;
1715	pMemLnx->Core.u.Mapping.R0Process = R0Process;
1716	*ppMem = &pMemLnx->Core;
1717	IPRT_LINUX_RESTORE_EFL_AC();
1718	return VINF_SUCCESS;
1719	}
1720
1721	/*
1722	* Bail out.
1723	*/
1724	rtR0MemObjLinuxDoMunmap(pv, pMemLnxToMap->Core.cb, pTask);
1725	}
1726	rtR0MemObjDelete(&pMemLnx->Core);
1727	}
1728	#ifdef VBOX_USE_PAE_HACK
1729	__free_page(pDummyPage);
1730	#endif
1731
1732	IPRT_LINUX_RESTORE_EFL_AC();
1733	return rc;
1734	}
1735
1736
1737	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1738	{
1739	NOREF(pMem);
1740	NOREF(offSub);
1741	NOREF(cbSub);
1742	NOREF(fProt);
1743	return VERR_NOT_SUPPORTED;
1744	}
1745
1746
1747	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1748	{
1749	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1750
1751	if (pMemLnx->cPages)
1752	return page_to_phys(pMemLnx->apPages[iPage]);
1753
1754	switch (pMemLnx->Core.enmType)
1755	{
1756	case RTR0MEMOBJTYPE_CONT:
1757	return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
1758
1759	case RTR0MEMOBJTYPE_PHYS:
1760	return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
1761
1762	/* the parent knows */
1763	case RTR0MEMOBJTYPE_MAPPING:
1764	return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
1765
1766	/* cPages > 0 */
1767	case RTR0MEMOBJTYPE_LOW:
1768	case RTR0MEMOBJTYPE_LOCK:
1769	case RTR0MEMOBJTYPE_PHYS_NC:
1770	case RTR0MEMOBJTYPE_PAGE:
1771	default:
1772	AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
1773	/* fall thru */
1774
1775	case RTR0MEMOBJTYPE_RES_VIRT:
1776	return NIL_RTHCPHYS;
1777	}
1778	}
1779

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c@ 78381

Download in other formats: