memobj-r0drv-darwin.cpp@ 83070

Last change on this file since 83070 was 83070, checked in by vboxsync, 5 years ago
IPRT/memobj-r0drv-darwin.cpp: Workarounds for vm_fault misbehaving on kernel allocations in processes with hardened runtime enabled. bugref:9466
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 58.7 KB

Line
1	/* $Id: memobj-r0drv-darwin.cpp 83070 2020-02-13 21:21:02Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Darwin.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#define RTMEM_NO_WRAP_TO_EF_APIS /* circular dependency otherwise. */
32	#include "the-darwin-kernel.h"
33	#include "internal/iprt.h"
34	#include <iprt/memobj.h>
35
36	#include <iprt/asm.h>
37	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
38	# include <iprt/x86.h>
39	# include <iprt/asm-amd64-x86.h>
40	#endif
41	#include <iprt/assert.h>
42	#include <iprt/log.h>
43	#include <iprt/mem.h>
44	#include <iprt/param.h>
45	#include <iprt/process.h>
46	#include <iprt/semaphore.h>
47	#include <iprt/string.h>
48	#include <iprt/thread.h>
49	#include "internal/memobj.h"
50
51
52	/*********************************************************************************************************************************
53	* Defined Constants And Macros *
54	*********************************************************************************************************************************/
55	#define MY_PRINTF(...) do { printf(__VA_ARGS__); kprintf(__VA_ARGS__); } while (0)
56
57	/#define USE_VM_MAP_WIRE - may re-enable later when non-mapped allocations are added. /
58
59
60	/*********************************************************************************************************************************
61	* Structures and Typedefs *
62	*********************************************************************************************************************************/
63	/**
64	* The Darwin version of the memory object structure.
65	*/
66	typedef struct RTR0MEMOBJDARWIN
67	{
68	/** The core structure. */
69	RTR0MEMOBJINTERNAL Core;
70	/** Pointer to the memory descriptor created for allocated and locked memory. */
71	IOMemoryDescriptor *pMemDesc;
72	/** Pointer to the memory mapping object for mapped memory. */
73	IOMemoryMap *pMemMap;
74	} RTR0MEMOBJDARWIN, *PRTR0MEMOBJDARWIN;
75
76	/**
77	* Common thread_call_allocate/thread_call_enter argument package.
78	*/
79	typedef struct RTR0MEMOBJDARWINTHREADARGS
80	{
81	int32_t volatile rc;
82	RTSEMEVENTMULTI hEvent;
83	} RTR0MEMOBJDARWINTHREADARGS;
84
85
86	/**
87	* Arguments for rtR0MemObjNativeAllockWorkOnKernelThread.
88	*/
89	typedef struct RTR0MEMOBJDARWINALLOCARGS
90	{
91	RTR0MEMOBJDARWINTHREADARGS Core;
92	PPRTR0MEMOBJINTERNAL ppMem;
93	size_t cb;
94	bool fExecutable;
95	bool fContiguous;
96	mach_vm_address_t PhysMask;
97	uint64_t MaxPhysAddr;
98	RTR0MEMOBJTYPE enmType;
99	size_t uAlignment;
100	} RTR0MEMOBJDARWINALLOCARGS;
101
102	/**
103	* Arguments for rtR0MemObjNativeProtectWorkOnKernelThread.
104	*/
105	typedef struct RTR0MEMOBJDARWINPROTECTARGS
106	{
107	RTR0MEMOBJDARWINTHREADARGS Core;
108	PRTR0MEMOBJINTERNAL pMem;
109	size_t offSub;
110	size_t cbSub;
111	uint32_t fProt;
112	} RTR0MEMOBJDARWINPROTECTARGS;
113
114
115	/*********************************************************************************************************************************
116	* Internal Functions *
117	*********************************************************************************************************************************/
118	static void rtR0MemObjNativeAllockWorkerOnKernelThread(void pvUser0, void pvUser1);
119	static int rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt);
120	static void rtR0MemObjNativeProtectWorkerOnKernelThread(void pvUser0, void pvUser1);
121
122
123	/**
124	* Touch the pages to force the kernel to create or write-enable the page table
125	* entries.
126	*
127	* This is necessary since the kernel gets upset if we take a page fault when
128	* preemption is disabled and/or we own a simple lock (same thing). It has no
129	* problems with us disabling interrupts when taking the traps, weird stuff.
130	*
131	* (This is basically a way of invoking vm_fault on a range of pages.)
132	*
133	* @param pv Pointer to the first page.
134	* @param cb The number of bytes.
135	*/
136	static void rtR0MemObjDarwinTouchPages(void *pv, size_t cb)
137	{
138	uint32_t volatile pu32 = (uint32_t volatile )pv;
139	for (;;)
140	{
141	ASMAtomicCmpXchgU32(pu32, 0xdeadbeef, 0xdeadbeef);
142	if (cb <= PAGE_SIZE)
143	break;
144	cb -= PAGE_SIZE;
145	pu32 += PAGE_SIZE / sizeof(uint32_t);
146	}
147	}
148
149
150	/**
151	* Read (sniff) every page in the range to make sure there are some page tables
152	* entries backing it.
153	*
154	* This is just to be sure vm_protect didn't remove stuff without re-adding it
155	* if someone should try write-protect something.
156	*
157	* @param pv Pointer to the first page.
158	* @param cb The number of bytes.
159	*/
160	static void rtR0MemObjDarwinSniffPages(void const *pv, size_t cb)
161	{
162	uint32_t volatile pu32 = (uint32_t volatile )pv;
163	uint32_t volatile u32Counter = 0;
164	for (;;)
165	{
166	u32Counter += *pu32;
167
168	if (cb <= PAGE_SIZE)
169	break;
170	cb -= PAGE_SIZE;
171	pu32 += PAGE_SIZE / sizeof(uint32_t);
172	}
173	}
174
175
176	/**
177	* Gets the virtual memory map the specified object is mapped into.
178	*
179	* @returns VM map handle on success, NULL if no map.
180	* @param pMem The memory object.
181	*/
182	DECLINLINE(vm_map_t) rtR0MemObjDarwinGetMap(PRTR0MEMOBJINTERNAL pMem)
183	{
184	switch (pMem->enmType)
185	{
186	case RTR0MEMOBJTYPE_PAGE:
187	case RTR0MEMOBJTYPE_LOW:
188	case RTR0MEMOBJTYPE_CONT:
189	return kernel_map;
190
191	case RTR0MEMOBJTYPE_PHYS:
192	case RTR0MEMOBJTYPE_PHYS_NC:
193	if (pMem->pv)
194	return kernel_map;
195	return NULL;
196
197	case RTR0MEMOBJTYPE_LOCK:
198	return pMem->u.Lock.R0Process == NIL_RTR0PROCESS
199	? kernel_map
200	: get_task_map((task_t)pMem->u.Lock.R0Process);
201
202	case RTR0MEMOBJTYPE_RES_VIRT:
203	return pMem->u.ResVirt.R0Process == NIL_RTR0PROCESS
204	? kernel_map
205	: get_task_map((task_t)pMem->u.ResVirt.R0Process);
206
207	case RTR0MEMOBJTYPE_MAPPING:
208	return pMem->u.Mapping.R0Process == NIL_RTR0PROCESS
209	? kernel_map
210	: get_task_map((task_t)pMem->u.Mapping.R0Process);
211
212	default:
213	return NULL;
214	}
215	}
216
217	#if 0 /* not necessary after all*/
218	/* My vm_map mockup. */
219	struct my_vm_map
220	{
221	struct { char pad[8]; } lock;
222	struct my_vm_map_header
223	{
224	struct vm_map_links
225	{
226	void *prev;
227	void *next;
228	vm_map_offset_t start;
229	vm_map_offset_t end;
230	} links;
231	int nentries;
232	boolean_t entries_pageable;
233	} hdr;
234	pmap_t pmap;
235	vm_map_size_t size;
236	};
237
238
239	/**
240	* Gets the minimum map address, this is similar to get_map_min.
241	*
242	* @returns The start address of the map.
243	* @param pMap The map.
244	*/
245	static vm_map_offset_t rtR0MemObjDarwinGetMapMin(vm_map_t pMap)
246	{
247	/* lazy discovery of the correct offset. The apple guys is a wonderfully secretive bunch. */
248	static int32_t volatile s_offAdjust = INT32_MAX;
249	int32_t off = s_offAdjust;
250	if (off == INT32_MAX)
251	{
252	for (off = 0; ; off += sizeof(pmap_t))
253	{
254	if ((pmap_t )((uint8_t *)kernel_map + off) == kernel_pmap)
255	break;
256	AssertReturn(off <= RT_MAX(RT_OFFSETOF(struct my_vm_map, pmap) * 4, 1024), 0x1000);
257	}
258	ASMAtomicWriteS32(&s_offAdjust, off - RT_OFFSETOF(struct my_vm_map, pmap));
259	}
260
261	/* calculate it. */
262	struct my_vm_map pMyMap = (struct my_vm_map )((uint8_t *)pMap + off);
263	return pMyMap->hdr.links.start;
264	}
265	#endif /* unused */
266
267	#ifdef RT_STRICT
268	# if 0 /* unused */
269
270	/**
271	* Read from a physical page.
272	*
273	* @param HCPhys The address to start reading at.
274	* @param cb How many bytes to read.
275	* @param pvDst Where to put the bytes. This is zero'd on failure.
276	*/
277	static void rtR0MemObjDarwinReadPhys(RTHCPHYS HCPhys, size_t cb, void *pvDst)
278	{
279	memset(pvDst, '\0', cb);
280
281	IOAddressRange aRanges[1] = { { (mach_vm_address_t)HCPhys, RT_ALIGN_Z(cb, PAGE_SIZE) } };
282	IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges),
283	kIODirectionIn, NULL /task/);
284	if (pMemDesc)
285	{
286	#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
287	IOMemoryMap *pMemMap = pMemDesc->createMappingInTask(kernel_task, 0, kIOMapAnywhere \| kIOMapDefaultCache);
288	#else
289	IOMemoryMap *pMemMap = pMemDesc->map(kernel_task, 0, kIOMapAnywhere \| kIOMapDefaultCache);
290	#endif
291	if (pMemMap)
292	{
293	void const pvSrc = (void const )(uintptr_t)pMemMap->getVirtualAddress();
294	memcpy(pvDst, pvSrc, cb);
295	pMemMap->release();
296	}
297	else
298	MY_PRINTF("rtR0MemObjDarwinReadPhys: createMappingInTask failed; HCPhys=%llx\n", HCPhys);
299
300	pMemDesc->release();
301	}
302	else
303	MY_PRINTF("rtR0MemObjDarwinReadPhys: withAddressRanges failed; HCPhys=%llx\n", HCPhys);
304	}
305
306
307	/**
308	* Gets the PTE for a page.
309	*
310	* @returns the PTE.
311	* @param pvPage The virtual address to get the PTE for.
312	*/
313	static uint64_t rtR0MemObjDarwinGetPTE(void *pvPage)
314	{
315	RTUINT64U u64;
316	RTCCUINTREG cr3 = ASMGetCR3();
317	RTCCUINTREG cr4 = ASMGetCR4();
318	bool fPAE = false;
319	bool fLMA = false;
320	if (cr4 & X86_CR4_PAE)
321	{
322	fPAE = true;
323	uint32_t fExtFeatures = ASMCpuId_EDX(0x80000001);
324	if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
325	{
326	uint64_t efer = ASMRdMsr(MSR_K6_EFER);
327	if (efer & MSR_K6_EFER_LMA)
328	fLMA = true;
329	}
330	}
331
332	if (fLMA)
333	{
334	/* PML4 */
335	rtR0MemObjDarwinReadPhys((cr3 & ~(RTCCUINTREG)PAGE_OFFSET_MASK) \| (((uint64_t)(uintptr_t)pvPage >> X86_PML4_SHIFT) & X86_PML4_MASK) * 8, 8, &u64);
336	if (!(u64.u & X86_PML4E_P))
337	{
338	MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PML4E !p\n", pvPage);
339	return 0;
340	}
341
342	/* PDPTR */
343	rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) \| (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64) * 8, 8, &u64);
344	if (!(u64.u & X86_PDPE_P))
345	{
346	MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDPTE !p\n", pvPage);
347	return 0;
348	}
349	if (u64.u & X86_PDPE_LM_PS)
350	return (u64.u & ~(uint64_t)(_1G -1)) \| ((uintptr_t)pvPage & (_1G -1));
351
352	/* PD */
353	rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) \| (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64);
354	if (!(u64.u & X86_PDE_P))
355	{
356	MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDE !p\n", pvPage);
357	return 0;
358	}
359	if (u64.u & X86_PDE_PS)
360	return (u64.u & ~(uint64_t)(_2M -1)) \| ((uintptr_t)pvPage & (_2M -1));
361
362	/* PT */
363	rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) \| (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64);
364	if (!(u64.u & X86_PTE_P))
365	{
366	MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PTE !p\n", pvPage);
367	return 0;
368	}
369	return u64.u;
370	}
371
372	if (fPAE)
373	{
374	/* PDPTR */
375	rtR0MemObjDarwinReadPhys((u64.u & X86_CR3_PAE_PAGE_MASK) \| (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE) * 8, 8, &u64);
376	if (!(u64.u & X86_PDE_P))
377	return 0;
378
379	/* PD */
380	rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) \| (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64);
381	if (!(u64.u & X86_PDE_P))
382	return 0;
383	if (u64.u & X86_PDE_PS)
384	return (u64.u & ~(uint64_t)(_2M -1)) \| ((uintptr_t)pvPage & (_2M -1));
385
386	/* PT */
387	rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) \| (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64);
388	if (!(u64.u & X86_PTE_P))
389	return 0;
390	return u64.u;
391	}
392
393	/* PD */
394	rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) \| (((uintptr_t)pvPage >> X86_PD_SHIFT) & X86_PD_MASK) * 4, 4, &u64);
395	if (!(u64.au32[0] & X86_PDE_P))
396	return 0;
397	if (u64.au32[0] & X86_PDE_PS)
398	return (u64.u & ~(uint64_t)(_2M -1)) \| ((uintptr_t)pvPage & (_2M -1));
399
400	/* PT */
401	rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) \| (((uintptr_t)pvPage >> X86_PT_SHIFT) & X86_PT_MASK) * 4, 4, &u64);
402	if (!(u64.au32[0] & X86_PTE_P))
403	return 0;
404	return u64.au32[0];
405
406	return 0;
407	}
408
409	# endif /* unused */
410	#endif /* RT_STRICT */
411
412	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
413	{
414	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)pMem;
415	IPRT_DARWIN_SAVE_EFL_AC();
416
417	/*
418	* Release the IOMemoryDescriptor or/and IOMemoryMap associated with the object.
419	*/
420	if (pMemDarwin->pMemDesc)
421	{
422	pMemDarwin->pMemDesc->complete();
423	pMemDarwin->pMemDesc->release();
424	pMemDarwin->pMemDesc = NULL;
425	}
426
427	if (pMemDarwin->pMemMap)
428	{
429	pMemDarwin->pMemMap->release();
430	pMemDarwin->pMemMap = NULL;
431	}
432
433	/*
434	* Release any memory that we've allocated or locked.
435	*/
436	switch (pMemDarwin->Core.enmType)
437	{
438	case RTR0MEMOBJTYPE_LOW:
439	case RTR0MEMOBJTYPE_PAGE:
440	case RTR0MEMOBJTYPE_CONT:
441	break;
442
443	case RTR0MEMOBJTYPE_LOCK:
444	{
445	#ifdef USE_VM_MAP_WIRE
446	vm_map_t Map = pMemDarwin->Core.u.Lock.R0Process != NIL_RTR0PROCESS
447	? get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process)
448	: kernel_map;
449	kern_return_t kr = vm_map_unwire(Map,
450	(vm_map_offset_t)pMemDarwin->Core.pv,
451	(vm_map_offset_t)pMemDarwin->Core.pv + pMemDarwin->Core.cb,
452	0 /* not user */);
453	AssertRC(kr == KERN_SUCCESS); /** @todo don't ignore... */
454	#endif
455	break;
456	}
457
458	case RTR0MEMOBJTYPE_PHYS:
459	/*if (pMemDarwin->Core.u.Phys.fAllocated)
460	IOFreePhysical(pMemDarwin->Core.u.Phys.PhysBase, pMemDarwin->Core.cb);*/
461	Assert(!pMemDarwin->Core.u.Phys.fAllocated);
462	break;
463
464	case RTR0MEMOBJTYPE_PHYS_NC:
465	AssertMsgFailed(("RTR0MEMOBJTYPE_PHYS_NC\n"));
466	IPRT_DARWIN_RESTORE_EFL_AC();
467	return VERR_INTERNAL_ERROR;
468
469	case RTR0MEMOBJTYPE_RES_VIRT:
470	AssertMsgFailed(("RTR0MEMOBJTYPE_RES_VIRT\n"));
471	IPRT_DARWIN_RESTORE_EFL_AC();
472	return VERR_INTERNAL_ERROR;
473
474	case RTR0MEMOBJTYPE_MAPPING:
475	/* nothing to do here. */
476	break;
477
478	default:
479	AssertMsgFailed(("enmType=%d\n", pMemDarwin->Core.enmType));
480	IPRT_DARWIN_RESTORE_EFL_AC();
481	return VERR_INTERNAL_ERROR;
482	}
483
484	IPRT_DARWIN_RESTORE_EFL_AC();
485	return VINF_SUCCESS;
486	}
487
488
489	/**
490	* This is a helper function to executes @a pfnWorker in the context of the
491	* kernel_task
492	*
493	* @returns IPRT status code - result from pfnWorker or dispatching error.
494	* @param pfnWorker The function to call.
495	* @param pArgs The arguments to pass to the function.
496	*/
497	static int rtR0MemObjDarwinDoInKernelTaskThread(thread_call_func_t pfnWorker, RTR0MEMOBJDARWINTHREADARGS *pArgs)
498	{
499	pArgs->rc = VERR_IPE_UNINITIALIZED_STATUS;
500	pArgs->hEvent = NIL_RTSEMEVENTMULTI;
501	int rc = RTSemEventMultiCreate(&pArgs->hEvent);
502	if (RT_SUCCESS(rc))
503	{
504	thread_call_t hCall = thread_call_allocate(pfnWorker, (void *)pArgs);
505	if (hCall)
506	{
507	boolean_t fRc = thread_call_enter(hCall);
508	AssertLogRel(fRc == FALSE);
509
510	rc = RTSemEventMultiWaitEx(pArgs->hEvent, RTSEMWAIT_FLAGS_INDEFINITE \| RTSEMWAIT_FLAGS_UNINTERRUPTIBLE,
511	RT_INDEFINITE_WAIT);
512	AssertLogRelRC(rc);
513
514	rc = pArgs->rc;
515	thread_call_free(hCall);
516	}
517	else
518	rc = VERR_NO_MEMORY;
519	RTSemEventMultiDestroy(pArgs->hEvent);
520	}
521	return rc;
522	}
523
524
525	/**
526	* Signals result to thread waiting in rtR0MemObjDarwinDoInKernelTaskThread.
527	*
528	* @param pArgs The argument structure.
529	* @param rc The IPRT status code to signal.
530	*/
531	static void rtR0MemObjDarwinSignalThreadWaitinOnTask(RTR0MEMOBJDARWINTHREADARGS volatile *pArgs, int rc)
532	{
533	if (ASMAtomicCmpXchgS32(&pArgs->rc, rc, VERR_IPE_UNINITIALIZED_STATUS))
534	{
535	rc = RTSemEventMultiSignal(pArgs->hEvent);
536	AssertLogRelRC(rc);
537	}
538	}
539
540
541	/**
542	* Kernel memory alloc worker that uses inTaskWithPhysicalMask.
543	*
544	* @returns IPRT status code.
545	* @retval VERR_ADDRESS_TOO_BIG try another way.
546	*
547	* @param ppMem Where to return the memory object.
548	* @param cb The page aligned memory size.
549	* @param fExecutable Whether the mapping needs to be executable.
550	* @param fContiguous Whether the backing memory needs to be contiguous.
551	* @param PhysMask The mask for the backing memory (i.e. range). Use 0 if
552	* you don't care that much or is speculating.
553	* @param MaxPhysAddr The max address to verify the result against. Use
554	* UINT64_MAX if it doesn't matter.
555	* @param enmType The object type.
556	* @param uAlignment The allocation alignment (in bytes).
557	* @param fOnKernelThread Set if we're already on the kernel thread.
558	*/
559	static int rtR0MemObjNativeAllocWorker(PPRTR0MEMOBJINTERNAL ppMem, size_t cb,
560	bool fExecutable, bool fContiguous,
561	mach_vm_address_t PhysMask, uint64_t MaxPhysAddr,
562	RTR0MEMOBJTYPE enmType, size_t uAlignment, bool fOnKernelThread)
563	{
564	int rc;
565
566	/*
567	* Because of process code signing properties leaking into kernel space in
568	* in XNU's vm_fault.c code, we have to defer allocations of exec memory to
569	* a thread running in the kernel_task to get consistent results here.
570	* Believing is trouble is caused by the page_nx() + exec check, it was
571	* introduced in 10.10.5 (or 10.11.0).
572	* (PS. Doubt this is in anyway intentional behaviour, but rather unforseen
573	* consequences from a pragmatic approach to modifying the complicate VM code.)
574	*/
575	if (!fExecutable \|\| fOnKernelThread)
576	{ /* likely */ }
577	else
578	{
579	RTR0MEMOBJDARWINALLOCARGS Args;
580	Args.ppMem = ppMem;
581	Args.cb = cb;
582	Args.fExecutable = fExecutable;
583	Args.fContiguous = fContiguous;
584	Args.PhysMask = PhysMask;
585	Args.MaxPhysAddr = MaxPhysAddr;
586	Args.enmType = enmType;
587	Args.uAlignment = uAlignment;
588	return rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeAllockWorkerOnKernelThread, &Args.Core);
589	}
590
591	/*
592	* Try inTaskWithPhysicalMask first, but since we don't quite trust that it
593	* actually respects the physical memory mask (10.5.x is certainly busted),
594	* we'll use rtR0MemObjNativeAllocCont as a fallback for dealing with that.
595	*
596	* The kIOMemoryKernelUserShared flag just forces the result to be page aligned.
597	*
598	* The kIOMemoryMapperNone flag is required since 10.8.2 (IOMMU changes?).
599	*/
600
601	/* This is an old fudge from the snow leoard days: "Is it only on snow leopard?
602	Seen allocating memory for the VM structure, last page corrupted or
603	inaccessible." Made it only apply to snow leopard and older for now. */
604	size_t cbFudged = cb;
605	if (version_major >= 11 /* 10 = 10.7.x = Lion. */)
606	{ /* likely */ }
607	else
608	cbFudged += PAGE_SIZE;
609
610	IOOptionBits fOptions = kIOMemoryKernelUserShared \| kIODirectionInOut;
611	if (fContiguous)
612	{
613	fOptions \|= kIOMemoryPhysicallyContiguous;
614	if ( version_major > 12
615	\|\| (version_major == 12 && version_minor >= 2) /* 10.8.2 = Mountain Kitten */ )
616	fOptions \|= kIOMemoryHostPhysicallyContiguous; /* (Just to make ourselves clear, in case the xnu code changes.) */
617	}
618	if (version_major >= 12 /* 12 = 10.8.x = Mountain Kitten */)
619	fOptions \|= kIOMemoryMapperNone;
620
621	#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 && 0 /* enable when/if necessary */
622	/* Paranoia: Don't misrepresent our intentions, we won't map kernel executable memory into ring-0. */
623	if (fExecutable && version_major >= 11 /* 10.7.x = Lion, as below */)
624	{
625	fOptions &= ~kIOMemoryKernelUserShared;
626	if (uAlignment < PAGE_SIZE)
627	uAlignment = PAGE_SIZE;
628	}
629	#endif
630
631	/* The public initWithPhysicalMask virtual method appeared in 10.7.0, in
632	versions 10.5.0 up to 10.7.0 it was private, and 10.4.8-10.5.0 it was
633	x86 only and didn't have the alignment parameter (slot was different too). */
634	uint64_t uAlignmentActual = uAlignment;
635	IOBufferMemoryDescriptor *pMemDesc;
636	#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070
637	if (version_major >= 11 /* 11 = 10.7.x = Lion, could probably allow 10.5.0+ here if we really wanted to. */)
638	{
639	/* Starting with 10.6.x the physical mask is ignored if alignment is higher
640	than 1. The assumption seems to be that inTaskWithPhysicalMask() should
641	be used and the alignment inferred from the PhysMask argument. */
642	if (MaxPhysAddr != UINT64_MAX)
643	{
644	Assert(RT_ALIGN_64(PhysMask, uAlignment) == PhysMask);
645	uAlignmentActual = 1;
646	}
647
648	pMemDesc = new IOBufferMemoryDescriptor;
649	if (pMemDesc)
650	{
651	if (pMemDesc->initWithPhysicalMask(kernel_task, fOptions, cbFudged, uAlignmentActual, PhysMask))
652	{ /* likely */ }
653	else
654	{
655	pMemDesc->release();
656	pMemDesc = NULL;
657	}
658	}
659	}
660	else
661	#endif
662	pMemDesc = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task, fOptions, cbFudged, PhysMask);
663	if (pMemDesc)
664	{
665	IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
666	if (IORet == kIOReturnSuccess)
667	{
668	void *pv = pMemDesc->getBytesNoCopy(0, cbFudged);
669	if (pv)
670	{
671	/*
672	* Check if it's all below 4GB.
673	*/
674	addr64_t AddrPrev = 0;
675	MaxPhysAddr &= ~(uint64_t)PAGE_OFFSET_MASK;
676	for (IOByteCount off = 0; off < cb; off += PAGE_SIZE)
677	{
678	#ifdef __LP64__
679	addr64_t Addr = pMemDesc->getPhysicalSegment(off, NULL, kIOMemoryMapperNone);
680	#else
681	addr64_t Addr = pMemDesc->getPhysicalSegment64(off, NULL);
682	#endif
683	if ( Addr > MaxPhysAddr
684	\|\| !Addr
685	\|\| (Addr & PAGE_OFFSET_MASK)
686	\|\| ( fContiguous
687	&& !off
688	&& Addr == AddrPrev + PAGE_SIZE))
689	{
690	/* Buggy API, try allocate the memory another way. */
691	pMemDesc->complete();
692	pMemDesc->release();
693	if (PhysMask)
694	{
695	kprintf("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x - buggy API!\n",
696	(size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions);
697	LogRel(("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x - buggy API!\n",
698	(size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions));
699	}
700	return VERR_ADDRESS_TOO_BIG;
701	}
702	AddrPrev = Addr;
703	}
704
705	/*
706	* Check that it's aligned correctly.
707	*/
708	if ((uintptr_t)pv & (uAlignment - 1))
709	{
710	pMemDesc->complete();
711	pMemDesc->release();
712	if (PhysMask)
713	{
714	kprintf("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x) - buggy API!!\n",
715	pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions);
716	LogRel(("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x) - buggy API!\n",
717	pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions));
718	}
719	return VERR_NOT_SUPPORTED;
720	}
721
722	#ifdef RT_STRICT
723	/* check that the memory is actually mapped. */
724	//addr64_t Addr = pMemDesc->getPhysicalSegment64(0, NULL);
725	//printf("rtR0MemObjNativeAllocWorker: pv=%p %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr);
726	RTTHREADPREEMPTSTATE State = RTTHREADPREEMPTSTATE_INITIALIZER;
727	RTThreadPreemptDisable(&State);
728	rtR0MemObjDarwinTouchPages(pv, cb);
729	RTThreadPreemptRestore(&State);
730	#endif
731
732	/*
733	* Create the IPRT memory object.
734	*/
735	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), enmType, pv, cb);
736	if (pMemDarwin)
737	{
738	if (fContiguous)
739	{
740	#ifdef __LP64__
741	addr64_t PhysBase64 = pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone);
742	#else
743	addr64_t PhysBase64 = pMemDesc->getPhysicalSegment64(0, NULL);
744	#endif
745	RTHCPHYS PhysBase = PhysBase64; Assert(PhysBase == PhysBase64);
746	if (enmType == RTR0MEMOBJTYPE_CONT)
747	pMemDarwin->Core.u.Cont.Phys = PhysBase;
748	else if (enmType == RTR0MEMOBJTYPE_PHYS)
749	pMemDarwin->Core.u.Phys.PhysBase = PhysBase;
750	else
751	AssertMsgFailed(("enmType=%d\n", enmType));
752	}
753
754	if (fExecutable)
755	{
756	rc = rtR0MemObjNativeProtectWorker(&pMemDarwin->Core, 0, cb,
757	RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC);
758	#ifdef RT_STRICT
759	if (RT_SUCCESS(rc))
760	{
761	/* check that the memory is actually mapped. */
762	RTTHREADPREEMPTSTATE State2 = RTTHREADPREEMPTSTATE_INITIALIZER;
763	RTThreadPreemptDisable(&State2);
764	rtR0MemObjDarwinTouchPages(pv, cb);
765	RTThreadPreemptRestore(&State2);
766	}
767	#endif
768	/* Bug 6226: Ignore KERN_PROTECTION_FAILURE on Leopard and older. */
769	if ( rc == VERR_PERMISSION_DENIED
770	&& version_major <= 10 /* 10 = 10.6.x = Snow Leopard. */)
771	rc = VINF_SUCCESS;
772	}
773	else
774	rc = VINF_SUCCESS;
775	if (RT_SUCCESS(rc))
776	{
777	pMemDarwin->pMemDesc = pMemDesc;
778	*ppMem = &pMemDarwin->Core;
779	return VINF_SUCCESS;
780	}
781
782	rtR0MemObjDelete(&pMemDarwin->Core);
783	}
784
785	if (enmType == RTR0MEMOBJTYPE_PHYS_NC)
786	rc = VERR_NO_PHYS_MEMORY;
787	else if (enmType == RTR0MEMOBJTYPE_LOW)
788	rc = VERR_NO_LOW_MEMORY;
789	else if (enmType == RTR0MEMOBJTYPE_CONT)
790	rc = VERR_NO_CONT_MEMORY;
791	else
792	rc = VERR_NO_MEMORY;
793	}
794	else
795	rc = VERR_MEMOBJ_INIT_FAILED;
796
797	pMemDesc->complete();
798	}
799	else
800	rc = RTErrConvertFromDarwinIO(IORet);
801	pMemDesc->release();
802	}
803	else
804	rc = VERR_MEMOBJ_INIT_FAILED;
805	Assert(rc != VERR_ADDRESS_TOO_BIG);
806	return rc;
807	}
808
809
810	/**
811	* rtR0MemObjNativeAllocWorker kernel_task wrapper function.
812	*/
813	static void rtR0MemObjNativeAllockWorkerOnKernelThread(void pvUser0, void pvUser1)
814	{
815	AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1);
816	RTR0MEMOBJDARWINALLOCARGS volatile pArgs = (RTR0MEMOBJDARWINALLOCARGS volatile )pvUser0;
817	int rc = rtR0MemObjNativeAllocWorker(pArgs->ppMem, pArgs->cb, pArgs->fExecutable, pArgs->fContiguous, pArgs->PhysMask,
818	pArgs->MaxPhysAddr, pArgs->enmType, pArgs->uAlignment, true /fOnKernelThread/);
819	rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc);
820	}
821
822
823	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
824	{
825	IPRT_DARWIN_SAVE_EFL_AC();
826
827	int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous /, 0 / PhysMask */, UINT64_MAX,
828	RTR0MEMOBJTYPE_PAGE, PAGE_SIZE, false /fOnKernelThread/);
829
830	IPRT_DARWIN_RESTORE_EFL_AC();
831	return rc;
832	}
833
834
835	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
836	{
837	IPRT_DARWIN_SAVE_EFL_AC();
838
839	/*
840	* Try IOMallocPhysical/IOMallocAligned first.
841	* Then try optimistically without a physical address mask, which will always
842	* end up using IOMallocAligned.
843	*
844	* (See bug comment in the worker and IOBufferMemoryDescriptor::initWithPhysicalMask.)
845	*/
846	int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, ~(uint32_t)PAGE_OFFSET_MASK,
847	_4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, false /fOnKernelThread/);
848	if (rc == VERR_ADDRESS_TOO_BIG)
849	rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous /, 0 / PhysMask */,
850	_4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, false /fOnKernelThread/);
851
852	IPRT_DARWIN_RESTORE_EFL_AC();
853	return rc;
854	}
855
856
857	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
858	{
859	IPRT_DARWIN_SAVE_EFL_AC();
860
861	int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, true /* fContiguous */,
862	~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE,
863	RTR0MEMOBJTYPE_CONT, PAGE_SIZE, false /fOnKernelThread/);
864
865	/*
866	* Workaround for bogus IOKernelAllocateContiguous behavior, just in case.
867	* cb <= PAGE_SIZE allocations take a different path, using a different allocator.
868	*/
869	if (RT_FAILURE(rc) && cb <= PAGE_SIZE)
870	rc = rtR0MemObjNativeAllocWorker(ppMem, cb + PAGE_SIZE, fExecutable, true /* fContiguous */,
871	~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE,
872	RTR0MEMOBJTYPE_CONT, PAGE_SIZE, false /fOnKernelThread/);
873	IPRT_DARWIN_RESTORE_EFL_AC();
874	return rc;
875	}
876
877
878	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
879	{
880	if (uAlignment != PAGE_SIZE)
881	{
882	/* See rtR0MemObjNativeAllocWorker: */
883	if (version_major < 9 /* 9 = 10.5.x = Snow Leopard */)
884	return VERR_NOT_SUPPORTED;
885	}
886
887	IPRT_DARWIN_SAVE_EFL_AC();
888
889	/*
890	* Translate the PhysHighest address into a mask.
891	*/
892	int rc;
893	if (PhysHighest == NIL_RTHCPHYS)
894	rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable /, true / fContiguous */,
895	uAlignment <= PAGE_SIZE ? 0 : ~(mach_vm_address_t)(uAlignment - 1) /* PhysMask*/,
896	UINT64_MAX, RTR0MEMOBJTYPE_PHYS, uAlignment, false /fOnKernelThread/);
897	else
898	{
899	mach_vm_address_t PhysMask = 0;
900	PhysMask = ~(mach_vm_address_t)0;
901	while (PhysMask > (PhysHighest \| PAGE_OFFSET_MASK))
902	PhysMask >>= 1;
903	AssertReturn(PhysMask + 1 <= cb, VERR_INVALID_PARAMETER);
904	PhysMask &= ~(mach_vm_address_t)(uAlignment - 1);
905
906	rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable /, true / fContiguous */,
907	PhysMask, PhysHighest, RTR0MEMOBJTYPE_PHYS, uAlignment, false /fOnKernelThread/);
908	}
909
910	IPRT_DARWIN_RESTORE_EFL_AC();
911	return rc;
912	}
913
914
915	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
916	{
917	/** @todo rtR0MemObjNativeAllocPhys / darwin.
918	* This might be a bit problematic and may very well require having to create our own
919	* object which we populate with pages but without mapping it into any address space.
920	* Estimate is 2-3 days.
921	*/
922	RT_NOREF(ppMem, cb, PhysHighest);
923	return VERR_NOT_SUPPORTED;
924	}
925
926
927	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
928	{
929	AssertReturn(uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE, VERR_NOT_SUPPORTED);
930	IPRT_DARWIN_SAVE_EFL_AC();
931
932	/*
933	* Create a descriptor for it (the validation is always true on intel macs, but
934	* as it doesn't harm us keep it in).
935	*/
936	int rc = VERR_ADDRESS_TOO_BIG;
937	IOAddressRange aRanges[1] = { { Phys, cb } };
938	if ( aRanges[0].address == Phys
939	&& aRanges[0].length == cb)
940	{
941	IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges),
942	kIODirectionInOut, NULL /task/);
943	if (pMemDesc)
944	{
945	#ifdef __LP64__
946	Assert(Phys == pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone));
947	#else
948	Assert(Phys == pMemDesc->getPhysicalSegment64(0, NULL));
949	#endif
950
951	/*
952	* Create the IPRT memory object.
953	*/
954	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_PHYS, NULL, cb);
955	if (pMemDarwin)
956	{
957	pMemDarwin->Core.u.Phys.PhysBase = Phys;
958	pMemDarwin->Core.u.Phys.fAllocated = false;
959	pMemDarwin->Core.u.Phys.uCachePolicy = uCachePolicy;
960	pMemDarwin->pMemDesc = pMemDesc;
961	*ppMem = &pMemDarwin->Core;
962	IPRT_DARWIN_RESTORE_EFL_AC();
963	return VINF_SUCCESS;
964	}
965
966	rc = VERR_NO_MEMORY;
967	pMemDesc->release();
968	}
969	else
970	rc = VERR_MEMOBJ_INIT_FAILED;
971	}
972	else
973	AssertMsgFailed(("%#llx %llx\n", (unsigned long long)Phys, (unsigned long long)cb));
974	IPRT_DARWIN_RESTORE_EFL_AC();
975	return rc;
976	}
977
978
979	/**
980	* Internal worker for locking down pages.
981	*
982	* @return IPRT status code.
983	*
984	* @param ppMem Where to store the memory object pointer.
985	* @param pv First page.
986	* @param cb Number of bytes.
987	* @param fAccess The desired access, a combination of RTMEM_PROT_READ
988	* and RTMEM_PROT_WRITE.
989	* @param Task The task \a pv and \a cb refers to.
990	*/
991	static int rtR0MemObjNativeLock(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, task_t Task)
992	{
993	IPRT_DARWIN_SAVE_EFL_AC();
994	NOREF(fAccess);
995	#ifdef USE_VM_MAP_WIRE
996	vm_map_t Map = get_task_map(Task);
997	Assert(Map);
998
999	/*
1000	* First try lock the memory.
1001	*/
1002	int rc = VERR_LOCK_FAILED;
1003	kern_return_t kr = vm_map_wire(get_task_map(Task),
1004	(vm_map_offset_t)pv,
1005	(vm_map_offset_t)pv + cb,
1006	VM_PROT_DEFAULT,
1007	0 /* not user */);
1008	if (kr == KERN_SUCCESS)
1009	{
1010	/*
1011	* Create the IPRT memory object.
1012	*/
1013	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK, pv, cb);
1014	if (pMemDarwin)
1015	{
1016	pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task;
1017	*ppMem = &pMemDarwin->Core;
1018
1019	IPRT_DARWIN_RESTORE_EFL_AC();
1020	return VINF_SUCCESS;
1021	}
1022
1023	kr = vm_map_unwire(get_task_map(Task), (vm_map_offset_t)pv, (vm_map_offset_t)pv + cb, 0 /* not user */);
1024	Assert(kr == KERN_SUCCESS);
1025	rc = VERR_NO_MEMORY;
1026	}
1027
1028	#else
1029
1030	/*
1031	* Create a descriptor and try lock it (prepare).
1032	*/
1033	int rc = VERR_MEMOBJ_INIT_FAILED;
1034	IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRange((vm_address_t)pv, cb, kIODirectionInOut, Task);
1035	if (pMemDesc)
1036	{
1037	IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
1038	if (IORet == kIOReturnSuccess)
1039	{
1040	/*
1041	* Create the IPRT memory object.
1042	*/
1043	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK, pv, cb);
1044	if (pMemDarwin)
1045	{
1046	pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task;
1047	pMemDarwin->pMemDesc = pMemDesc;
1048	*ppMem = &pMemDarwin->Core;
1049
1050	IPRT_DARWIN_RESTORE_EFL_AC();
1051	return VINF_SUCCESS;
1052	}
1053
1054	pMemDesc->complete();
1055	rc = VERR_NO_MEMORY;
1056	}
1057	else
1058	rc = VERR_LOCK_FAILED;
1059	pMemDesc->release();
1060	}
1061	#endif
1062	IPRT_DARWIN_RESTORE_EFL_AC();
1063	return rc;
1064	}
1065
1066
1067	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
1068	{
1069	return rtR0MemObjNativeLock(ppMem, (void *)R3Ptr, cb, fAccess, (task_t)R0Process);
1070	}
1071
1072
1073	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1074	{
1075	return rtR0MemObjNativeLock(ppMem, pv, cb, fAccess, kernel_task);
1076	}
1077
1078
1079	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1080	{
1081	RT_NOREF(ppMem, pvFixed, cb, uAlignment);
1082	return VERR_NOT_SUPPORTED;
1083	}
1084
1085
1086	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1087	{
1088	RT_NOREF(ppMem, R3PtrFixed, cb, uAlignment, R0Process);
1089	return VERR_NOT_SUPPORTED;
1090	}
1091
1092
1093	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
1094	unsigned fProt, size_t offSub, size_t cbSub)
1095	{
1096	RT_NOREF(fProt);
1097	AssertReturn(pvFixed == (void *)-1, VERR_NOT_SUPPORTED);
1098
1099	/*
1100	* Check that the specified alignment is supported.
1101	*/
1102	if (uAlignment > PAGE_SIZE)
1103	return VERR_NOT_SUPPORTED;
1104	Assert(!offSub \|\| cbSub);
1105
1106	IPRT_DARWIN_SAVE_EFL_AC();
1107
1108	/*
1109	* Must have a memory descriptor that we can map.
1110	*/
1111	int rc = VERR_INVALID_PARAMETER;
1112	PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap;
1113	if (pMemToMapDarwin->pMemDesc)
1114	{
1115	/* The kIOMapPrefault option was added in 10.10.0; causes PTEs to be populated with
1116	INTEL_PTE_WIRED to be set, just like we desire (see further down). However, till
1117	10.13.0 it was not available for use on kernel mappings. Oh, fudge. */
1118	#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
1119	static uint32_t volatile s_fOptions = UINT32_MAX;
1120	uint32_t fOptions = s_fOptions;
1121	if (RT_UNLIKELY(fOptions == UINT32_MAX))
1122	s_fOptions = fOptions = version_major >= 17 ? 0x10000000 /kIOMapPrefault/ : 0; /* Since 10.13.0 (High Sierra). */
1123
1124	IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask(kernel_task,
1125	0,
1126	kIOMapAnywhere \| kIOMapDefaultCache \| fOptions,
1127	offSub,
1128	cbSub);
1129	#else
1130	IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map(kernel_task,
1131	0,
1132	kIOMapAnywhere \| kIOMapDefaultCache,
1133	offSub,
1134	cbSub);
1135	#endif
1136	if (pMemMap)
1137	{
1138	IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress();
1139	void pv = (void )(uintptr_t)VirtAddr;
1140	if ((uintptr_t)pv == VirtAddr && pv != NULL)
1141	{
1142	//#ifdef __LP64__
1143	// addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone);
1144	//#else
1145	// addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL);
1146	//#endif
1147	// MY_PRINTF("pv=%p: %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr);
1148
1149	// /*
1150	// * Explicitly lock it so that we're sure it is present and that
1151	// * its PTEs cannot be recycled.
1152	// * Note! withAddressRange() doesn't work as it adds kIOMemoryTypeVirtual64
1153	// * to the options which causes prepare() to not wire the pages.
1154	// * This is probably a bug.
1155	// */
1156	// IOAddressRange Range = { (mach_vm_address_t)pv, cbSub };
1157	// IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withOptions(&Range,
1158	// 1 /* count */,
1159	// 0 /* offset */,
1160	// kernel_task,
1161	// kIODirectionInOut \| kIOMemoryTypeVirtual,
1162	// kIOMapperSystem);
1163	// if (pMemDesc)
1164	// {
1165	// IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
1166	// if (IORet == kIOReturnSuccess)
1167	// {
1168	/* HACK ALERT! On kernels older than 10.10 (xnu version 14), we need to fault in
1169	the pages here so they can safely be accessed from inside simple
1170	locks and when preemption is disabled (no page-ins allowed).
1171	Note! This touching does not cause INTEL_PTE_WIRED (bit 10) to be set as we go
1172	thru general #PF and vm_fault doesn't figure it should be wired or something. */
1173	rtR0MemObjDarwinTouchPages(pv, cbSub ? cbSub : pMemToMap->cb);
1174	/** @todo First, the memory should've been mapped by now, and second, it
1175	* should have the wired attribute in the PTE (bit 10). Neither seems to
1176	* be the case. The disabled locking code doesn't make any difference,
1177	* which is extremely odd, and breaks rtR0MemObjNativeGetPagePhysAddr
1178	* (getPhysicalSegment64 -> 64 for the lock descriptor. */
1179	//#ifdef __LP64__
1180	// addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone);
1181	//#else
1182	// addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL);
1183	//#endif
1184	// MY_PRINTF("pv=%p: %8llx %8llx (%d)\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr2, 2);
1185
1186	/*
1187	* Create the IPRT memory object.
1188	*/
1189	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING,
1190	pv, cbSub ? cbSub : pMemToMap->cb);
1191	if (pMemDarwin)
1192	{
1193	pMemDarwin->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1194	pMemDarwin->pMemMap = pMemMap;
1195	// pMemDarwin->pMemDesc = pMemDesc;
1196	*ppMem = &pMemDarwin->Core;
1197
1198	IPRT_DARWIN_RESTORE_EFL_AC();
1199	return VINF_SUCCESS;
1200	}
1201
1202	// pMemDesc->complete();
1203	// rc = VERR_NO_MEMORY;
1204	// }
1205	// else
1206	// rc = RTErrConvertFromDarwinIO(IORet);
1207	// pMemDesc->release();
1208	// }
1209	// else
1210	// rc = VERR_MEMOBJ_INIT_FAILED;
1211	}
1212	else if (pv)
1213	rc = VERR_ADDRESS_TOO_BIG;
1214	else
1215	rc = VERR_MAP_FAILED;
1216	pMemMap->release();
1217	}
1218	else
1219	rc = VERR_MAP_FAILED;
1220	}
1221
1222	IPRT_DARWIN_RESTORE_EFL_AC();
1223	return rc;
1224	}
1225
1226
1227	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1228	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub)
1229	{
1230	RT_NOREF(fProt);
1231
1232	/*
1233	* Check for unsupported things.
1234	*/
1235	AssertReturn(R3PtrFixed == (RTR3PTR)-1, VERR_NOT_SUPPORTED);
1236	if (uAlignment > PAGE_SIZE)
1237	return VERR_NOT_SUPPORTED;
1238	Assert(!offSub \|\| cbSub);
1239
1240	IPRT_DARWIN_SAVE_EFL_AC();
1241
1242	/*
1243	* Must have a memory descriptor.
1244	*/
1245	int rc = VERR_INVALID_PARAMETER;
1246	PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap;
1247	if (pMemToMapDarwin->pMemDesc)
1248	{
1249	#if MAC_OS_X_VERSION_MIN_REQUIRED >= 101000 /* The kIOMapPrefault option was added in 10.10.0. */
1250	IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process,
1251	0,
1252	kIOMapAnywhere \| kIOMapDefaultCache \| kIOMapPrefault,
1253	offSub,
1254	cbSub);
1255	#elif MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
1256	static uint32_t volatile s_fOptions = UINT32_MAX;
1257	uint32_t fOptions = s_fOptions;
1258	if (RT_UNLIKELY(fOptions == UINT32_MAX))
1259	s_fOptions = fOptions = version_major >= 14 ? 0x10000000 /kIOMapPrefault/ : 0; /* Since 10.10.0. */
1260	IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process,
1261	0,
1262	kIOMapAnywhere \| kIOMapDefaultCache \| fOptions,
1263	offSub,
1264	cbSub);
1265	#else
1266	IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map((task_t)R0Process,
1267	0,
1268	kIOMapAnywhere \| kIOMapDefaultCache,
1269	offSub,
1270	cbSub);
1271	#endif
1272	if (pMemMap)
1273	{
1274	IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress();
1275	void pv = (void )(uintptr_t)VirtAddr;
1276	if ((uintptr_t)pv == VirtAddr && pv != NULL)
1277	{
1278	/*
1279	* Create the IPRT memory object.
1280	*/
1281	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING,
1282	pv, cbSub ? cbSub : pMemToMap->cb);
1283	if (pMemDarwin)
1284	{
1285	pMemDarwin->Core.u.Mapping.R0Process = R0Process;
1286	pMemDarwin->pMemMap = pMemMap;
1287	*ppMem = &pMemDarwin->Core;
1288
1289	IPRT_DARWIN_RESTORE_EFL_AC();
1290	return VINF_SUCCESS;
1291	}
1292
1293	rc = VERR_NO_MEMORY;
1294	}
1295	else if (pv)
1296	rc = VERR_ADDRESS_TOO_BIG;
1297	else
1298	rc = VERR_MAP_FAILED;
1299	pMemMap->release();
1300	}
1301	else
1302	rc = VERR_MAP_FAILED;
1303	}
1304
1305	IPRT_DARWIN_RESTORE_EFL_AC();
1306	return rc;
1307	}
1308
1309
1310	/**
1311	* Worker for rtR0MemObjNativeProtect that's typically called in a different
1312	* context.
1313	*/
1314	static int rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1315	{
1316	IPRT_DARWIN_SAVE_EFL_AC();
1317
1318	/* Get the map for the object. */
1319	vm_map_t pVmMap = rtR0MemObjDarwinGetMap(pMem);
1320	if (!pVmMap)
1321	{
1322	IPRT_DARWIN_RESTORE_EFL_AC();
1323	return VERR_NOT_SUPPORTED;
1324	}
1325
1326	/*
1327	* Convert the protection.
1328	*/
1329	vm_prot_t fMachProt;
1330	switch (fProt)
1331	{
1332	case RTMEM_PROT_NONE:
1333	fMachProt = VM_PROT_NONE;
1334	break;
1335	case RTMEM_PROT_READ:
1336	fMachProt = VM_PROT_READ;
1337	break;
1338	case RTMEM_PROT_READ \| RTMEM_PROT_WRITE:
1339	fMachProt = VM_PROT_READ \| VM_PROT_WRITE;
1340	break;
1341	case RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
1342	fMachProt = VM_PROT_READ \| VM_PROT_WRITE \| VM_PROT_EXECUTE;
1343	break;
1344	case RTMEM_PROT_WRITE:
1345	fMachProt = VM_PROT_WRITE \| VM_PROT_READ; /* never write-only */
1346	break;
1347	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
1348	fMachProt = VM_PROT_WRITE \| VM_PROT_EXECUTE \| VM_PROT_READ; /* never write-only or execute-only */
1349	break;
1350	case RTMEM_PROT_EXEC:
1351	fMachProt = VM_PROT_EXECUTE \| VM_PROT_READ; /* never execute-only */
1352	break;
1353	default:
1354	AssertFailedReturn(VERR_INVALID_PARAMETER);
1355	}
1356
1357	/*
1358	* Do the job.
1359	*/
1360	vm_offset_t Start = (uintptr_t)pMem->pv + offSub;
1361	kern_return_t krc = vm_protect(pVmMap,
1362	Start,
1363	cbSub,
1364	false,
1365	fMachProt);
1366	if (krc != KERN_SUCCESS)
1367	{
1368	static int s_cComplaints = 0;
1369	if (s_cComplaints < 10)
1370	{
1371	s_cComplaints++;
1372	printf("rtR0MemObjNativeProtect: vm_protect(%p,%p,%p,false,%#x) -> %d\n",
1373	pVmMap, (void )Start, (void )cbSub, fMachProt, krc);
1374
1375	kern_return_t krc2;
1376	vm_offset_t pvReal = Start;
1377	vm_size_t cbReal = 0;
1378	mach_msg_type_number_t cInfo = VM_REGION_BASIC_INFO_COUNT;
1379	struct vm_region_basic_info Info;
1380	RT_ZERO(Info);
1381	krc2 = vm_region(pVmMap, &pvReal, &cbReal, VM_REGION_BASIC_INFO, (vm_region_info_t)&Info, &cInfo, NULL);
1382	printf("rtR0MemObjNativeProtect: basic info - krc2=%d pv=%p cb=%p prot=%#x max=%#x inh=%#x shr=%d rvd=%d off=%#x behavior=%#x wired=%#x\n",
1383	krc2, (void )pvReal, (void )cbReal, Info.protection, Info.max_protection, Info.inheritance,
1384	Info.shared, Info.reserved, Info.offset, Info.behavior, Info.user_wired_count);
1385	}
1386	IPRT_DARWIN_RESTORE_EFL_AC();
1387	return RTErrConvertFromDarwinKern(krc);
1388	}
1389
1390	/*
1391	* Touch the pages if they should be writable afterwards and accessible
1392	* from code which should never fault. vm_protect() may leave pages
1393	* temporarily write protected, possibly due to pmap no-upgrade rules?
1394	*
1395	* This is the same trick (or HACK ALERT if you like) as applied in
1396	* rtR0MemObjNativeMapKernel.
1397	*/
1398	if ( pMem->enmType != RTR0MEMOBJTYPE_MAPPING
1399	\|\| pMem->u.Mapping.R0Process == NIL_RTR0PROCESS)
1400	{
1401	if (fProt & RTMEM_PROT_WRITE)
1402	rtR0MemObjDarwinTouchPages((void *)Start, cbSub);
1403	/*
1404	* Sniff (read) read-only pages too, just to be sure.
1405	*/
1406	else if (fProt & (RTMEM_PROT_READ \| RTMEM_PROT_EXEC))
1407	rtR0MemObjDarwinSniffPages((void const *)Start, cbSub);
1408	}
1409
1410	IPRT_DARWIN_RESTORE_EFL_AC();
1411	return VINF_SUCCESS;
1412	}
1413
1414
1415	/**
1416	* rtR0MemObjNativeProtect kernel_task wrapper function.
1417	*/
1418	static void rtR0MemObjNativeProtectWorkerOnKernelThread(void pvUser0, void pvUser1)
1419	{
1420	AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1);
1421	RTR0MEMOBJDARWINPROTECTARGS pArgs = (RTR0MEMOBJDARWINPROTECTARGS )pvUser0;
1422	int rc = rtR0MemObjNativeProtectWorker(pArgs->pMem, pArgs->offSub, pArgs->cbSub, pArgs->fProt);
1423	rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc);
1424	}
1425
1426
1427	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1428	{
1429	/*
1430	* The code won't work right because process codesigning properties leaks
1431	* into kernel_map memory management. So, if the user process we're running
1432	* in has CS restrictions active, we cannot play around with the EXEC
1433	* protection because some vm_fault.c think we're modifying the process map
1434	* or something. Looks like this problem was introduced in 10.10.5 or/and
1435	* 10.11.0, so for for Yosemite and up we'll just push the work off to a thread
1436	* running in the kernel_task context and hope it has be better chance of
1437	* consistent results.
1438	*/
1439	int rc;
1440	if ( version_major >= 14 /* 10.10 = Yosemite */
1441	&& rtR0MemObjDarwinGetMap(pMem) == kernel_map)
1442	{
1443	RTR0MEMOBJDARWINPROTECTARGS Args;
1444	Args.pMem = pMem;
1445	Args.offSub = offSub;
1446	Args.cbSub = cbSub;
1447	Args.fProt = fProt;
1448	rc = rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeProtectWorkerOnKernelThread, &Args.Core);
1449	}
1450	else
1451	rc = rtR0MemObjNativeProtectWorker(pMem, offSub, cbSub, fProt);
1452	return rc;
1453	}
1454
1455
1456	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1457	{
1458	RTHCPHYS PhysAddr;
1459	PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)pMem;
1460	IPRT_DARWIN_SAVE_EFL_AC();
1461
1462	#ifdef USE_VM_MAP_WIRE
1463	/*
1464	* Locked memory doesn't have a memory descriptor and
1465	* needs to be handled differently.
1466	*/
1467	if (pMemDarwin->Core.enmType == RTR0MEMOBJTYPE_LOCK)
1468	{
1469	ppnum_t PgNo;
1470	if (pMemDarwin->Core.u.Lock.R0Process == NIL_RTR0PROCESS)
1471	PgNo = pmap_find_phys(kernel_pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE);
1472	else
1473	{
1474	/*
1475	* From what I can tell, Apple seems to have locked up the all the
1476	* available interfaces that could help us obtain the pmap_t of a task
1477	* or vm_map_t.
1478
1479	* So, we'll have to figure out where in the vm_map_t structure it is
1480	* and read it our selves. ASSUMING that kernel_pmap is pointed to by
1481	* kernel_map->pmap, we scan kernel_map to locate the structure offset.
1482	* Not nice, but it will hopefully do the job in a reliable manner...
1483	*
1484	* (get_task_pmap, get_map_pmap or vm_map_pmap is what we really need btw.)
1485	*/
1486	static int s_offPmap = -1;
1487	if (RT_UNLIKELY(s_offPmap == -1))
1488	{
1489	pmap_t const p = (pmap_t )kernel_map;
1490	pmap_t const * const pEnd = p + 64;
1491	for (; p < pEnd; p++)
1492	if (*p == kernel_pmap)
1493	{
1494	s_offPmap = (uintptr_t)p - (uintptr_t)kernel_map;
1495	break;
1496	}
1497	AssertReturn(s_offPmap >= 0, NIL_RTHCPHYS);
1498	}
1499	pmap_t Pmap = (pmap_t )((uintptr_t)get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process) + s_offPmap);
1500	PgNo = pmap_find_phys(Pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE);
1501	}
1502
1503	IPRT_DARWIN_RESTORE_EFL_AC();
1504	AssertReturn(PgNo, NIL_RTHCPHYS);
1505	PhysAddr = (RTHCPHYS)PgNo << PAGE_SHIFT;
1506	Assert((PhysAddr >> PAGE_SHIFT) == PgNo);
1507	}
1508	else
1509	#endif /* USE_VM_MAP_WIRE */
1510	{
1511	/*
1512	* Get the memory descriptor.
1513	*/
1514	IOMemoryDescriptor *pMemDesc = pMemDarwin->pMemDesc;
1515	if (!pMemDesc)
1516	pMemDesc = pMemDarwin->pMemMap->getMemoryDescriptor();
1517	AssertReturn(pMemDesc, NIL_RTHCPHYS);
1518
1519	/*
1520	* If we've got a memory descriptor, use getPhysicalSegment64().
1521	*/
1522	#ifdef __LP64__
1523	addr64_t Addr = pMemDesc->getPhysicalSegment(iPage * PAGE_SIZE, NULL, kIOMemoryMapperNone);
1524	#else
1525	addr64_t Addr = pMemDesc->getPhysicalSegment64(iPage * PAGE_SIZE, NULL);
1526	#endif
1527	IPRT_DARWIN_RESTORE_EFL_AC();
1528	AssertMsgReturn(Addr, ("iPage=%u\n", iPage), NIL_RTHCPHYS);
1529	PhysAddr = Addr;
1530	AssertMsgReturn(PhysAddr == Addr, ("PhysAddr=%RHp Addr=%RX64\n", PhysAddr, (uint64_t)Addr), NIL_RTHCPHYS);
1531	}
1532
1533	return PhysAddr;
1534	}
1535

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp@ 83070

Download in other formats: