NEMR3Native-darwin.cpp@ 92351

Last change on this file since 92351 was 92351, checked in by vboxsync, 3 years ago
VMM/NEMR3Native-darwin.cpp,VMM/Makefile.kmk: Fon't link directly against Hypervisor.framework as it would require us to update our SDK if we want to use an optional feature on a newer release. Load and resolve everything dynamically instead (which is not difficult as the API is pretty barebones...), bugref:9044
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 125.2 KB

Line
1	/* $Id: NEMR3Native-darwin.cpp 92351 2021-11-11 10:01:54Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 macOS backend using Hypervisor.framework.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	*/
9
10	/*
11	* Copyright (C) 2020 Oracle Corporation
12	*
13	* This file is part of VirtualBox Open Source Edition (OSE), as
14	* available from http://www.virtualbox.org. This file is free software;
15	* you can redistribute it and/or modify it under the terms of the GNU
16	* General Public License (GPL) as published by the Free Software
17	* Foundation, in version 2 as it comes in the "COPYING" file of the
18	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20	*/
21
22
23	/*********************************************************************************************************************************
24	* Header Files *
25	*********************************************************************************************************************************/
26	#define LOG_GROUP LOG_GROUP_NEM
27	#define VMCPU_INCL_CPUM_GST_CTX
28	#include <VBox/vmm/nem.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/em.h>
31	#include <VBox/vmm/apic.h>
32	#include <VBox/vmm/pdm.h>
33	#include <VBox/vmm/hm.h>
34	#include <VBox/vmm/hm_vmx.h>
35	#include <VBox/vmm/dbgftrace.h>
36	#include "VMXInternal.h"
37	#include "NEMInternal.h"
38	#include <VBox/vmm/vmcc.h>
39	#include "dtrace/VBoxVMM.h"
40
41	#include <iprt/asm.h>
42	#include <iprt/ldr.h>
43	#include <iprt/path.h>
44	#include <iprt/string.h>
45	#include <iprt/system.h>
46	#include <iprt/utf16.h>
47
48
49	/*********************************************************************************************************************************
50	* Defined Constants And Macros *
51	*********************************************************************************************************************************/
52	/* No nested hwvirt (for now). */
53	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
54	# undef VBOX_WITH_NESTED_HWVIRT_VMX
55	#endif
56
57
58	/** @name HV return codes.
59	* @{ */
60	/** Operation was successful. */
61	#define HV_SUCCESS 0
62	/** An error occurred during operation. */
63	#define HV_ERROR 0xfae94001
64	/** The operation could not be completed right now, try again. */
65	#define HV_BUSY 0xfae94002
66	/** One of the parameters passed wis invalid. */
67	#define HV_BAD_ARGUMENT 0xfae94003
68	/** Not enough resources left to fulfill the operation. */
69	#define HV_NO_RESOURCES 0xfae94005
70	/** The device could not be found. */
71	#define HV_NO_DEVICE 0xfae94006
72	/** The operation is not supportd on this platform with this configuration. */
73	#define HV_UNSUPPORTED 0xfae94007
74	/** @} */
75
76
77	/** @name HV memory protection flags.
78	* @{ */
79	/** Memory is readable. */
80	#define HV_MEMORY_READ RT_BIT_64(0)
81	/** Memory is writeable. */
82	#define HV_MEMORY_WRITE RT_BIT_64(1)
83	/** Memory is executable. */
84	#define HV_MEMORY_EXEC RT_BIT_64(2)
85	/** @} */
86
87
88	/** @name HV shadow VMCS protection flags.
89	* @{ */
90	/** Shadow VMCS field is not accessible. */
91	#define HV_SHADOW_VMCS_NONE 0
92	/** Shadow VMCS fild is readable. */
93	#define HV_SHADOW_VMCS_READ RT_BIT_64(0)
94	/** Shadow VMCS field is writeable. */
95	#define HV_SHADOW_VMCS_WRITE RT_BIT_64(1)
96	/** @} */
97
98
99	/** Default VM creation flags. */
100	#define HV_VM_DEFAULT 0
101	/** Default guest address space creation flags. */
102	#define HV_VM_SPACE_DEFAULT 0
103	/** Default vCPU creation flags. */
104	#define HV_VCPU_DEFAULT 0
105
106	#define HV_DEADLINE_FOREVER UINT64_MAX
107
108
109	/*********************************************************************************************************************************
110	* Structures and Typedefs *
111	*********************************************************************************************************************************/
112
113	/** HV return code type. */
114	typedef uint32_t hv_return_t;
115	/** HV capability bitmask. */
116	typedef uint64_t hv_capability_t;
117	/** Option bitmask type when creating a VM. */
118	typedef uint64_t hv_vm_options_t;
119	/** Option bitmask when creating a vCPU. */
120	typedef uint64_t hv_vcpu_options_t;
121	/** HV memory protection flags type. */
122	typedef uint64_t hv_memory_flags_t;
123	/** Shadow VMCS protection flags. */
124	typedef uint64_t hv_shadow_flags_t;
125	/** Guest physical address type. */
126	typedef uint64_t hv_gpaddr_t;
127
128
129	/**
130	* VMX Capability enumeration.
131	*/
132	typedef enum
133	{
134	HV_VMX_CAP_PINBASED = 0,
135	HV_VMX_CAP_PROCBASED,
136	HV_VMX_CAP_PROCBASED2,
137	HV_VMX_CAP_ENTRY,
138	HV_VMX_CAP_EXIT,
139	HV_VMX_CAP_PREEMPTION_TIMER = 32
140	} hv_vmx_capability_t;
141
142
143	/**
144	* HV x86 register enumeration.
145	*/
146	typedef enum
147	{
148	HV_X86_RIP = 0,
149	HV_X86_RFLAGS,
150	HV_X86_RAX,
151	HV_X86_RCX,
152	HV_X86_RDX,
153	HV_X86_RBX,
154	HV_X86_RSI,
155	HV_X86_RDI,
156	HV_X86_RSP,
157	HV_X86_RBP,
158	HV_X86_R8,
159	HV_X86_R9,
160	HV_X86_R10,
161	HV_X86_R11,
162	HV_X86_R12,
163	HV_X86_R13,
164	HV_X86_R14,
165	HV_X86_R15,
166	HV_X86_CS,
167	HV_X86_SS,
168	HV_X86_DS,
169	HV_X86_ES,
170	HV_X86_FS,
171	HV_X86_GS,
172	HV_X86_IDT_BASE,
173	HV_X86_IDT_LIMIT,
174	HV_X86_GDT_BASE,
175	HV_X86_GDT_LIMIT,
176	HV_X86_LDTR,
177	HV_X86_LDT_BASE,
178	HV_X86_LDT_LIMIT,
179	HV_X86_LDT_AR,
180	HV_X86_TR,
181	HV_X86_TSS_BASE,
182	HV_X86_TSS_LIMIT,
183	HV_X86_TSS_AR,
184	HV_X86_CR0,
185	HV_X86_CR1,
186	HV_X86_CR2,
187	HV_X86_CR3,
188	HV_X86_CR4,
189	HV_X86_DR0,
190	HV_X86_DR1,
191	HV_X86_DR2,
192	HV_X86_DR3,
193	HV_X86_DR4,
194	HV_X86_DR5,
195	HV_X86_DR6,
196	HV_X86_DR7,
197	HV_X86_TPR,
198	HV_X86_XCR0,
199	HV_X86_REGISTERS_MAX
200	} hv_x86_reg_t;
201
202
203	typedef hv_return_t FN_HV_CAPABILITY(hv_capability_t capability, uint64_t *valu);
204	typedef hv_return_t FN_HV_VM_CREATE(hv_vm_options_t flags);
205	typedef hv_return_t FN_HV_VM_DESTROY(void);
206	typedef hv_return_t FN_HV_VM_SPACE_CREATE(hv_vm_space_t *asid);
207	typedef hv_return_t FN_HV_VM_SPACE_DESTROY(hv_vm_space_t asid);
208	typedef hv_return_t FN_HV_VM_MAP(const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
209	typedef hv_return_t FN_HV_VM_UNMAP(hv_gpaddr_t gpa, size_t size);
210	typedef hv_return_t FN_HV_VM_PROTECT(hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
211	typedef hv_return_t FN_HV_VM_MAP_SPACE(hv_vm_space_t asid, const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
212	typedef hv_return_t FN_HV_VM_UNMAP_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size);
213	typedef hv_return_t FN_HV_VM_PROTECT_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
214	typedef hv_return_t FN_HV_VM_SYNC_TSC(uint64_t tsc);
215
216	typedef hv_return_t FN_HV_VCPU_CREATE(hv_vcpuid_t *vcpu, hv_vcpu_options_t flags);
217	typedef hv_return_t FN_HV_VCPU_DESTROY(hv_vcpuid_t vcpu);
218	typedef hv_return_t FN_HV_VCPU_SET_SPACE(hv_vcpuid_t vcpu, hv_vm_space_t asid);
219	typedef hv_return_t FN_HV_VCPU_READ_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t *value);
220	typedef hv_return_t FN_HV_VCPU_WRITE_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t value);
221	typedef hv_return_t FN_HV_VCPU_READ_FPSTATE(hv_vcpuid_t vcpu, void *buffer, size_t size);
222	typedef hv_return_t FN_HV_VCPU_WRITE_FPSTATE(hv_vcpuid_t vcpu, const void *buffer, size_t size);
223	typedef hv_return_t FN_HV_VCPU_ENABLE_NATIVE_MSR(hv_vcpuid_t vcpu, uint32_t msr, bool enable);
224	typedef hv_return_t FN_HV_VCPU_READ_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t *value);
225	typedef hv_return_t FN_HV_VCPU_WRITE_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t value);
226	typedef hv_return_t FN_HV_VCPU_FLUSH(hv_vcpuid_t vcpu);
227	typedef hv_return_t FN_HV_VCPU_INVALIDATE_TLB(hv_vcpuid_t vcpu);
228	typedef hv_return_t FN_HV_VCPU_RUN(hv_vcpuid_t vcpu);
229	typedef hv_return_t FN_HV_VCPU_RUN_UNTIL(hv_vcpuid_t vcpu, uint64_t deadline);
230	typedef hv_return_t FN_HV_VCPU_INTERRUPT(hv_vcpuid_t *vcpus, unsigned int vcpu_count);
231	typedef hv_return_t FN_HV_VCPU_GET_EXEC_TIME(hv_vcpuid_t vcpus, uint64_t time);
232
233	typedef hv_return_t FN_HV_VMX_VCPU_READ_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
234	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
235
236	typedef hv_return_t FN_HV_VMX_VCPU_READ_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
237	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
238	typedef hv_return_t FN_HV_VMX_VCPU_SET_SHADOW_ACCESS(hv_vcpuid_t vcpu, uint32_t field, hv_shadow_flags_t flags);
239
240	typedef hv_return_t FN_HV_VMX_READ_CAPABILITY(hv_vmx_capability_t field, uint64_t *value);
241	typedef hv_return_t FN_HV_VMX_VCPU_SET_APIC_ADDRESS(hv_vcpuid_t vcpu, hv_gpaddr_t gpa);
242
243
244	/*********************************************************************************************************************************
245	* Global Variables *
246	*********************************************************************************************************************************/
247	/** NEM_DARWIN_PAGE_STATE_XXX names. */
248	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
249	/** MSRs. */
250	static SUPHWVIRTMSRS g_HmMsrs;
251	/** VMX: Set if swapping EFER is supported. */
252	static bool g_fHmVmxSupportsVmcsEfer = false;
253	/** @name APIs imported from Hypervisor.framework.
254	* @{ */
255	static FN_HV_CAPABILITY g_pfnHvCapability = NULL; / Since 10.15 */
256	static FN_HV_VM_CREATE g_pfnHvVmCreate = NULL; / Since 10.10 */
257	static FN_HV_VM_DESTROY g_pfnHvVmDestroy = NULL; / Since 10.10 */
258	static FN_HV_VM_SPACE_CREATE g_pfnHvVmSpaceCreate = NULL; / Since 10.15 */
259	static FN_HV_VM_SPACE_DESTROY g_pfnHvVmSpaceDestroy = NULL; / Since 10.15 */
260	static FN_HV_VM_MAP g_pfnHvVmMap = NULL; / Since 10.10 */
261	static FN_HV_VM_UNMAP g_pfnHvVmUnmap = NULL; / Since 10.10 */
262	static FN_HV_VM_PROTECT g_pfnHvVmProtect = NULL; / Since 10.10 */
263	static FN_HV_VM_MAP_SPACE g_pfnHvVmMapSpace = NULL; / Since 10.15 */
264	static FN_HV_VM_UNMAP_SPACE g_pfnHvVmUnmapSpace = NULL; / Since 10.15 */
265	static FN_HV_VM_PROTECT_SPACE g_pfnHvVmProtectSpace = NULL; / Since 10.15 */
266	static FN_HV_VM_SYNC_TSC g_pfnHvVmSyncTsc = NULL; / Since 10.10 */
267
268	static FN_HV_VCPU_CREATE g_pfnHvVCpuCreate = NULL; / Since 10.10 */
269	static FN_HV_VCPU_DESTROY g_pfnHvVCpuDestroy = NULL; / Since 10.10 */
270	static FN_HV_VCPU_SET_SPACE g_pfnHvVCpuSetSpace = NULL; / Since 10.15 */
271	static FN_HV_VCPU_READ_REGISTER g_pfnHvVCpuReadRegister = NULL; / Since 10.10 */
272	static FN_HV_VCPU_WRITE_REGISTER g_pfnHvVCpuWriteRegister = NULL; / Since 10.10 */
273	static FN_HV_VCPU_READ_FPSTATE g_pfnHvVCpuReadFpState = NULL; / Since 10.10 */
274	static FN_HV_VCPU_WRITE_FPSTATE g_pfnHvVCpuWriteFpState = NULL; / Since 10.10 */
275	static FN_HV_VCPU_ENABLE_NATIVE_MSR g_pfnHvVCpuEnableNativeMsr = NULL; / Since 10.10 */
276	static FN_HV_VCPU_READ_MSR g_pfnHvVCpuReadMsr = NULL; / Since 10.10 */
277	static FN_HV_VCPU_WRITE_MSR g_pfnHvVCpuWriteMsr = NULL; / Since 10.10 */
278	static FN_HV_VCPU_FLUSH g_pfnHvVCpuFlush = NULL; / Since 10.10 */
279	static FN_HV_VCPU_INVALIDATE_TLB g_pfnHvVCpuInvalidateTlb = NULL; / Since 10.10 */
280	static FN_HV_VCPU_RUN g_pfnHvVCpuRun = NULL; / Since 10.10 */
281	static FN_HV_VCPU_RUN_UNTIL g_pfnHvVCpuRunUntil = NULL; / Since 10.15 */
282	static FN_HV_VCPU_INTERRUPT g_pfnHvVCpuInterrupt = NULL; / Since 10.10 */
283	static FN_HV_VCPU_GET_EXEC_TIME g_pfnHvVCpuGetExecTime = NULL; / Since 10.10 */
284
285	static FN_HV_VMX_READ_CAPABILITY g_pfnHvVmxReadCapability = NULL; / Since 10.10 */
286	static FN_HV_VMX_VCPU_READ_VMCS g_pfnHvVmxVCpuReadVmcs = NULL; / Since 10.10 */
287	static FN_HV_VMX_VCPU_WRITE_VMCS g_pfnHvVmxVCpuWriteVmcs = NULL; / Since 10.10 */
288	static FN_HV_VMX_VCPU_READ_SHADOW_VMCS g_pfnHvVmxVCpuReadShadowVmcs = NULL; / Since 10.15 */
289	static FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS g_pfnHvVmxVCpuWriteShadowVmcs = NULL; / Since 10.15 */
290	static FN_HV_VMX_VCPU_SET_SHADOW_ACCESS g_pfnHvVmxVCpuSetShadowAccess = NULL; / Since 10.15 */
291	static FN_HV_VMX_VCPU_SET_APIC_ADDRESS g_pfnHvVmxVCpuSetApicAddress = NULL; / Since 10.10 */
292	/** @} */
293
294
295	/**
296	* Import instructions.
297	*/
298	static const struct
299	{
300	bool fOptional; /*< Set if import is optional. /
301	void ppfn; /< The function pointer variable. */
302	const char pszName; /< The function name. /
303	} g_aImports[] =
304	{
305	#define NEM_DARWIN_IMPORT(a_fOptional, a_Pfn, a_Name) { (a_fOptional), (void **)&(a_Pfn), #a_Name }
306	NEM_DARWIN_IMPORT(true, g_pfnHvCapability, hv_capability),
307	NEM_DARWIN_IMPORT(false, g_pfnHvVmCreate, hv_vm_create),
308	NEM_DARWIN_IMPORT(false, g_pfnHvVmDestroy, hv_vm_destroy),
309	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceCreate, hv_vm_space_create),
310	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceDestroy, hv_vm_space_destroy),
311	NEM_DARWIN_IMPORT(false, g_pfnHvVmMap, hv_vm_map),
312	NEM_DARWIN_IMPORT(false, g_pfnHvVmUnmap, hv_vm_unmap),
313	NEM_DARWIN_IMPORT(false, g_pfnHvVmProtect, hv_vm_protect),
314	NEM_DARWIN_IMPORT(true, g_pfnHvVmMapSpace, hv_vm_map_space),
315	NEM_DARWIN_IMPORT(true, g_pfnHvVmUnmapSpace, hv_vm_unmap_space),
316	NEM_DARWIN_IMPORT(true, g_pfnHvVmProtectSpace, hv_vm_protect_space),
317	NEM_DARWIN_IMPORT(false, g_pfnHvVmSyncTsc, hv_vm_sync_tsc),
318
319	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuCreate, hv_vcpu_create),
320	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuDestroy, hv_vcpu_destroy),
321	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuSetSpace, hv_vcpu_set_space),
322	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadRegister, hv_vcpu_read_register),
323	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteRegister, hv_vcpu_write_register),
324	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadFpState, hv_vcpu_read_fpstate),
325	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteFpState, hv_vcpu_write_fpstate),
326	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuEnableNativeMsr, hv_vcpu_enable_native_msr),
327	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadMsr, hv_vcpu_read_msr),
328	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteMsr, hv_vcpu_write_msr),
329	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuFlush, hv_vcpu_flush),
330	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInvalidateTlb, hv_vcpu_invalidate_tlb),
331	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuRun, hv_vcpu_run),
332	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuRunUntil, hv_vcpu_run_until),
333	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInterrupt, hv_vcpu_interrupt),
334	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuGetExecTime, hv_vcpu_get_exec_time),
335	NEM_DARWIN_IMPORT(false, g_pfnHvVmxReadCapability, hv_vmx_read_capability),
336	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuReadVmcs, hv_vmx_vcpu_read_vmcs),
337	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuWriteVmcs, hv_vmx_vcpu_write_vmcs),
338	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuReadShadowVmcs, hv_vmx_vcpu_read_shadow_vmcs),
339	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuWriteShadowVmcs, hv_vmx_vcpu_write_shadow_vmcs),
340	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuSetShadowAccess, hv_vmx_vcpu_set_shadow_access),
341	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuSetApicAddress, hv_vmx_vcpu_set_apic_address),
342	#undef NEM_DARWIN_IMPORT
343	};
344
345
346	/*
347	* Let the preprocessor alias the APIs to import variables for better autocompletion.
348	*/
349	#ifndef IN_SLICKEDIT
350	# define hv_capability g_pfnHvCapability
351	# define hv_vm_create g_pfnHvVmCreate
352	# define hv_vm_destroy g_pfnHvVmDestroy
353	# define hv_vm_space_create g_pfnHvVmSpaceCreate
354	# define hv_vm_space_destroy g_pfnHvVmSpaceDestroy
355	# define hv_vm_map g_pfnHvVmMap
356	# define hv_vm_unmap g_pfnHvVmUnmap
357	# define hv_vm_protect g_pfnHvVmProtect
358	# define hv_vm_map_space g_pfnHvVmMapSpace
359	# define hv_vm_unmap_space g_pfnHvVmUnmapSpace
360	# define hv_vm_protect_space g_pfnHvVmProtectSpace
361	# define hv_vm_sync_tsc g_pfnHvVmSyncTsc
362
363	# define hv_vcpu_create g_pfnHvVCpuCreate
364	# define hv_vcpu_destroy g_pfnHvVCpuDestroy
365	# define hv_vcpu_set_space g_pfnHvVCpuSetSpace
366	# define hv_vcpu_read_register g_pfnHvVCpuReadRegister
367	# define hv_vcpu_write_register g_pfnHvVCpuWriteRegister
368	# define hv_vcpu_read_fpstate g_pfnHvVCpuReadFpState
369	# define hv_vcpu_write_fpstate g_pfnHvVCpuWriteFpState
370	# define hv_vcpu_enable_native_msr g_pfnHvVCpuEnableNativeMsr
371	# define hv_vcpu_read_msr g_pfnHvVCpuReadMsr
372	# define hv_vcpu_write_msr g_pfnHvVCpuWriteMsr
373	# define hv_vcpu_flush g_pfnHvVCpuFlush
374	# define hv_vcpu_invalidate_tlb g_pfnHvVCpuInvalidateTlb
375	# define hv_vcpu_run g_pfnHvVCpuRun
376	# define hv_vcpu_run_until g_pfnHvVCpuRunUntil
377	# define hv_vcpu_interrupt g_pfnHvVCpuInterrupt
378	# define hv_vcpu_get_exec_time g_pfnHvVCpuGetExecTime
379
380	# define hv_vmx_read_capability g_pfnHvVmxReadCapability
381	# define hv_vmx_vcpu_read_vmcs g_pfnHvVmxVCpuReadVmcs
382	# define hv_vmx_vcpu_write_vmcs g_pfnHvVmxVCpuWriteVmcs
383	# define hv_vmx_vcpu_read_shadow_vmcs g_pfnHvVmxVCpuReadShadowVmcs
384	# define hv_vmx_vcpu_write_shadow_vmcs g_pfnHvVmxVCpuWriteShadowVmcs
385	# define hv_vmx_vcpu_set_shadow_access g_pfnHvVmxVCpuSetShadowAccess
386	# define hv_vmx_vcpu_set_apic_address g_pfnHvVmxVCpuSetApicAddress
387	#endif
388
389
390	/*********************************************************************************************************************************
391	* Internal Functions *
392	*********************************************************************************************************************************/
393
394	/**
395	* Converts a HV return code to a VBox status code.
396	*
397	* @returns VBox status code.
398	* @param hrc The HV return code to convert.
399	*/
400	DECLINLINE(int) nemR3DarwinHvSts2Rc(hv_return_t hrc)
401	{
402	if (hrc == HV_SUCCESS)
403	return VINF_SUCCESS;
404
405	switch (hrc)
406	{
407	case HV_ERROR: return VERR_INVALID_STATE;
408	case HV_BUSY: return VERR_RESOURCE_BUSY;
409	case HV_BAD_ARGUMENT: return VERR_INVALID_PARAMETER;
410	case HV_NO_RESOURCES: return VERR_OUT_OF_RESOURCES;
411	case HV_NO_DEVICE: return VERR_NOT_FOUND;
412	case HV_UNSUPPORTED: return VERR_NOT_SUPPORTED;
413	}
414
415	return VERR_IPE_UNEXPECTED_STATUS;
416	}
417
418
419	/**
420	* Unmaps the given guest physical address range (page aligned).
421	*
422	* @returns VBox status code.
423	* @param GCPhys The guest physical address to start unmapping at.
424	* @param cb The size of the range to unmap in bytes.
425	*/
426	DECLINLINE(int) nemR3DarwinUnmap(RTGCPHYS GCPhys, size_t cb)
427	{
428	LogFlowFunc(("Unmapping %RGp LB %zu\n", GCPhys, cb));
429	hv_return_t hrc = hv_vm_unmap(GCPhys, cb);
430	return nemR3DarwinHvSts2Rc(hrc);
431	}
432
433
434	/**
435	* Maps a given guest physical address range backed by the given memory with the given
436	* protection flags.
437	*
438	* @returns VBox status code.
439	* @param GCPhys The guest physical address to start mapping.
440	* @param pvRam The R3 pointer of the memory to back the range with.
441	* @param cb The size of the range, page aligned.
442	* @param fPageProt The page protection flags to use for this range, combination of NEM_PAGE_PROT_XXX
443	*/
444	DECLINLINE(int) nemR3DarwinMap(RTGCPHYS GCPhys, void *pvRam, size_t cb, uint32_t fPageProt)
445	{
446	LogFlowFunc(("Mapping %RGp LB %zu fProt=%#x\n", GCPhys, cb, fPageProt));
447
448	hv_memory_flags_t fHvMemProt = 0;
449	if (fPageProt & NEM_PAGE_PROT_READ)
450	fHvMemProt \|= HV_MEMORY_READ;
451	if (fPageProt & NEM_PAGE_PROT_WRITE)
452	fHvMemProt \|= HV_MEMORY_WRITE;
453	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
454	fHvMemProt \|= HV_MEMORY_EXEC;
455
456	hv_return_t hrc = hv_vm_map(pvRam, GCPhys, cb, fHvMemProt);
457	return nemR3DarwinHvSts2Rc(hrc);
458	}
459
460
461	#if 0 /* unused */
462	DECLINLINE(int) nemR3DarwinProtectPage(RTGCPHYS GCPhys, size_t cb, uint32_t fPageProt)
463	{
464	hv_memory_flags_t fHvMemProt = 0;
465	if (fPageProt & NEM_PAGE_PROT_READ)
466	fHvMemProt \|= HV_MEMORY_READ;
467	if (fPageProt & NEM_PAGE_PROT_WRITE)
468	fHvMemProt \|= HV_MEMORY_WRITE;
469	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
470	fHvMemProt \|= HV_MEMORY_EXEC;
471
472	hv_return_t hrc = hv_vm_protect(GCPhys, cb, fHvMemProt);
473	return nemR3DarwinHvSts2Rc(hrc);
474	}
475	#endif
476
477
478	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
479	{
480	PGMPAGEMAPLOCK Lock;
481	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
482	if (RT_SUCCESS(rc))
483	PGMPhysReleasePageMappingLock(pVM, &Lock);
484	return rc;
485	}
486
487
488	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
489	{
490	PGMPAGEMAPLOCK Lock;
491	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
492	if (RT_SUCCESS(rc))
493	PGMPhysReleasePageMappingLock(pVM, &Lock);
494	return rc;
495	}
496
497
498	/**
499	* Worker that maps pages into Hyper-V.
500	*
501	* This is used by the PGM physical page notifications as well as the memory
502	* access VMEXIT handlers.
503	*
504	* @returns VBox status code.
505	* @param pVM The cross context VM structure.
506	* @param pVCpu The cross context virtual CPU structure of the
507	* calling EMT.
508	* @param GCPhysSrc The source page address.
509	* @param GCPhysDst The hyper-V destination page. This may differ from
510	* GCPhysSrc when A20 is disabled.
511	* @param fPageProt NEM_PAGE_PROT_XXX.
512	* @param pu2State Our page state (input/output).
513	* @param fBackingChanged Set if the page backing is being changed.
514	* @thread EMT(pVCpu)
515	*/
516	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
517	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
518	{
519	/*
520	* Looks like we need to unmap a page before we can change the backing
521	* or even modify the protection. This is going to be REALLY efficient.
522	* PGM lends us two bits to keep track of the state here.
523	*/
524	RT_NOREF(pVCpu);
525	uint8_t const u2OldState = *pu2State;
526	uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_DARWIN_PAGE_STATE_WRITABLE
527	: fPageProt & NEM_PAGE_PROT_READ ? NEM_DARWIN_PAGE_STATE_READABLE : NEM_DARWIN_PAGE_STATE_UNMAPPED;
528	if ( fBackingChanged
529	\|\| u2NewState != u2OldState)
530	{
531	if (u2OldState > NEM_DARWIN_PAGE_STATE_UNMAPPED)
532	{
533	int rc = nemR3DarwinUnmap(GCPhysDst, X86_PAGE_SIZE);
534	if (RT_SUCCESS(rc))
535	{
536	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
537	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
538	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
539	if (u2NewState == NEM_DARWIN_PAGE_STATE_UNMAPPED)
540	{
541	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
542	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
543	return VINF_SUCCESS;
544	}
545	}
546	else
547	{
548	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
549	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
550	return VERR_NEM_INIT_FAILED;
551	}
552	}
553	}
554
555	/*
556	* Writeable mapping?
557	*/
558	if (fPageProt & NEM_PAGE_PROT_WRITE)
559	{
560	void *pvPage;
561	int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
562	if (RT_SUCCESS(rc))
563	{
564	rc = nemR3DarwinMap(GCPhysDst, pvPage, X86_PAGE_SIZE, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
565	if (RT_SUCCESS(rc))
566	{
567	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
568	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
569	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
570	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
571	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
572	return VINF_SUCCESS;
573	}
574	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
575	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst));
576	return VERR_NEM_INIT_FAILED;
577	}
578	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
579	return rc;
580	}
581
582	if (fPageProt & NEM_PAGE_PROT_READ)
583	{
584	const void *pvPage;
585	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
586	if (RT_SUCCESS(rc))
587	{
588	rc = nemR3DarwinMap(GCPhysDst, (void *)pvPage, X86_PAGE_SIZE, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE);
589	if (RT_SUCCESS(rc))
590	{
591	*pu2State = NEM_DARWIN_PAGE_STATE_READABLE;
592	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
593	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
594	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
595	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
596	return VINF_SUCCESS;
597	}
598	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
599	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
600	return VERR_NEM_INIT_FAILED;
601	}
602	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
603	return rc;
604	}
605
606	/* We already unmapped it above. */
607	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
608	return VINF_SUCCESS;
609	}
610
611
612	#ifdef LOG_ENABLED
613	/**
614	* Logs the current CPU state.
615	*/
616	static void nemR3DarwinLogState(PVMCC pVM, PVMCPUCC pVCpu)
617	{
618	if (LogIs3Enabled())
619	{
620	#if 0
621	char szRegs[4096];
622	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
623	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
624	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
625	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
626	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
627	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
628	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
629	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
630	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
631	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
632	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
633	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
634	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
635	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
636	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
637	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
638	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
639	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
640	" efer=%016VR{efer}\n"
641	" pat=%016VR{pat}\n"
642	" sf_mask=%016VR{sf_mask}\n"
643	"krnl_gs_base=%016VR{krnl_gs_base}\n"
644	" lstar=%016VR{lstar}\n"
645	" star=%016VR{star} cstar=%016VR{cstar}\n"
646	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
647	);
648
649	char szInstr[256];
650	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
651	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
652	szInstr, sizeof(szInstr), NULL);
653	Log3(("%s%s\n", szRegs, szInstr));
654	#else
655	RT_NOREF(pVM, pVCpu);
656	#endif
657	}
658	}
659	#endif /* LOG_ENABLED */
660
661
662	DECLINLINE(int) nemR3DarwinReadVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t *pData)
663	{
664	uint64_t u64Data;
665	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
666	if (RT_LIKELY(hrc == HV_SUCCESS))
667	{
668	*pData = (uint16_t)u64Data;
669	return VINF_SUCCESS;
670	}
671
672	return nemR3DarwinHvSts2Rc(hrc);
673	}
674
675
676	DECLINLINE(int) nemR3DarwinReadVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t *pData)
677	{
678	uint64_t u64Data;
679	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
680	if (RT_LIKELY(hrc == HV_SUCCESS))
681	{
682	*pData = (uint32_t)u64Data;
683	return VINF_SUCCESS;
684	}
685
686	return nemR3DarwinHvSts2Rc(hrc);
687	}
688
689
690	DECLINLINE(int) nemR3DarwinReadVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t *pData)
691	{
692	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, pData);
693	if (RT_LIKELY(hrc == HV_SUCCESS))
694	return VINF_SUCCESS;
695
696	return nemR3DarwinHvSts2Rc(hrc);
697	}
698
699
700	DECLINLINE(int) nemR3DarwinWriteVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t u16Val)
701	{
702	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u16Val);
703	if (RT_LIKELY(hrc == HV_SUCCESS))
704	return VINF_SUCCESS;
705
706	return nemR3DarwinHvSts2Rc(hrc);
707	}
708
709
710	DECLINLINE(int) nemR3DarwinWriteVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t u32Val)
711	{
712	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u32Val);
713	if (RT_LIKELY(hrc == HV_SUCCESS))
714	return VINF_SUCCESS;
715
716	return nemR3DarwinHvSts2Rc(hrc);
717	}
718
719
720	DECLINLINE(int) nemR3DarwinWriteVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t u64Val)
721	{
722	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u64Val);
723	if (RT_LIKELY(hrc == HV_SUCCESS))
724	return VINF_SUCCESS;
725
726	return nemR3DarwinHvSts2Rc(hrc);
727	}
728
729	DECLINLINE(int) nemR3DarwinMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Val)
730	{
731	hv_return_t hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, idMsr, pu64Val);
732	if (RT_LIKELY(hrc == HV_SUCCESS))
733	return VINF_SUCCESS;
734
735	return nemR3DarwinHvSts2Rc(hrc);
736	}
737
738	#if 0 /unused/
739	DECLINLINE(int) nemR3DarwinMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Val)
740	{
741	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, idMsr, u64Val);
742	if (RT_LIKELY(hrc == HV_SUCCESS))
743	return VINF_SUCCESS;
744
745	return nemR3DarwinHvSts2Rc(hrc);
746	}
747	#endif
748
749	static int nemR3DarwinCopyStateFromHv(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
750	{
751	#define READ_GREG(a_GReg, a_Value) \
752	do \
753	{ \
754	hrc = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, (a_GReg), &(a_Value)); \
755	if (RT_LIKELY(hrc == HV_SUCCESS)) \
756	{ /* likely */ } \
757	else \
758	return VERR_INTERNAL_ERROR; \
759	} while(0)
760	#define READ_VMCS_FIELD(a_Field, a_Value) \
761	do \
762	{ \
763	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &(a_Value)); \
764	if (RT_LIKELY(hrc == HV_SUCCESS)) \
765	{ /* likely */ } \
766	else \
767	return VERR_INTERNAL_ERROR; \
768	} while(0)
769	#define READ_VMCS16_FIELD(a_Field, a_Value) \
770	do \
771	{ \
772	uint64_t u64Data; \
773	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
774	if (RT_LIKELY(hrc == HV_SUCCESS)) \
775	{ (a_Value) = (uint16_t)u64Data; } \
776	else \
777	return VERR_INTERNAL_ERROR; \
778	} while(0)
779	#define READ_VMCS32_FIELD(a_Field, a_Value) \
780	do \
781	{ \
782	uint64_t u64Data; \
783	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
784	if (RT_LIKELY(hrc == HV_SUCCESS)) \
785	{ (a_Value) = (uint32_t)u64Data; } \
786	else \
787	return VERR_INTERNAL_ERROR; \
788	} while(0)
789	#define READ_MSR(a_Msr, a_Value) \
790	do \
791	{ \
792	hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, (a_Msr), &(a_Value)); \
793	if (RT_LIKELY(hrc == HV_SUCCESS)) \
794	{ /* likely */ } \
795	else \
796	AssertFailedReturn(VERR_INTERNAL_ERROR); \
797	} while(0)
798
799	RT_NOREF(pVM);
800	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
801
802	/* GPRs */
803	hv_return_t hrc;
804	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
805	{
806	if (fWhat & CPUMCTX_EXTRN_RAX)
807	READ_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
808	if (fWhat & CPUMCTX_EXTRN_RCX)
809	READ_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
810	if (fWhat & CPUMCTX_EXTRN_RDX)
811	READ_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
812	if (fWhat & CPUMCTX_EXTRN_RBX)
813	READ_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
814	if (fWhat & CPUMCTX_EXTRN_RSP)
815	READ_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
816	if (fWhat & CPUMCTX_EXTRN_RBP)
817	READ_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
818	if (fWhat & CPUMCTX_EXTRN_RSI)
819	READ_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
820	if (fWhat & CPUMCTX_EXTRN_RDI)
821	READ_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
822	if (fWhat & CPUMCTX_EXTRN_R8_R15)
823	{
824	READ_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
825	READ_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
826	READ_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
827	READ_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
828	READ_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
829	READ_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
830	READ_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
831	READ_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
832	}
833	}
834
835	/* RIP & Flags */
836	if (fWhat & CPUMCTX_EXTRN_RIP)
837	READ_GREG(HV_X86_RIP, pVCpu->cpum.GstCtx.rip);
838	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
839	READ_GREG(HV_X86_RFLAGS, pVCpu->cpum.GstCtx.rflags.u);
840
841	/* Segments */
842	#define READ_SEG(a_SReg, a_enmName) \
843	do { \
844	READ_VMCS16_FIELD(VMX_VMCS16_GUEST_ ## a_enmName ## _SEL, (a_SReg).Sel); \
845	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _LIMIT, (a_SReg).u32Limit); \
846	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _ACCESS_RIGHTS, (a_SReg).Attr.u); \
847	READ_VMCS_FIELD(VMX_VMCS_GUEST_ ## a_enmName ## _BASE, (a_SReg).u64Base); \
848	(a_SReg).ValidSel = (a_SReg).Sel; \
849	} while (0)
850	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
851	{
852	if (fWhat & CPUMCTX_EXTRN_ES)
853	READ_SEG(pVCpu->cpum.GstCtx.es, ES);
854	if (fWhat & CPUMCTX_EXTRN_CS)
855	READ_SEG(pVCpu->cpum.GstCtx.cs, CS);
856	if (fWhat & CPUMCTX_EXTRN_SS)
857	READ_SEG(pVCpu->cpum.GstCtx.ss, SS);
858	if (fWhat & CPUMCTX_EXTRN_DS)
859	READ_SEG(pVCpu->cpum.GstCtx.ds, DS);
860	if (fWhat & CPUMCTX_EXTRN_FS)
861	READ_SEG(pVCpu->cpum.GstCtx.fs, FS);
862	if (fWhat & CPUMCTX_EXTRN_GS)
863	READ_SEG(pVCpu->cpum.GstCtx.gs, GS);
864	}
865
866	/* Descriptor tables and the task segment. */
867	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
868	{
869	if (fWhat & CPUMCTX_EXTRN_LDTR)
870	READ_SEG(pVCpu->cpum.GstCtx.ldtr, LDTR);
871
872	if (fWhat & CPUMCTX_EXTRN_TR)
873	{
874	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
875	avoid to trigger sanity assertions around the code, always fix this. */
876	READ_SEG(pVCpu->cpum.GstCtx.tr, TR);
877	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
878	{
879	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
880	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
881	break;
882	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
883	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
884	break;
885	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
886	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
887	break;
888	}
889	}
890	if (fWhat & CPUMCTX_EXTRN_IDTR)
891	{
892	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_IDTR_LIMIT, pVCpu->cpum.GstCtx.idtr.cbIdt);
893	READ_VMCS_FIELD(VMX_VMCS_GUEST_IDTR_BASE, pVCpu->cpum.GstCtx.idtr.pIdt);
894	}
895	if (fWhat & CPUMCTX_EXTRN_GDTR)
896	{
897	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_GDTR_LIMIT, pVCpu->cpum.GstCtx.gdtr.cbGdt);
898	READ_VMCS_FIELD(VMX_VMCS_GUEST_GDTR_BASE, pVCpu->cpum.GstCtx.gdtr.pGdt);
899	}
900	}
901
902	/* Control registers. */
903	bool fMaybeChangedMode = false;
904	bool fUpdateCr3 = false;
905	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
906	{
907	uint64_t u64CrTmp = 0;
908
909	if (fWhat & CPUMCTX_EXTRN_CR0)
910	{
911	READ_GREG(HV_X86_CR0, u64CrTmp);
912	if (pVCpu->cpum.GstCtx.cr0 != u64CrTmp)
913	{
914	CPUMSetGuestCR0(pVCpu, u64CrTmp);
915	fMaybeChangedMode = true;
916	}
917	}
918	if (fWhat & CPUMCTX_EXTRN_CR2)
919	READ_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
920	if (fWhat & CPUMCTX_EXTRN_CR3)
921	{
922	READ_GREG(HV_X86_CR3, u64CrTmp);
923	if (pVCpu->cpum.GstCtx.cr3 != u64CrTmp)
924	{
925	CPUMSetGuestCR3(pVCpu, u64CrTmp);
926	fUpdateCr3 = true;
927	}
928	}
929	if (fWhat & CPUMCTX_EXTRN_CR4)
930	{
931	READ_GREG(HV_X86_CR4, u64CrTmp);
932	u64CrTmp &= ~VMX_V_CR4_FIXED0;
933
934	if (pVCpu->cpum.GstCtx.cr4 != u64CrTmp)
935	{
936	CPUMSetGuestCR4(pVCpu, u64CrTmp);
937	fMaybeChangedMode = true;
938	}
939	}
940	}
941	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
942	{
943	uint64_t u64Cr8 = 0;
944
945	READ_GREG(HV_X86_TPR, u64Cr8);
946	APICSetTpr(pVCpu, u64Cr8);
947	}
948
949	/* Debug registers. */
950	if (fWhat & CPUMCTX_EXTRN_DR7)
951	{
952	uint64_t u64Dr7;
953	READ_GREG(HV_X86_DR7, u64Dr7);
954	if (pVCpu->cpum.GstCtx.dr[7] != u64Dr7)
955	CPUMSetGuestDR7(pVCpu, u64Dr7);
956	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
957	}
958	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
959	{
960	uint64_t u64DrTmp;
961
962	READ_GREG(HV_X86_DR0, u64DrTmp);
963	if (pVCpu->cpum.GstCtx.dr[0] != u64DrTmp)
964	CPUMSetGuestDR0(pVCpu, u64DrTmp);
965	READ_GREG(HV_X86_DR1, u64DrTmp);
966	if (pVCpu->cpum.GstCtx.dr[1] != u64DrTmp)
967	CPUMSetGuestDR1(pVCpu, u64DrTmp);
968	READ_GREG(HV_X86_DR3, u64DrTmp);
969	if (pVCpu->cpum.GstCtx.dr[2] != u64DrTmp)
970	CPUMSetGuestDR2(pVCpu, u64DrTmp);
971	READ_GREG(HV_X86_DR3, u64DrTmp);
972	if (pVCpu->cpum.GstCtx.dr[3] != u64DrTmp)
973	CPUMSetGuestDR3(pVCpu, u64DrTmp);
974	}
975	if (fWhat & CPUMCTX_EXTRN_DR6)
976	{
977	uint64_t u64Dr6;
978	READ_GREG(HV_X86_DR7, u64Dr6);
979	if (pVCpu->cpum.GstCtx.dr[6] != u64Dr6)
980	CPUMSetGuestDR6(pVCpu, u64Dr6);
981	}
982
983	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
984	{
985	hrc = hv_vcpu_read_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
986	if (hrc == HV_SUCCESS)
987	{ /* likely */ }
988	else
989	return nemR3DarwinHvSts2Rc(hrc);
990	}
991
992	/* MSRs */
993	if (fWhat & CPUMCTX_EXTRN_EFER)
994	{
995	uint64_t u64Efer;
996
997	READ_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, u64Efer);
998	if (u64Efer != pVCpu->cpum.GstCtx.msrEFER)
999	{
1000	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, u64Efer));
1001	if ((u64Efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1002	PGMNotifyNxeChanged(pVCpu, RT_BOOL(u64Efer & MSR_K6_EFER_NXE));
1003	pVCpu->cpum.GstCtx.msrEFER = u64Efer;
1004	fMaybeChangedMode = true;
1005	}
1006	}
1007
1008	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1009	READ_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1010	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1011	{
1012	uint64_t u64Tmp;
1013	READ_MSR(MSR_IA32_SYSENTER_EIP, u64Tmp);
1014	pVCpu->cpum.GstCtx.SysEnter.eip = u64Tmp;
1015	READ_MSR(MSR_IA32_SYSENTER_ESP, u64Tmp);
1016	pVCpu->cpum.GstCtx.SysEnter.esp = u64Tmp;
1017	READ_MSR(MSR_IA32_SYSENTER_CS, u64Tmp);
1018	pVCpu->cpum.GstCtx.SysEnter.cs = u64Tmp;
1019	}
1020	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1021	{
1022	READ_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1023	READ_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1024	READ_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1025	READ_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1026	}
1027	#if 0
1028	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1029	{
1030	Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
1031	const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
1032	if (aValues[iReg].Reg64 != uOldBase)
1033	{
1034	Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
1035	pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
1036	int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
1037	AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
1038	}
1039	iReg++;
1040
1041	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
1042	#if 0 /def LOG_ENABLED/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
1043	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
1044	#endif
1045	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1046	GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType, WHvX64RegisterMsrMtrrDefType, "MSR MTRR_DEF_TYPE");
1047	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
1048	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
1049	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
1050	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000, WHvX64RegisterMsrMtrrFix4kC0000, "MSR MTRR_FIX_4K_C0000");
1051	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000, WHvX64RegisterMsrMtrrFix4kC8000, "MSR MTRR_FIX_4K_C8000");
1052	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000, WHvX64RegisterMsrMtrrFix4kD0000, "MSR MTRR_FIX_4K_D0000");
1053	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000, WHvX64RegisterMsrMtrrFix4kD8000, "MSR MTRR_FIX_4K_D8000");
1054	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000, WHvX64RegisterMsrMtrrFix4kE0000, "MSR MTRR_FIX_4K_E0000");
1055	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000, WHvX64RegisterMsrMtrrFix4kE8000, "MSR MTRR_FIX_4K_E8000");
1056	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000, WHvX64RegisterMsrMtrrFix4kF0000, "MSR MTRR_FIX_4K_F0000");
1057	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000, WHvX64RegisterMsrMtrrFix4kF8000, "MSR MTRR_FIX_4K_F8000");
1058	GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX");
1059	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
1060	}
1061	#endif
1062
1063	/* Almost done, just update extrn flags and maybe change PGM mode. */
1064	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1065	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1066	pVCpu->cpum.GstCtx.fExtrn = 0;
1067
1068	/* Typical. */
1069	if (!fMaybeChangedMode && !fUpdateCr3)
1070	return VINF_SUCCESS;
1071
1072	/*
1073	* Slow.
1074	*/
1075	if (fMaybeChangedMode)
1076	{
1077	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
1078	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1079	}
1080
1081	if (fUpdateCr3)
1082	{
1083	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3, false /fPdpesMapped/);
1084	if (rc == VINF_SUCCESS)
1085	{ /* likely */ }
1086	else
1087	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1088	}
1089
1090	return VINF_SUCCESS;
1091	#undef READ_GREG
1092	#undef READ_VMCS_FIELD
1093	#undef READ_VMCS32_FIELD
1094	#undef READ_SEG
1095	#undef READ_MSR
1096	}
1097
1098
1099	/**
1100	* State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
1101	* and nemHCWinHandleMemoryAccessPageCheckerCallback.
1102	*/
1103	typedef struct NEMHCDARWINHMACPCCSTATE
1104	{
1105	/** Input: Write access. */
1106	bool fWriteAccess;
1107	/** Output: Set if we did something. */
1108	bool fDidSomething;
1109	/** Output: Set it we should resume. */
1110	bool fCanResume;
1111	} NEMHCDARWINHMACPCCSTATE;
1112
1113	/**
1114	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1115	* Worker for nemR3WinHandleMemoryAccess; pvUser points to a
1116	* NEMHCDARWINHMACPCCSTATE structure. }
1117	*/
1118	static DECLCALLBACK(int)
1119	nemR3DarwinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1120	{
1121	NEMHCDARWINHMACPCCSTATE pState = (NEMHCDARWINHMACPCCSTATE )pvUser;
1122	pState->fDidSomething = false;
1123	pState->fCanResume = false;
1124
1125	uint8_t u2State = pInfo->u2NemState;
1126
1127	/*
1128	* Consolidate current page state with actual page protection and access type.
1129	* We don't really consider downgrades here, as they shouldn't happen.
1130	*/
1131	int rc;
1132	switch (u2State)
1133	{
1134	case NEM_DARWIN_PAGE_STATE_UNMAPPED:
1135	case NEM_DARWIN_PAGE_STATE_NOT_SET:
1136	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1137	{
1138	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1139	return VINF_SUCCESS;
1140	}
1141
1142	/* Don't bother remapping it if it's a write request to a non-writable page. */
1143	if ( pState->fWriteAccess
1144	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1145	{
1146	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1147	return VINF_SUCCESS;
1148	}
1149
1150	/* Map the page. */
1151	rc = nemHCNativeSetPhysPage(pVM,
1152	pVCpu,
1153	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1154	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1155	pInfo->fNemProt,
1156	&u2State,
1157	true /fBackingState/);
1158	pInfo->u2NemState = u2State;
1159	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1160	GCPhys, g_apszPageStates[u2State], rc));
1161	pState->fDidSomething = true;
1162	pState->fCanResume = true;
1163	return rc;
1164
1165	case NEM_DARWIN_PAGE_STATE_READABLE:
1166	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1167	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1168	{
1169	pState->fCanResume = true;
1170	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1171	return VINF_SUCCESS;
1172	}
1173	break;
1174
1175	case NEM_DARWIN_PAGE_STATE_WRITABLE:
1176	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1177	{
1178	/* We get spurious EPT exit violations when everything is fine (#3a case) but can resume without issues here... */
1179	pState->fCanResume = true;
1180	if (pInfo->u2OldNemState == NEM_DARWIN_PAGE_STATE_WRITABLE)
1181	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
1182	else
1183	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
1184	GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
1185	return VINF_SUCCESS;
1186	}
1187
1188	break;
1189
1190	default:
1191	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1192	}
1193
1194	/*
1195	* Unmap and restart the instruction.
1196	* If this fails, which it does every so often, just unmap everything for now.
1197	*/
1198	rc = nemR3DarwinUnmap(GCPhys, X86_PAGE_SIZE);
1199	if (RT_SUCCESS(rc))
1200	{
1201	pState->fDidSomething = true;
1202	pState->fCanResume = true;
1203	pInfo->u2NemState = NEM_DARWIN_PAGE_STATE_UNMAPPED;
1204	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
1205	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
1206	Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
1207	return VINF_SUCCESS;
1208	}
1209	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
1210	LogRel(("nemR3DarwinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s rc=%Rrc\n",
1211	GCPhys, g_apszPageStates[u2State], rc));
1212	return VERR_NEM_UNMAP_PAGES_FAILED;
1213	}
1214
1215
1216	DECL_FORCE_INLINE(bool) vmxHCShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1217	{
1218	RT_NOREF(pVCpu, pVmxTransient);
1219	return true;
1220	}
1221
1222
1223	DECL_FORCE_INLINE(bool) nemR3DarwinIsUnrestrictedGuest(PCVMCC pVM)
1224	{
1225	RT_NOREF(pVM);
1226	return true;
1227	}
1228
1229
1230	DECL_FORCE_INLINE(bool) nemR3DarwinIsNestedPaging(PCVMCC pVM)
1231	{
1232	RT_NOREF(pVM);
1233	return true;
1234	}
1235
1236
1237	DECL_FORCE_INLINE(bool) nemR3DarwinIsPreemptTimerUsed(PCVMCC pVM)
1238	{
1239	RT_NOREF(pVM);
1240	return false;
1241	}
1242
1243
1244	DECL_FORCE_INLINE(bool) nemR3DarwinIsVmxLbr(PCVMCC pVM)
1245	{
1246	RT_NOREF(pVM);
1247	return false;
1248	}
1249
1250
1251	/*
1252	* Instantiate the code we share with ring-0.
1253	*/
1254	//#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
1255	#define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
1256	#define VCPU_2_VMXSTATE(a_pVCpu) (a_pVCpu)->nem.s
1257	#define VM_IS_VMX_UNRESTRICTED_GUEST(a_pVM) nemR3DarwinIsUnrestrictedGuest((a_pVM))
1258	#define VM_IS_VMX_NESTED_PAGING(a_pVM) nemR3DarwinIsNestedPaging((a_pVM))
1259	#define VM_IS_VMX_PREEMPT_TIMER_USED(a_pVM) nemR3DarwinIsPreemptTimerUsed((a_pVM))
1260	#define VM_IS_VMX_LBR(a_pVM) nemR3DarwinIsVmxLbr((a_pVM))
1261
1262	#define VMX_VMCS_WRITE_16(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs16((a_pVCpu), (a_FieldEnc), (a_Val))
1263	#define VMX_VMCS_WRITE_32(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs32((a_pVCpu), (a_FieldEnc), (a_Val))
1264	#define VMX_VMCS_WRITE_64(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1265	#define VMX_VMCS_WRITE_NW(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1266
1267	#define VMX_VMCS_READ_16(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs16((a_pVCpu), (a_FieldEnc), (a_pVal))
1268	#define VMX_VMCS_READ_32(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs32((a_pVCpu), (a_FieldEnc), (a_pVal))
1269	#define VMX_VMCS_READ_64(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1270	#define VMX_VMCS_READ_NW(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1271
1272	#include "../VMMAll/VMXAllTemplate.cpp.h"
1273
1274	#undef VMX_VMCS_WRITE_16
1275	#undef VMX_VMCS_WRITE_32
1276	#undef VMX_VMCS_WRITE_64
1277	#undef VMX_VMCS_WRITE_NW
1278
1279	#undef VMX_VMCS_READ_16
1280	#undef VMX_VMCS_READ_32
1281	#undef VMX_VMCS_READ_64
1282	#undef VMX_VMCS_READ_NW
1283
1284	#undef VM_IS_VMX_PREEMPT_TIMER_USED
1285	#undef VM_IS_VMX_NESTED_PAGING
1286	#undef VM_IS_VMX_UNRESTRICTED_GUEST
1287	#undef VCPU_2_VMXSTATE
1288
1289
1290	/**
1291	* Exports the guest GP registers to HV for execution.
1292	*
1293	* @returns VBox status code.
1294	* @param pVCpu The cross context virtual CPU structure of the
1295	* calling EMT.
1296	*/
1297	static int nemR3DarwinExportGuestGprs(PVMCPUCC pVCpu)
1298	{
1299	#define WRITE_GREG(a_GReg, a_Value) \
1300	do \
1301	{ \
1302	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1303	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1304	{ /* likely */ } \
1305	else \
1306	return VERR_INTERNAL_ERROR; \
1307	} while(0)
1308
1309	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->nem.s.fCtxChanged);
1310	if (fCtxChanged & HM_CHANGED_GUEST_GPRS_MASK)
1311	{
1312	if (fCtxChanged & HM_CHANGED_GUEST_RAX)
1313	WRITE_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
1314	if (fCtxChanged & HM_CHANGED_GUEST_RCX)
1315	WRITE_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
1316	if (fCtxChanged & HM_CHANGED_GUEST_RDX)
1317	WRITE_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
1318	if (fCtxChanged & HM_CHANGED_GUEST_RBX)
1319	WRITE_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
1320	if (fCtxChanged & HM_CHANGED_GUEST_RSP)
1321	WRITE_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
1322	if (fCtxChanged & HM_CHANGED_GUEST_RBP)
1323	WRITE_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
1324	if (fCtxChanged & HM_CHANGED_GUEST_RSI)
1325	WRITE_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
1326	if (fCtxChanged & HM_CHANGED_GUEST_RDI)
1327	WRITE_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
1328	if (fCtxChanged & HM_CHANGED_GUEST_R8_R15)
1329	{
1330	WRITE_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
1331	WRITE_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
1332	WRITE_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
1333	WRITE_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
1334	WRITE_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
1335	WRITE_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
1336	WRITE_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
1337	WRITE_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
1338	}
1339
1340	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_GPRS_MASK);
1341	}
1342
1343	if (fCtxChanged & HM_CHANGED_GUEST_CR2)
1344	{
1345	WRITE_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
1346	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_CR2);
1347	}
1348
1349	return VINF_SUCCESS;
1350	#undef WRITE_GREG
1351	}
1352
1353
1354	/**
1355	* Converts the given CPUM externalized bitmask to the appropriate HM changed bitmask.
1356	*
1357	* @returns Bitmask of HM changed flags.
1358	* @param fCpumExtrn The CPUM extern bitmask.
1359	*/
1360	static uint64_t nemR3DarwinCpumExtrnToHmChanged(uint64_t fCpumExtrn)
1361	{
1362	uint64_t fHmChanged = 0;
1363
1364	/* Invert to gt a mask of things which are kept in CPUM. */
1365	uint64_t fCpumIntern = ~fCpumExtrn;
1366
1367	if (fCpumIntern & CPUMCTX_EXTRN_GPRS_MASK)
1368	{
1369	if (fCpumIntern & CPUMCTX_EXTRN_RAX)
1370	fHmChanged \|= HM_CHANGED_GUEST_RAX;
1371	if (fCpumIntern & CPUMCTX_EXTRN_RCX)
1372	fHmChanged \|= HM_CHANGED_GUEST_RCX;
1373	if (fCpumIntern & CPUMCTX_EXTRN_RDX)
1374	fHmChanged \|= HM_CHANGED_GUEST_RDX;
1375	if (fCpumIntern & CPUMCTX_EXTRN_RBX)
1376	fHmChanged \|= HM_CHANGED_GUEST_RBX;
1377	if (fCpumIntern & CPUMCTX_EXTRN_RSP)
1378	fHmChanged \|= HM_CHANGED_GUEST_RSP;
1379	if (fCpumIntern & CPUMCTX_EXTRN_RBP)
1380	fHmChanged \|= HM_CHANGED_GUEST_RBP;
1381	if (fCpumIntern & CPUMCTX_EXTRN_RSI)
1382	fHmChanged \|= HM_CHANGED_GUEST_RSI;
1383	if (fCpumIntern & CPUMCTX_EXTRN_RDI)
1384	fHmChanged \|= HM_CHANGED_GUEST_RDI;
1385	if (fCpumIntern & CPUMCTX_EXTRN_R8_R15)
1386	fHmChanged \|= HM_CHANGED_GUEST_R8_R15;
1387	}
1388
1389	/* RIP & Flags */
1390	if (fCpumIntern & CPUMCTX_EXTRN_RIP)
1391	fHmChanged \|= HM_CHANGED_GUEST_RIP;
1392	if (fCpumIntern & CPUMCTX_EXTRN_RFLAGS)
1393	fHmChanged \|= HM_CHANGED_GUEST_RFLAGS;
1394
1395	/* Segments */
1396	if (fCpumIntern & CPUMCTX_EXTRN_SREG_MASK)
1397	{
1398	if (fCpumIntern & CPUMCTX_EXTRN_ES)
1399	fHmChanged \|= HM_CHANGED_GUEST_ES;
1400	if (fCpumIntern & CPUMCTX_EXTRN_CS)
1401	fHmChanged \|= HM_CHANGED_GUEST_CS;
1402	if (fCpumIntern & CPUMCTX_EXTRN_SS)
1403	fHmChanged \|= HM_CHANGED_GUEST_SS;
1404	if (fCpumIntern & CPUMCTX_EXTRN_DS)
1405	fHmChanged \|= HM_CHANGED_GUEST_DS;
1406	if (fCpumIntern & CPUMCTX_EXTRN_FS)
1407	fHmChanged \|= HM_CHANGED_GUEST_FS;
1408	if (fCpumIntern & CPUMCTX_EXTRN_GS)
1409	fHmChanged \|= HM_CHANGED_GUEST_GS;
1410	}
1411
1412	/* Descriptor tables & task segment. */
1413	if (fCpumIntern & CPUMCTX_EXTRN_TABLE_MASK)
1414	{
1415	if (fCpumIntern & CPUMCTX_EXTRN_LDTR)
1416	fHmChanged \|= HM_CHANGED_GUEST_LDTR;
1417	if (fCpumIntern & CPUMCTX_EXTRN_TR)
1418	fHmChanged \|= HM_CHANGED_GUEST_TR;
1419	if (fCpumIntern & CPUMCTX_EXTRN_IDTR)
1420	fHmChanged \|= HM_CHANGED_GUEST_IDTR;
1421	if (fCpumIntern & CPUMCTX_EXTRN_GDTR)
1422	fHmChanged \|= HM_CHANGED_GUEST_GDTR;
1423	}
1424
1425	/* Control registers. */
1426	if (fCpumIntern & CPUMCTX_EXTRN_CR_MASK)
1427	{
1428	if (fCpumIntern & CPUMCTX_EXTRN_CR0)
1429	fHmChanged \|= HM_CHANGED_GUEST_CR0;
1430	if (fCpumIntern & CPUMCTX_EXTRN_CR2)
1431	fHmChanged \|= HM_CHANGED_GUEST_CR2;
1432	if (fCpumIntern & CPUMCTX_EXTRN_CR3)
1433	fHmChanged \|= HM_CHANGED_GUEST_CR3;
1434	if (fCpumIntern & CPUMCTX_EXTRN_CR4)
1435	fHmChanged \|= HM_CHANGED_GUEST_CR4;
1436	}
1437	if (fCpumIntern & CPUMCTX_EXTRN_APIC_TPR)
1438	fHmChanged \|= HM_CHANGED_GUEST_APIC_TPR;
1439
1440	/* Debug registers. */
1441	if (fCpumIntern & CPUMCTX_EXTRN_DR0_DR3)
1442	fHmChanged \|= HM_CHANGED_GUEST_DR0_DR3;
1443	if (fCpumIntern & CPUMCTX_EXTRN_DR6)
1444	fHmChanged \|= HM_CHANGED_GUEST_DR6;
1445	if (fCpumIntern & CPUMCTX_EXTRN_DR7)
1446	fHmChanged \|= HM_CHANGED_GUEST_DR7;
1447
1448	/* Floating point state. */
1449	if (fCpumIntern & CPUMCTX_EXTRN_X87)
1450	fHmChanged \|= HM_CHANGED_GUEST_X87;
1451	if (fCpumIntern & CPUMCTX_EXTRN_SSE_AVX)
1452	fHmChanged \|= HM_CHANGED_GUEST_SSE_AVX;
1453	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_XSAVE)
1454	fHmChanged \|= HM_CHANGED_GUEST_OTHER_XSAVE;
1455	if (fCpumIntern & CPUMCTX_EXTRN_XCRx)
1456	fHmChanged \|= HM_CHANGED_GUEST_XCRx;
1457
1458	/* MSRs */
1459	if (fCpumIntern & CPUMCTX_EXTRN_EFER)
1460	fHmChanged \|= HM_CHANGED_GUEST_EFER_MSR;
1461	if (fCpumIntern & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1462	fHmChanged \|= HM_CHANGED_GUEST_KERNEL_GS_BASE;
1463	if (fCpumIntern & CPUMCTX_EXTRN_SYSENTER_MSRS)
1464	fHmChanged \|= HM_CHANGED_GUEST_SYSENTER_MSR_MASK;
1465	if (fCpumIntern & CPUMCTX_EXTRN_SYSCALL_MSRS)
1466	fHmChanged \|= HM_CHANGED_GUEST_SYSCALL_MSRS;
1467	if (fCpumIntern & CPUMCTX_EXTRN_TSC_AUX)
1468	fHmChanged \|= HM_CHANGED_GUEST_TSC_AUX;
1469	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_MSRS)
1470	fHmChanged \|= HM_CHANGED_GUEST_OTHER_MSRS;
1471
1472	return fHmChanged;
1473	}
1474
1475
1476	/**
1477	* Exports the guest state to HV for execution.
1478	*
1479	* @returns VBox status code.
1480	* @param pVM The cross context VM structure.
1481	* @param pVCpu The cross context virtual CPU structure of the
1482	* calling EMT.
1483	* @param pVmxTransient The transient VMX structure.
1484	*/
1485	static int nemR3DarwinExportGuestState(PVMCC pVM, PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1486	{
1487	#define WRITE_GREG(a_GReg, a_Value) \
1488	do \
1489	{ \
1490	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1491	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1492	{ /* likely */ } \
1493	else \
1494	return VERR_INTERNAL_ERROR; \
1495	} while(0)
1496	#define WRITE_VMCS_FIELD(a_Field, a_Value) \
1497	do \
1498	{ \
1499	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), (a_Value)); \
1500	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1501	{ /* likely */ } \
1502	else \
1503	return VERR_INTERNAL_ERROR; \
1504	} while(0)
1505	#define WRITE_MSR(a_Msr, a_Value) \
1506	do \
1507	{ \
1508	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, (a_Msr), (a_Value)); \
1509	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1510	{ /* likely */ } \
1511	else \
1512	AssertFailedReturn(VERR_INTERNAL_ERROR); \
1513	} while(0)
1514
1515	RT_NOREF(pVM);
1516
1517	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL;
1518	if (!fWhat)
1519	return VINF_SUCCESS;
1520
1521	pVCpu->nem.s.fCtxChanged \|= nemR3DarwinCpumExtrnToHmChanged(pVCpu->cpum.GstCtx.fExtrn);
1522
1523	int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
1524	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1525
1526	rc = nemR3DarwinExportGuestGprs(pVCpu);
1527	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1528
1529	rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
1530	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1531
1532	VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
1533	if (rcStrict == VINF_SUCCESS)
1534	{ /* likely */ }
1535	else
1536	{
1537	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
1538	return VBOXSTRICTRC_VAL(rcStrict);
1539	}
1540
1541	vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
1542	vmxHCExportGuestRip(pVCpu);
1543	//vmxHCExportGuestRsp(pVCpu);
1544	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
1545
1546	rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
1547	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1548
1549	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1550	WRITE_GREG(HV_X86_TPR, CPUMGetGuestCR8(pVCpu));
1551
1552	/* Debug registers. */
1553	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1554	{
1555	WRITE_GREG(HV_X86_DR0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
1556	WRITE_GREG(HV_X86_DR1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
1557	WRITE_GREG(HV_X86_DR2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
1558	WRITE_GREG(HV_X86_DR3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
1559	}
1560	if (fWhat & CPUMCTX_EXTRN_DR6)
1561	WRITE_GREG(HV_X86_DR6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
1562	if (fWhat & CPUMCTX_EXTRN_DR7)
1563	WRITE_GREG(HV_X86_DR7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
1564
1565	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
1566	{
1567	hv_return_t hrc = hv_vcpu_write_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1568	if (hrc == HV_SUCCESS)
1569	{ /* likely */ }
1570	else
1571	return nemR3DarwinHvSts2Rc(hrc);
1572	}
1573
1574	/* MSRs */
1575	if (fWhat & CPUMCTX_EXTRN_EFER)
1576	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, pVCpu->cpum.GstCtx.msrEFER);
1577	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1578	WRITE_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1579	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1580	{
1581	WRITE_MSR(MSR_IA32_SYSENTER_CS, pVCpu->cpum.GstCtx.SysEnter.cs);
1582	WRITE_MSR(MSR_IA32_SYSENTER_EIP, pVCpu->cpum.GstCtx.SysEnter.eip);
1583	WRITE_MSR(MSR_IA32_SYSENTER_ESP, pVCpu->cpum.GstCtx.SysEnter.esp);
1584	}
1585	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1586	{
1587	WRITE_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1588	WRITE_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1589	WRITE_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1590	WRITE_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1591	}
1592	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1593	{
1594	hv_return_t hrc = hv_vmx_vcpu_set_apic_address(pVCpu->nem.s.hVCpuId, APICGetBaseMsrNoCheck(pVCpu) & PAGE_BASE_GC_MASK);
1595	if (RT_UNLIKELY(hrc != HV_SUCCESS))
1596	return nemR3DarwinHvSts2Rc(hrc);
1597
1598	#if 0
1599	ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
1600	#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
1601	ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
1602	#endif
1603	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1604	ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
1605	ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
1606	ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
1607	ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
1608	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000, pCtxMsrs->msr.MtrrFix4K_C0000);
1609	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000, pCtxMsrs->msr.MtrrFix4K_C8000);
1610	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000, pCtxMsrs->msr.MtrrFix4K_D0000);
1611	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000, pCtxMsrs->msr.MtrrFix4K_D8000);
1612	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000, pCtxMsrs->msr.MtrrFix4K_E0000);
1613	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000, pCtxMsrs->msr.MtrrFix4K_E8000);
1614	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000, pCtxMsrs->msr.MtrrFix4K_F0000);
1615	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000, pCtxMsrs->msr.MtrrFix4K_F8000);
1616	ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
1617	#if 0 /** @todo these registers aren't available? Might explain something.. .*/
1618	const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
1619	if (enmCpuVendor != CPUMCPUVENDOR_AMD)
1620	{
1621	ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
1622	ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
1623	}
1624	#endif
1625	#endif
1626	}
1627
1628	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0 /MSR_IA32_DEBUGCTL_LBR/);
1629
1630	#if 0 /** @todo */
1631	WRITE_GREG(HV_X86_TSS_BASE, );
1632	WRITE_GREG(HV_X86_TSS_LIMIT, );
1633	WRITE_GREG(HV_X86_TSS_AR, );
1634	WRITE_GREG(HV_X86_XCR0, );
1635	#endif
1636
1637	hv_vcpu_invalidate_tlb(pVCpu->nem.s.hVCpuId);
1638	hv_vcpu_flush(pVCpu->nem.s.hVCpuId);
1639
1640	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_NEM;
1641
1642	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
1643	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
1644	\| HM_CHANGED_GUEST_CR2
1645	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
1646	\| HM_CHANGED_GUEST_X87
1647	\| HM_CHANGED_GUEST_SSE_AVX
1648	\| HM_CHANGED_GUEST_OTHER_XSAVE
1649	\| HM_CHANGED_GUEST_XCRx
1650	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
1651	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
1652	\| HM_CHANGED_GUEST_TSC_AUX
1653	\| HM_CHANGED_GUEST_OTHER_MSRS
1654	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
1655
1656	return VINF_SUCCESS;
1657	#undef WRITE_GREG
1658	#undef WRITE_VMCS_FIELD
1659	}
1660
1661
1662	/**
1663	* Handles an exit from hv_vcpu_run().
1664	*
1665	* @returns VBox strict status code.
1666	* @param pVM The cross context VM structure.
1667	* @param pVCpu The cross context virtual CPU structure of the
1668	* calling EMT.
1669	* @param pVmxTransient The transient VMX structure.
1670	*/
1671	static VBOXSTRICTRC nemR3DarwinHandleExit(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1672	{
1673	uint32_t uExitReason;
1674	int rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
1675	AssertRC(rc);
1676	pVmxTransient->fVmcsFieldsRead = 0;
1677	pVmxTransient->fIsNestedGuest = false;
1678	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
1679	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
1680
1681	if (RT_UNLIKELY(pVmxTransient->fVMEntryFailed))
1682	AssertLogRelMsgFailedReturn(("Running guest failed for CPU #%u: %#x %u\n",
1683	pVCpu->idCpu, pVmxTransient->uExitReason, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
1684	VERR_NEM_IPE_0);
1685
1686	/** @todo Only copy the state on demand (requires changing to adhere to fCtxChanged from th VMX code
1687	* flags instead of the fExtrn one living in CPUM.
1688	*/
1689	rc = nemR3DarwinCopyStateFromHv(pVM, pVCpu, UINT64_MAX);
1690	AssertRCReturn(rc, rc);
1691
1692	#ifndef HMVMX_USE_FUNCTION_TABLE
1693	return vmxHCHandleExit(pVCpu, pVmxTransient);
1694	#else
1695	return g_aVMExitHandlers[pVmxTransient->uExitReason].pfn(pVCpu, pVmxTransient);
1696	#endif
1697	}
1698
1699
1700	/**
1701	* Worker for nemR3NativeInit that loads the Hypervisor.framwork shared library.
1702	*
1703	* @returns VBox status code.
1704	* @param fForced Whether the HMForced flag is set and we should
1705	* fail if we cannot initialize.
1706	* @param pErrInfo Where to always return error info.
1707	*/
1708	static int nemR3DarwinLoadHv(bool fForced, PRTERRINFO pErrInfo)
1709	{
1710	RTLDRMOD hMod = NIL_RTLDRMOD;
1711	static const char *s_pszHvPath = "/System/Library/Frameworks/Hypervisor.framework/Hypervisor";
1712
1713	int rc = RTLdrLoadEx(s_pszHvPath, &hMod, RTLDRLOAD_FLAGS_NO_UNLOAD \| RTLDRLOAD_FLAGS_NO_SUFFIX, pErrInfo);
1714	if (RT_SUCCESS(rc))
1715	{
1716	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
1717	{
1718	int rc2 = RTLdrGetSymbol(hMod, g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
1719	if (RT_SUCCESS(rc2))
1720	{
1721	if (g_aImports[i].fOptional)
1722	LogRel(("NEM: info: Found optional import Hypervisor!%s.\n",
1723	g_aImports[i].pszName));
1724	}
1725	else
1726	{
1727	*g_aImports[i].ppfn = NULL;
1728
1729	LogRel(("NEM: %s: Failed to import Hypervisor!%s: %Rrc\n",
1730	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
1731	g_aImports[i].pszName, rc2));
1732	if (!g_aImports[i].fOptional)
1733	{
1734	if (RTErrInfoIsSet(pErrInfo))
1735	RTErrInfoAddF(pErrInfo, rc2, ", Hypervisor!%s", g_aImports[i].pszName);
1736	else
1737	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: Hypervisor!%s", g_aImports[i].pszName);
1738	Assert(RT_FAILURE(rc));
1739	}
1740	}
1741	}
1742	if (RT_SUCCESS(rc))
1743	{
1744	Assert(!RTErrInfoIsSet(pErrInfo));
1745	}
1746
1747	RTLdrClose(hMod);
1748	}
1749	else
1750	{
1751	RTErrInfoAddF(pErrInfo, rc, "Failed to load Hypervisor.framwork: %s: %Rrc", s_pszHvPath, rc);
1752	rc = VERR_NEM_INIT_FAILED;
1753	}
1754
1755	return rc;
1756	}
1757
1758
1759	/**
1760	* Read and initialize the global capabilities supported by this CPU.
1761	*
1762	* @returns VBox status code.
1763	*/
1764	static int nemR3DarwinCapsInit(void)
1765	{
1766	RT_ZERO(g_HmMsrs);
1767
1768	hv_return_t hrc = hv_vmx_read_capability(HV_VMX_CAP_PINBASED, &g_HmMsrs.u.vmx.PinCtls.u);
1769	if (hrc == HV_SUCCESS)
1770	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED, &g_HmMsrs.u.vmx.ProcCtls.u);
1771	#if 0 /* Not available with our SDK. */
1772	if (hrc == HV_SUCCESS)
1773	hrc = hv_vmx_read_capability(HV_VMX_CAP_BASIC, &g_HmMsrs.u.vmx.u64Basic);
1774	#endif
1775	if (hrc == HV_SUCCESS)
1776	hrc = hv_vmx_read_capability(HV_VMX_CAP_ENTRY, &g_HmMsrs.u.vmx.EntryCtls.u);
1777	if (hrc == HV_SUCCESS)
1778	hrc = hv_vmx_read_capability(HV_VMX_CAP_EXIT, &g_HmMsrs.u.vmx.ExitCtls.u);
1779	#if 0 /* Not available with our SDK. */
1780	if (hrc == HV_SUCCESS)
1781	hrc = hv_vmx_read_capability(HV_VMX_CAP_MISC, &g_HmMsrs.u.vmx.u64Misc);
1782	if (hrc == HV_SUCCESS)
1783	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED0, &g_HmMsrs.u.vmx.u64Cr0Fixed0);
1784	if (hrc == HV_SUCCESS)
1785	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED1, &g_HmMsrs.u.vmx.u64Cr0Fixed1);
1786	if (hrc == HV_SUCCESS)
1787	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED0, &g_HmMsrs.u.vmx.u64Cr4Fixed0);
1788	if (hrc == HV_SUCCESS)
1789	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED1, &g_HmMsrs.u.vmx.u64Cr4Fixed1);
1790	if (hrc == HV_SUCCESS)
1791	hrc = hv_vmx_read_capability(HV_VMX_CAP_VMCS_ENUM, &g_HmMsrs.u.vmx.u64VmcsEnum);
1792	if ( hrc == HV_SUCCESS
1793	&& RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_TRUE_CTLS))
1794	{
1795	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PINBASED, &g_HmMsrs.u.vmx.TruePinCtls.u);
1796	if (hrc == HV_SUCCESS)
1797	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PROCBASED, &g_HmMsrs.u.vmx.TrueProcCtls.u);
1798	if (hrc == HV_SUCCESS)
1799	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_ENTRY, &g_HmMsrs.u.vmx.TrueEntryCtls.u);
1800	if (hrc == HV_SUCCESS)
1801	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_EXIT, &g_HmMsrs.u.vmx.TrueExitCtls.u);
1802	}
1803	#else /** @todo Not available with the current SDK used (available with 11.0+) but required for setting the CRx values properly. */
1804	g_HmMsrs.u.vmx.u64Cr0Fixed0 = 0x80000021;
1805	g_HmMsrs.u.vmx.u64Cr0Fixed1 = 0xffffffff;
1806	g_HmMsrs.u.vmx.u64Cr4Fixed0 = 0x2000;
1807	g_HmMsrs.u.vmx.u64Cr4Fixed1 = 0x1767ff;
1808	#endif
1809
1810	if ( hrc == HV_SUCCESS
1811	&& g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1812	{
1813	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED2, &g_HmMsrs.u.vmx.ProcCtls2.u);
1814
1815	#if 0 /* Not available with our SDK. */
1816	if ( hrc == HV_SUCCESS
1817	& g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & (VMX_PROC_CTLS2_EPT \| VMX_PROC_CTLS2_VPID))
1818	hrc = hv_vmx_read_capability(HV_VMX_CAP_EPT_VPID_CAP, &g_HmMsrs.u.vmx.u64EptVpidCaps);
1819	#endif
1820	g_HmMsrs.u.vmx.u64VmFunc = 0; /* No way to read that on macOS. */
1821	}
1822
1823	if (hrc == HV_SUCCESS)
1824	{
1825	/*
1826	* Check for EFER swapping support.
1827	*/
1828	g_fHmVmxSupportsVmcsEfer = true; //(g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
1829	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1830	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
1831	}
1832
1833	return nemR3DarwinHvSts2Rc(hrc);
1834	}
1835
1836
1837	/**
1838	* Sets up pin-based VM-execution controls in the VMCS.
1839	*
1840	* @returns VBox status code.
1841	* @param pVCpu The cross context virtual CPU structure.
1842	* @param pVmcsInfo The VMCS info. object.
1843	*/
1844	static int nemR3DarwinVmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1845	{
1846	//PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1847	uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
1848	uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1849
1850	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
1851	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
1852
1853	#if 0 /** @todo Use preemption timer */
1854	/* Enable the VMX-preemption timer. */
1855	if (pVM->hmr0.s.vmx.fUsePreemptTimer)
1856	{
1857	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
1858	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
1859	}
1860
1861	/* Enable posted-interrupt processing. */
1862	if (pVM->hm.s.fPostedIntrs)
1863	{
1864	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
1865	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
1866	fVal \|= VMX_PIN_CTLS_POSTED_INT;
1867	}
1868	#endif
1869
1870	if ((fVal & fZap) != fVal)
1871	{
1872	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1873	g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
1874	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1875	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1876	}
1877
1878	/* Commit it to the VMCS and update our cache. */
1879	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, fVal);
1880	AssertRC(rc);
1881	pVmcsInfo->u32PinCtls = fVal;
1882
1883	return VINF_SUCCESS;
1884	}
1885
1886
1887	/**
1888	* Sets up secondary processor-based VM-execution controls in the VMCS.
1889	*
1890	* @returns VBox status code.
1891	* @param pVCpu The cross context virtual CPU structure.
1892	* @param pVmcsInfo The VMCS info. object.
1893	*/
1894	static int nemR3DarwinVmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1895	{
1896	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1897	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
1898	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1899
1900	/* WBINVD causes a VM-exit. */
1901	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
1902	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
1903
1904	/* Enable the INVPCID instruction if we expose it to the guest and is supported
1905	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
1906	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
1907	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
1908	fVal \|= VMX_PROC_CTLS2_INVPCID;
1909
1910	#if 0 /** @todo */
1911	/* Enable VPID. */
1912	if (pVM->hmr0.s.vmx.fVpid)
1913	fVal \|= VMX_PROC_CTLS2_VPID;
1914
1915	if (pVM->hm.s.fVirtApicRegs)
1916	{
1917	/* Enable APIC-register virtualization. */
1918	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
1919	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
1920
1921	/* Enable virtual-interrupt delivery. */
1922	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
1923	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
1924	}
1925
1926	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
1927	where the TPR shadow resides. */
1928	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1929	* done dynamically. */
1930	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1931	{
1932	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
1933	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
1934	}
1935	#endif
1936
1937	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
1938	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
1939	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
1940	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
1941	fVal \|= VMX_PROC_CTLS2_RDTSCP;
1942
1943	#if 0
1944	/* Enable Pause-Loop exiting. */
1945	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
1946	&& pVM->hm.s.vmx.cPleGapTicks
1947	&& pVM->hm.s.vmx.cPleWindowTicks)
1948	{
1949	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
1950
1951	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
1952	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
1953	}
1954	#endif
1955
1956	if ((fVal & fZap) != fVal)
1957	{
1958	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1959	g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
1960	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
1961	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1962	}
1963
1964	/* Commit it to the VMCS and update our cache. */
1965	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
1966	AssertRC(rc);
1967	pVmcsInfo->u32ProcCtls2 = fVal;
1968
1969	return VINF_SUCCESS;
1970	}
1971
1972
1973	/**
1974	* Enables native access for the given MSR.
1975	*
1976	* @returns VBox status code.
1977	* @param pVCpu The cross context virtual CPU structure.
1978	* @param idMsr The MSR to enable native access for.
1979	*/
1980	static int nemR3DarwinMsrSetNative(PVMCPUCC pVCpu, uint32_t idMsr)
1981	{
1982	hv_return_t hrc = hv_vcpu_enable_native_msr(pVCpu->nem.s.hVCpuId, idMsr, true /enable/);
1983	if (hrc == HV_SUCCESS)
1984	return VINF_SUCCESS;
1985
1986	return nemR3DarwinHvSts2Rc(hrc);
1987	}
1988
1989
1990	/**
1991	* Sets up the MSR permissions which don't change through the lifetime of the VM.
1992	*
1993	* @returns VBox status code.
1994	* @param pVCpu The cross context virtual CPU structure.
1995	* @param pVmcsInfo The VMCS info. object.
1996	*/
1997	static int nemR3DarwinSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1998	{
1999	RT_NOREF(pVmcsInfo);
2000
2001	/*
2002	* The guest can access the following MSRs (read, write) without causing
2003	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2004	*/
2005	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2006	int rc;
2007	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_CS); AssertRCReturn(rc, rc);
2008	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_ESP); AssertRCReturn(rc, rc);
2009	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_EIP); AssertRCReturn(rc, rc);
2010	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_GS_BASE); AssertRCReturn(rc, rc);
2011	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_FS_BASE); AssertRCReturn(rc, rc);
2012
2013	/*
2014	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2015	* associated with then. We never need to intercept access (writes need to be
2016	* executed without causing a VM-exit, reads will #GP fault anyway).
2017	*
2018	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2019	* read/write them. We swap the guest/host MSR value using the
2020	* auto-load/store MSR area.
2021	*/
2022	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2023	{
2024	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_PRED_CMD);
2025	AssertRCReturn(rc, rc);
2026	}
2027	#if 0 /* Doesn't work. */
2028	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2029	{
2030	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_FLUSH_CMD);
2031	AssertRCReturn(rc, rc);
2032	}
2033	#endif
2034	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2035	{
2036	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SPEC_CTRL);
2037	AssertRCReturn(rc, rc);
2038	}
2039
2040	/*
2041	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2042	* required for 64-bit guests.
2043	*/
2044	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_LSTAR); AssertRCReturn(rc, rc);
2045	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K6_STAR); AssertRCReturn(rc, rc);
2046	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_SF_MASK); AssertRCReturn(rc, rc);
2047	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_KERNEL_GS_BASE); AssertRCReturn(rc, rc);
2048
2049	/* Required for enabling the RDTSCP instruction. */
2050	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_TSC_AUX); AssertRCReturn(rc, rc);
2051
2052	return VINF_SUCCESS;
2053	}
2054
2055
2056	/**
2057	* Sets up processor-based VM-execution controls in the VMCS.
2058	*
2059	* @returns VBox status code.
2060	* @param pVCpu The cross context virtual CPU structure.
2061	* @param pVmcsInfo The VMCS info. object.
2062	*/
2063	static int nemR3DarwinVmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2064	{
2065	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2066	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2067	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2068
2069	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2070	// \| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2071	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2072	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2073	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2074	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2075	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2076
2077	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2078	if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2079	\|\| (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2080	{
2081	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2082	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2083	}
2084
2085	/* Use TPR shadowing if supported by the CPU. */
2086	if ( PDMHasApic(pVM)
2087	&& (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
2088	{
2089	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2090	/* CR8 writes cause a VM-exit based on TPR threshold. */
2091	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2092	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2093	}
2094	else
2095	{
2096	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2097	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2098	}
2099
2100	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2101	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2102	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2103
2104	if ((fVal & fZap) != fVal)
2105	{
2106	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2107	g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2108	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2109	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2110	}
2111
2112	/* Commit it to the VMCS and update our cache. */
2113	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2114	AssertRC(rc);
2115	pVmcsInfo->u32ProcCtls = fVal;
2116
2117	/* Set up MSR permissions that don't change through the lifetime of the VM. */
2118	rc = nemR3DarwinSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2119	AssertRCReturn(rc, rc);
2120
2121	/*
2122	* Set up secondary processor-based VM-execution controls
2123	* (we assume the CPU to always support it as we rely on unrestricted guest execution support).
2124	*/
2125	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
2126	return nemR3DarwinVmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2127	}
2128
2129
2130	/**
2131	* Sets up miscellaneous (everything other than Pin, Processor and secondary
2132	* Processor-based VM-execution) control fields in the VMCS.
2133	*
2134	* @returns VBox status code.
2135	* @param pVCpu The cross context virtual CPU structure.
2136	* @param pVmcsInfo The VMCS info. object.
2137	*/
2138	static int nemR3DarwinVmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2139	{
2140	int rc = VINF_SUCCESS;
2141	//rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo); TODO
2142	if (RT_SUCCESS(rc))
2143	{
2144	uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2145	uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2146
2147	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2148	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2149
2150	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2151	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2152
2153	#if 0 /** @todo */
2154	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fLbr)
2155	{
2156	rc = VMXWriteVmcsNw(VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2157	AssertRC(rc);
2158	}
2159	#endif
2160	return VINF_SUCCESS;
2161	}
2162	else
2163	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2164	return rc;
2165	}
2166
2167
2168	/**
2169	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2170	*
2171	* We shall setup those exception intercepts that don't change during the
2172	* lifetime of the VM here. The rest are done dynamically while loading the
2173	* guest state.
2174	*
2175	* @param pVCpu The cross context virtual CPU structure.
2176	* @param pVmcsInfo The VMCS info. object.
2177	*/
2178	static void nemR3DarwinVmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2179	{
2180	/*
2181	* The following exceptions are always intercepted:
2182	*
2183	* #AC - To prevent the guest from hanging the CPU and for dealing with
2184	* split-lock detecting host configs.
2185	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2186	* recursive #DBs can cause a CPU hang.
2187	*/
2188	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2189	\| RT_BIT(X86_XCPT_DB);
2190
2191	/* Commit it to the VMCS. */
2192	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2193	AssertRC(rc);
2194
2195	/* Update our cache of the exception bitmap. */
2196	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2197	}
2198
2199
2200	/**
2201	* Initialize the VMCS information field for the given vCPU.
2202	*
2203	* @returns VBox status code.
2204	* @param pVCpu The cross context virtual CPU structure of the
2205	* calling EMT.
2206	*/
2207	static int nemR3DarwinInitVmcs(PVMCPU pVCpu)
2208	{
2209	int rc = nemR3DarwinVmxSetupVmcsPinCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2210	if (RT_SUCCESS(rc))
2211	{
2212	rc = nemR3DarwinVmxSetupVmcsProcCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2213	if (RT_SUCCESS(rc))
2214	{
2215	rc = nemR3DarwinVmxSetupVmcsMiscCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2216	if (RT_SUCCESS(rc))
2217	{
2218	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &pVCpu->nem.s.VmcsInfo.u32EntryCtls);
2219	if (RT_SUCCESS(rc))
2220	{
2221	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_EXIT, &pVCpu->nem.s.VmcsInfo.u32ExitCtls);
2222	if (RT_SUCCESS(rc))
2223	{
2224	nemR3DarwinVmxSetupVmcsXcptBitmap(pVCpu, &pVCpu->nem.s.VmcsInfo);
2225	return VINF_SUCCESS;
2226	}
2227	else
2228	LogRelFunc(("Failed to read the exit controls. rc=%Rrc\n", rc));
2229	}
2230	else
2231	LogRelFunc(("Failed to read the entry controls. rc=%Rrc\n", rc));
2232	}
2233	else
2234	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
2235	}
2236	else
2237	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
2238	}
2239	else
2240	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
2241
2242	return rc;
2243	}
2244
2245
2246	/**
2247	* Try initialize the native API.
2248	*
2249	* This may only do part of the job, more can be done in
2250	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
2251	*
2252	* @returns VBox status code.
2253	* @param pVM The cross context VM structure.
2254	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
2255	* the latter we'll fail if we cannot initialize.
2256	* @param fForced Whether the HMForced flag is set and we should
2257	* fail if we cannot initialize.
2258	*/
2259	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
2260	{
2261	AssertReturn(!pVM->nem.s.fCreatedVm, VERR_WRONG_ORDER);
2262
2263	/*
2264	* Some state init.
2265	*/
2266
2267	/*
2268	* Error state.
2269	* The error message will be non-empty on failure and 'rc' will be set too.
2270	*/
2271	RTERRINFOSTATIC ErrInfo;
2272	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
2273	int rc = nemR3DarwinLoadHv(fForced, pErrInfo);
2274	if (RT_SUCCESS(rc))
2275	{
2276	hv_return_t hrc = hv_vm_create(HV_VM_DEFAULT);
2277	if (hrc == HV_SUCCESS)
2278	{
2279	pVM->nem.s.fCreatedVm = true;
2280
2281	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
2282	Log(("NEM: Marked active!\n"));
2283	PGMR3EnableNemMode(pVM);
2284
2285	/* Register release statistics */
2286	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2287	{
2288	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
2289	STAMR3RegisterF(pVM, &pNemCpu->StatExitPortIo, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of port I/O exits", "/NEM/CPU%u/ExitPortIo", idCpu);
2290	STAMR3RegisterF(pVM, &pNemCpu->StatExitMemUnmapped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unmapped memory exits", "/NEM/CPU%u/ExitMemUnmapped", idCpu);
2291	STAMR3RegisterF(pVM, &pNemCpu->StatExitMemIntercept, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of intercepted memory exits", "/NEM/CPU%u/ExitMemIntercept", idCpu);
2292	STAMR3RegisterF(pVM, &pNemCpu->StatExitHalt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of HLT exits", "/NEM/CPU%u/ExitHalt", idCpu);
2293	STAMR3RegisterF(pVM, &pNemCpu->StatExitInterruptWindow, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of HLT exits", "/NEM/CPU%u/ExitInterruptWindow", idCpu);
2294	STAMR3RegisterF(pVM, &pNemCpu->StatExitCpuId, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of CPUID exits", "/NEM/CPU%u/ExitCpuId", idCpu);
2295	STAMR3RegisterF(pVM, &pNemCpu->StatExitMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of MSR access exits", "/NEM/CPU%u/ExitMsr", idCpu);
2296	STAMR3RegisterF(pVM, &pNemCpu->StatExitException, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of exception exits", "/NEM/CPU%u/ExitException", idCpu);
2297	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionBp, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #BP exits", "/NEM/CPU%u/ExitExceptionBp", idCpu);
2298	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionDb, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #DB exits", "/NEM/CPU%u/ExitExceptionDb", idCpu);
2299	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionGp, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #GP exits", "/NEM/CPU%u/ExitExceptionGp", idCpu);
2300	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionGpMesa, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #GP exits from mesa driver", "/NEM/CPU%u/ExitExceptionGpMesa", idCpu);
2301	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUd, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #UD exits", "/NEM/CPU%u/ExitExceptionUd", idCpu);
2302	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUdHandled, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of handled #UD exits", "/NEM/CPU%u/ExitExceptionUdHandled", idCpu);
2303	STAMR3RegisterF(pVM, &pNemCpu->StatExitUnrecoverable, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unrecoverable exits", "/NEM/CPU%u/ExitUnrecoverable", idCpu);
2304	STAMR3RegisterF(pVM, &pNemCpu->StatGetMsgTimeout, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of get message timeouts/alerts", "/NEM/CPU%u/GetMsgTimeout", idCpu);
2305	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuSuccess, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of successful CPU stops", "/NEM/CPU%u/StopCpuSuccess", idCpu);
2306	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPending, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pending CPU stops", "/NEM/CPU%u/StopCpuPending", idCpu);
2307	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPendingAlerts,STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pending CPU stop alerts", "/NEM/CPU%u/StopCpuPendingAlerts", idCpu);
2308	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPendingOdd, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of odd pending CPU stops (see code)", "/NEM/CPU%u/StopCpuPendingOdd", idCpu);
2309	STAMR3RegisterF(pVM, &pNemCpu->StatCancelChangedState, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel changed state", "/NEM/CPU%u/CancelChangedState", idCpu);
2310	STAMR3RegisterF(pVM, &pNemCpu->StatCancelAlertedThread, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel alerted EMT", "/NEM/CPU%u/CancelAlertedEMT", idCpu);
2311	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPre, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pre execution FF breaks", "/NEM/CPU%u/BreakOnFFPre", idCpu);
2312	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPost, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of post execution FF breaks", "/NEM/CPU%u/BreakOnFFPost", idCpu);
2313	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnCancel, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel execution breaks", "/NEM/CPU%u/BreakOnCancel", idCpu);
2314	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnStatus, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of status code breaks", "/NEM/CPU%u/BreakOnStatus", idCpu);
2315	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnDemand, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of on-demand state imports", "/NEM/CPU%u/ImportOnDemand", idCpu);
2316	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of state imports on loop return", "/NEM/CPU%u/ImportOnReturn", idCpu);
2317	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturnSkipped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of skipped state imports on loop return", "/NEM/CPU%u/ImportOnReturnSkipped", idCpu);
2318	STAMR3RegisterF(pVM, &pNemCpu->StatQueryCpuTick, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of TSC queries", "/NEM/CPU%u/QueryCpuTick", idCpu);
2319	}
2320	}
2321	else
2322	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
2323	"hv_vm_create() failed: %#x", hrc);
2324	}
2325
2326	/*
2327	* We only fail if in forced mode, otherwise just log the complaint and return.
2328	*/
2329	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
2330	if ( (fForced \|\| !fFallback)
2331	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
2332	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
2333
2334	if (RTErrInfoIsSet(pErrInfo))
2335	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
2336	return VINF_SUCCESS;
2337	}
2338
2339
2340	/**
2341	* Worker to create the vCPU handle on the EMT running it later on (as required by HV).
2342	*
2343	* @returns VBox status code
2344	* @param pVM The VM handle.
2345	* @param pVCpu The vCPU handle.
2346	* @param idCpu ID of the CPU to create.
2347	*/
2348	static DECLCALLBACK(int) nemR3DarwinNativeInitVCpuOnEmt(PVM pVM, PVMCPU pVCpu, VMCPUID idCpu)
2349	{
2350	hv_return_t hrc = hv_vcpu_create(&pVCpu->nem.s.hVCpuId, HV_VCPU_DEFAULT);
2351	if (hrc != HV_SUCCESS)
2352	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
2353	"Call to hv_vcpu_create failed on vCPU %u: %#x (%Rrc)", idCpu, hrc, nemR3DarwinHvSts2Rc(hrc));
2354
2355	if (idCpu == 0)
2356	{
2357	/* First call initializs the MSR structure holding the capabilities of the host CPU. */
2358	int rc = nemR3DarwinCapsInit();
2359	AssertRCReturn(rc, rc);
2360	}
2361
2362	int rc = nemR3DarwinInitVmcs(pVCpu);
2363	AssertRCReturn(rc, rc);
2364
2365	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2366
2367	return VINF_SUCCESS;
2368	}
2369
2370
2371	/**
2372	* Worker to destroy the vCPU handle on the EMT running it later on (as required by HV).
2373	*
2374	* @returns VBox status code
2375	* @param pVCpu The vCPU handle.
2376	*/
2377	static DECLCALLBACK(int) nemR3DarwinNativeTermVCpuOnEmt(PVMCPU pVCpu)
2378	{
2379	hv_return_t hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
2380	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
2381	return VINF_SUCCESS;
2382	}
2383
2384
2385	/**
2386	* This is called after CPUMR3Init is done.
2387	*
2388	* @returns VBox status code.
2389	* @param pVM The VM handle..
2390	*/
2391	int nemR3NativeInitAfterCPUM(PVM pVM)
2392	{
2393	/*
2394	* Validate sanity.
2395	*/
2396	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
2397	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
2398
2399	/*
2400	* Setup the EMTs.
2401	*/
2402	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2403	{
2404	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2405
2406	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitVCpuOnEmt, 3, pVM, pVCpu, idCpu);
2407	if (RT_FAILURE(rc))
2408	{
2409	/* Rollback. */
2410	while (idCpu--)
2411	VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeTermVCpuOnEmt, 1, pVCpu);
2412
2413	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to hv_vcpu_create failed: %Rrc", rc);
2414	}
2415	}
2416
2417	pVM->nem.s.fCreatedEmts = true;
2418	return VINF_SUCCESS;
2419	}
2420
2421
2422	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
2423	{
2424	NOREF(pVM); NOREF(enmWhat);
2425	return VINF_SUCCESS;
2426	}
2427
2428
2429	int nemR3NativeTerm(PVM pVM)
2430	{
2431	/*
2432	* Delete the VM.
2433	*/
2434
2435	for (VMCPUID idCpu = pVM->cCpus - 1; idCpu > 0; idCpu--)
2436	{
2437	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2438
2439	/*
2440	* Apple's documentation states that the vCPU should be destroyed
2441	* on the thread running the vCPU but as all the other EMTs are gone
2442	* at this point, destroying the VM would hang.
2443	*
2444	* We seem to be at luck here though as destroying apparently works
2445	* from EMT(0) as well.
2446	*/
2447	hv_return_t hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
2448	Assert(hrc == HV_SUCCESS);
2449	}
2450
2451	hv_vcpu_destroy(pVM->apCpusR3[0]->nem.s.hVCpuId);
2452	pVM->nem.s.fCreatedEmts = false;
2453
2454	if (pVM->nem.s.fCreatedVm)
2455	{
2456	hv_return_t hrc = hv_vm_destroy();
2457	if (hrc != HV_SUCCESS)
2458	LogRel(("NEM: hv_vm_destroy() failed with %#x\n", hrc));
2459
2460	pVM->nem.s.fCreatedVm = false;
2461	}
2462	return VINF_SUCCESS;
2463	}
2464
2465
2466	/**
2467	* VM reset notification.
2468	*
2469	* @param pVM The cross context VM structure.
2470	*/
2471	void nemR3NativeReset(PVM pVM)
2472	{
2473	RT_NOREF(pVM);
2474	}
2475
2476
2477	/**
2478	* Reset CPU due to INIT IPI or hot (un)plugging.
2479	*
2480	* @param pVCpu The cross context virtual CPU structure of the CPU being
2481	* reset.
2482	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
2483	*/
2484	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
2485	{
2486	RT_NOREF(fInitIpi);
2487	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2488	}
2489
2490
2491	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2492	{
2493	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags));
2494	#ifdef LOG_ENABLED
2495	if (LogIs3Enabled())
2496	nemR3DarwinLogState(pVM, pVCpu);
2497	#endif
2498
2499	/*
2500	* Try switch to NEM runloop state.
2501	*/
2502	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2503	{ /* likely */ }
2504	else
2505	{
2506	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2507	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
2508	return VINF_SUCCESS;
2509	}
2510
2511	/*
2512	* The run loop.
2513	*
2514	* Current approach to state updating to use the sledgehammer and sync
2515	* everything every time. This will be optimized later.
2516	*/
2517
2518	VMXTRANSIENT VmxTransient;
2519	RT_ZERO(VmxTransient);
2520	VmxTransient.pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
2521
2522	const bool fSingleStepping = DBGFIsStepping(pVCpu);
2523	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2524	for (unsigned iLoop = 0;; iLoop++)
2525	{
2526	/*
2527	* Check and process force flag actions, some of which might require us to go back to ring-3.
2528	*/
2529	rcStrict = vmxHCCheckForceFlags(pVCpu, false /fIsNestedGuest/, fSingleStepping);
2530	if (rcStrict == VINF_SUCCESS)
2531	{ /likely / }
2532	else
2533	break;
2534
2535	/*
2536	* Evaluate events to be injected into the guest.
2537	*
2538	* Events in TRPM can be injected without inspecting the guest state.
2539	* If any new events (interrupts/NMI) are pending currently, we try to set up the
2540	* guest to cause a VM-exit the next time they are ready to receive the event.
2541	*/
2542	if (TRPMHasTrap(pVCpu))
2543	vmxHCTrpmTrapToPendingEvent(pVCpu);
2544
2545	uint32_t fIntrState;
2546	rcStrict = vmxHCEvaluatePendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, &fIntrState);
2547
2548	/*
2549	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
2550	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
2551	* also result in triple-faulting the VM.
2552	*
2553	* With nested-guests, the above does not apply since unrestricted guest execution is a
2554	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
2555	*/
2556	rcStrict = vmxHCInjectPendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, fIntrState, fSingleStepping);
2557	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2558	{ /* likely */ }
2559	else
2560	{
2561	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fSingleStepping),
2562	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
2563	break;
2564	}
2565
2566	int rc = nemR3DarwinExportGuestState(pVM, pVCpu, &VmxTransient);
2567	AssertRCReturn(rc, rc);
2568
2569	/*
2570	* Poll timers and run for a bit.
2571	*/
2572	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
2573	* the whole polling job when timers have changed... */
2574	uint64_t offDeltaIgnored;
2575	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
2576	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
2577	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2578	{
2579	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
2580	{
2581	LogFlowFunc(("Running vCPU\n"));
2582	pVCpu->nem.s.Event.fPending = false;
2583
2584	hv_return_t hrc;
2585	if (hv_vcpu_run_until)
2586	hrc = hv_vcpu_run_until(pVCpu->nem.s.hVCpuId, HV_DEADLINE_FOREVER);
2587	else
2588	hrc = hv_vcpu_run(pVCpu->nem.s.hVCpuId);
2589
2590	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
2591	if (hrc == HV_SUCCESS)
2592	{
2593	/*
2594	* Deal with the message.
2595	*/
2596	rcStrict = nemR3DarwinHandleExit(pVM, pVCpu, &VmxTransient);
2597	if (rcStrict == VINF_SUCCESS)
2598	{ /* hopefully likely */ }
2599	else
2600	{
2601	LogFlow(("NEM/%u: breaking: nemR3DarwinHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2602	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2603	break;
2604	}
2605	}
2606	else
2607	{
2608	AssertLogRelMsgFailedReturn(("hv_vcpu_run()) failed for CPU #%u: %#x %u\n",
2609	pVCpu->idCpu, hrc, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
2610	VERR_NEM_IPE_0);
2611	}
2612
2613	/*
2614	* If no relevant FFs are pending, loop.
2615	*/
2616	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2617	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2618	continue;
2619
2620	/** @todo Try handle pending flags, not just return to EM loops. Take care
2621	* not to set important RCs here unless we've handled a message. */
2622	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
2623	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
2624	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
2625	}
2626	else
2627	{
2628	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
2629	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
2630	}
2631	}
2632	else
2633	{
2634	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
2635	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
2636	}
2637	break;
2638	} /* the run loop */
2639
2640
2641	/*
2642	* Convert any pending HM events back to TRPM due to premature exits.
2643	*
2644	* This is because execution may continue from IEM and we would need to inject
2645	* the event from there (hence place it back in TRPM).
2646	*/
2647	if (pVCpu->nem.s.Event.fPending)
2648	{
2649	vmxHCPendingEventToTrpmTrap(pVCpu);
2650	Assert(!pVCpu->nem.s.Event.fPending);
2651
2652	/* Clear the events from the VMCS. */
2653	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
2654	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
2655	}
2656
2657
2658	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
2659	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2660
2661	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL))
2662	{
2663	/* Try anticipate what we might need. */
2664	uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK;
2665	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
2666	\|\| RT_FAILURE(rcStrict))
2667	fImport = CPUMCTX_EXTRN_ALL;
2668	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
2669	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2670	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
2671
2672	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
2673	{
2674	/* Only import what is external currently. */
2675	int rc2 = nemR3DarwinCopyStateFromHv(pVM, pVCpu, fImport);
2676	if (RT_SUCCESS(rc2))
2677	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
2678	else if (RT_SUCCESS(rcStrict))
2679	rcStrict = rc2;
2680	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
2681	{
2682	pVCpu->cpum.GstCtx.fExtrn = 0;
2683	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2684	}
2685	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
2686	}
2687	else
2688	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2689	}
2690	else
2691	{
2692	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2693	pVCpu->cpum.GstCtx.fExtrn = 0;
2694	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2695	}
2696
2697	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
2698	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
2699	return rcStrict;
2700	}
2701
2702
2703	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
2704	{
2705	NOREF(pVM);
2706	return PGMPhysIsA20Enabled(pVCpu);
2707	}
2708
2709
2710	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
2711	{
2712	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
2713	return false;
2714	}
2715
2716
2717	/**
2718	* Forced flag notification call from VMEmt.h.
2719	*
2720	* This is only called when pVCpu is in the VMCPUSTATE_STARTED_EXEC_NEM state.
2721	*
2722	* @param pVM The cross context VM structure.
2723	* @param pVCpu The cross context virtual CPU structure of the CPU
2724	* to be notified.
2725	* @param fFlags Notification flags, VMNOTIFYFF_FLAGS_XXX.
2726	*/
2727	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
2728	{
2729	LogFlowFunc(("pVM=%p pVCpu=%p fFlags=%#x\n", pVM, pVCpu, fFlags));
2730
2731	hv_return_t hrc = hv_vcpu_interrupt(&pVCpu->nem.s.hVCpuId, 1);
2732	if (hrc != HV_SUCCESS)
2733	LogRel(("NEM: hv_vcpu_interrupt(%u, 1) failed with %#x\n", pVCpu->nem.s.hVCpuId, hrc));
2734	}
2735
2736
2737	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
2738	uint8_t pu2State, uint32_t puNemRange)
2739	{
2740	RT_NOREF(pVM, puNemRange);
2741
2742	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p\n", GCPhys, cb, pvR3));
2743	#if defined(VBOX_WITH_PGM_NEM_MODE)
2744	if (pvR3)
2745	{
2746	int rc = nemR3DarwinMap(GCPhys, pvR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
2747	if (RT_SUCCESS(rc))
2748	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
2749	else
2750	{
2751	LogRel(("NEMR3NotifyPhysRamRegister: GCPhys=%RGp LB %RGp pvR3=%p rc=%Rrc\n", GCPhys, cb, pvR3, rc));
2752	return VERR_NEM_MAP_PAGES_FAILED;
2753	}
2754	}
2755	return VINF_SUCCESS;
2756	#else
2757	RT_NOREF(pVM, GCPhys, cb, pvR3);
2758	return VERR_NEM_MAP_PAGES_FAILED;
2759	#endif
2760	}
2761
2762
2763	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
2764	{
2765	RT_NOREF(pVM);
2766	return false;
2767	}
2768
2769
2770	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
2771	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
2772	{
2773	RT_NOREF(pVM, puNemRange);
2774
2775	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d)\n",
2776	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, *pu2State));
2777
2778	#if defined(VBOX_WITH_PGM_NEM_MODE)
2779	/*
2780	* Unmap the RAM we're replacing.
2781	*/
2782	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
2783	{
2784	int rc = nemR3DarwinUnmap(GCPhys, cb);
2785	if (RT_SUCCESS(rc))
2786	{ /* likely */ }
2787	else if (pvMmio2)
2788	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rc(ignored)\n",
2789	GCPhys, cb, fFlags, rc));
2790	else
2791	{
2792	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
2793	GCPhys, cb, fFlags, rc));
2794	return VERR_NEM_UNMAP_PAGES_FAILED;
2795	}
2796	}
2797
2798	/*
2799	* Map MMIO2 if any.
2800	*/
2801	if (pvMmio2)
2802	{
2803	Assert(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2);
2804	int rc = nemR3DarwinMap(GCPhys, pvMmio2, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
2805	if (RT_SUCCESS(rc))
2806	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
2807	else
2808	{
2809	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x pvMmio2=%p: Map -> rc=%Rrc\n",
2810	GCPhys, cb, fFlags, pvMmio2, rc));
2811	return VERR_NEM_MAP_PAGES_FAILED;
2812	}
2813	}
2814	else
2815	{
2816	Assert(!(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2));
2817	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
2818	}
2819
2820	#else
2821	RT_NOREF(pVM, GCPhys, cb, pvRam, pvMmio2);
2822	*pu2State = (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE) ? UINT8_MAX : NEM_DARWIN_PAGE_STATE_UNMAPPED;
2823	#endif
2824	return VINF_SUCCESS;
2825	}
2826
2827
2828	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
2829	void pvRam, void pvMmio2, uint32_t *puNemRange)
2830	{
2831	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
2832	return VINF_SUCCESS;
2833	}
2834
2835
2836	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
2837	void pvMmio2, uint8_t pu2State)
2838	{
2839	RT_NOREF(pVM);
2840
2841	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p\n",
2842	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State));
2843
2844	int rc = VINF_SUCCESS;
2845	#if defined(VBOX_WITH_PGM_NEM_MODE)
2846	/*
2847	* Unmap the MMIO2 pages.
2848	*/
2849	/** @todo If we implement aliasing (MMIO2 page aliased into MMIO range),
2850	* we may have more stuff to unmap even in case of pure MMIO... */
2851	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
2852	{
2853	rc = nemR3DarwinUnmap(GCPhys, cb);
2854	if (RT_FAILURE(rc))
2855	{
2856	LogRel2(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
2857	GCPhys, cb, fFlags, rc));
2858	rc = VERR_NEM_UNMAP_PAGES_FAILED;
2859	}
2860	}
2861
2862	/*
2863	* Restore the RAM we replaced.
2864	*/
2865	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
2866	{
2867	AssertPtr(pvRam);
2868	rc = nemR3DarwinMap(GCPhys, pvRam, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
2869	if (RT_SUCCESS(rc))
2870	{ /* likely */ }
2871	else
2872	{
2873	LogRel(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp pvMmio2=%p rc=%Rrc\n", GCPhys, cb, pvMmio2, rc));
2874	rc = VERR_NEM_MAP_PAGES_FAILED;
2875	}
2876	if (pu2State)
2877	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
2878	}
2879	/* Mark the pages as unmapped if relevant. */
2880	else if (pu2State)
2881	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
2882
2883	RT_NOREF(pvMmio2);
2884	#else
2885	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
2886	if (pu2State)
2887	*pu2State = UINT8_MAX;
2888	rc = VERR_NEM_UNMAP_PAGES_FAILED;
2889	#endif
2890	return rc;
2891	}
2892
2893
2894	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
2895	void *pvBitmap, size_t cbBitmap)
2896	{
2897	RT_NOREF(pVM, GCPhys, cb, uNemRange, pvBitmap, cbBitmap);
2898	AssertFailed();
2899	return VERR_NOT_IMPLEMENTED;
2900	}
2901
2902
2903	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
2904	uint8_t *pu2State)
2905	{
2906	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags);
2907
2908	Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
2909	*pu2State = UINT8_MAX;
2910	return VINF_SUCCESS;
2911	}
2912
2913
2914	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
2915	uint32_t fFlags, uint8_t *pu2State)
2916	{
2917	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p\n",
2918	GCPhys, cb, pvPages, fFlags, pu2State));
2919	*pu2State = UINT8_MAX;
2920
2921	#if defined(VBOX_WITH_PGM_NEM_MODE)
2922	/*
2923	* (Re-)map readonly.
2924	*/
2925	AssertPtrReturn(pvPages, VERR_INVALID_POINTER);
2926	int rc = nemR3DarwinMap(GCPhys, pvPages, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE);
2927	if (RT_SUCCESS(rc))
2928	*pu2State = NEM_DARWIN_PAGE_STATE_READABLE;
2929	else
2930	{
2931	LogRel(("nemR3NativeNotifyPhysRomRegisterLate: GCPhys=%RGp LB %RGp pvPages=%p fFlags=%#x rc=%Rrc\n",
2932	GCPhys, cb, pvPages, fFlags, rc));
2933	return VERR_NEM_MAP_PAGES_FAILED;
2934	}
2935	RT_NOREF(pVM, fFlags);
2936	return VINF_SUCCESS;
2937	#else
2938	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags);
2939	return VERR_NEM_MAP_PAGES_FAILED;
2940	#endif
2941	}
2942
2943
2944	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
2945	RTR3PTR pvMemR3, uint8_t *pu2State)
2946	{
2947	RT_NOREF(pVM);
2948
2949	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
2950	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
2951
2952	*pu2State = UINT8_MAX;
2953	#if defined(VBOX_WITH_PGM_NEM_MODE)
2954	if (pvMemR3)
2955	{
2956	int rc = nemR3DarwinMap(GCPhys, pvMemR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
2957	if (RT_SUCCESS(rc))
2958	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
2959	else
2960	AssertLogRelMsgFailed(("NEMHCNotifyHandlerPhysicalDeregister: nemR3DarwinMap(,%p,%RGp,%RGp,) -> %Rrc\n",
2961	pvMemR3, GCPhys, cb, rc));
2962	}
2963	RT_NOREF(enmKind);
2964	#else
2965	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
2966	AssertFailed();
2967	#endif
2968	}
2969
2970
2971	static int nemHCJustUnmapPage(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
2972	{
2973	if (*pu2State <= NEM_DARWIN_PAGE_STATE_UNMAPPED)
2974	{
2975	Log5(("nemHCJustUnmapPage: %RGp == unmapped\n", GCPhysDst));
2976	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
2977	return VINF_SUCCESS;
2978	}
2979
2980	int rc = nemR3DarwinUnmap(GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
2981	if (RT_SUCCESS(rc))
2982	{
2983	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
2984	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2985	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
2986	Log5(("nemHCJustUnmapPage: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
2987	return VINF_SUCCESS;
2988	}
2989	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
2990	LogRel(("nemHCJustUnmapPage(%RGp): failed! rc=%Rrc\n",
2991	GCPhysDst, rc));
2992	return VERR_NEM_IPE_6;
2993	}
2994
2995
2996	/**
2997	* Called when the A20 state changes.
2998	*
2999	* @param pVCpu The CPU the A20 state changed on.
3000	* @param fEnabled Whether it was enabled (true) or disabled.
3001	*/
3002	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
3003	{
3004	Log(("NEMR3NotifySetA20: fEnabled=%RTbool\n", fEnabled));
3005	RT_NOREF(pVCpu, fEnabled);
3006	}
3007
3008
3009	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
3010	{
3011	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
3012	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
3013	}
3014
3015
3016	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
3017	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
3018	{
3019	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
3020	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
3021	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
3022	}
3023
3024
3025	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
3026	PGMPAGETYPE enmType, uint8_t *pu2State)
3027	{
3028	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3029	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3030	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
3031
3032	return nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3033	}
3034
3035
3036	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
3037	PGMPAGETYPE enmType, uint8_t *pu2State)
3038	{
3039	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3040	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3041	RT_NOREF(HCPhys, pvR3, fPageProt, enmType)
3042
3043	nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3044	}
3045
3046
3047	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
3048	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
3049	{
3050	Log5(("NEMHCNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3051	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
3052	RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType);
3053
3054	nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3055	}
3056
3057
3058	/**
3059	* Interface for importing state on demand (used by IEM).
3060	*
3061	* @returns VBox status code.
3062	* @param pVCpu The cross context CPU structure.
3063	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3064	*/
3065	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
3066	{
3067	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
3068
3069	return nemR3DarwinCopyStateFromHv(pVCpu->pVMR3, pVCpu, fWhat);
3070	}
3071
3072
3073	/**
3074	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
3075	*
3076	* @returns VBox status code.
3077	* @param pVCpu The cross context CPU structure.
3078	* @param pcTicks Where to return the CPU tick count.
3079	* @param puAux Where to return the TSC_AUX register value.
3080	*/
3081	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
3082	{
3083	LogFlowFunc(("pVCpu=%p pcTicks=%RX64 puAux=%RX32\n", pVCpu, pcTicks, puAux));
3084	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
3085
3086	int rc = nemR3DarwinMsrRead(pVCpu, MSR_IA32_TSC, pcTicks);
3087	if ( RT_SUCCESS(rc)
3088	&& puAux)
3089	{
3090	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX)
3091	{
3092	/** @todo Why the heck is puAux a uint32_t?. */
3093	uint64_t u64Aux;
3094	rc = nemR3DarwinMsrRead(pVCpu, MSR_K8_TSC_AUX, &u64Aux);
3095	if (RT_SUCCESS(rc))
3096	*puAux = (uint32_t)u64Aux;
3097	}
3098	else
3099	*puAux = CPUMGetGuestTscAux(pVCpu);
3100	}
3101
3102	return rc;
3103	}
3104
3105
3106	/**
3107	* Resumes CPU clock (TSC) on all virtual CPUs.
3108	*
3109	* This is called by TM when the VM is started, restored, resumed or similar.
3110	*
3111	* @returns VBox status code.
3112	* @param pVM The cross context VM structure.
3113	* @param pVCpu The cross context CPU structure of the calling EMT.
3114	* @param uPausedTscValue The TSC value at the time of pausing.
3115	*/
3116	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
3117	{
3118	LogFlowFunc(("pVM=%p pVCpu=%p uPausedTscValue=%RX64\n", pVCpu, uPausedTscValue));
3119	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
3120	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
3121
3122	hv_return_t hrc = hv_vm_sync_tsc(uPausedTscValue);
3123	if (RT_LIKELY(hrc == HV_SUCCESS))
3124	return VINF_SUCCESS;
3125
3126	return nemR3DarwinHvSts2Rc(hrc);
3127	}
3128
3129
3130	/** @page pg_nem_darwin NEM/darwin - Native Execution Manager, macOS.
3131	*
3132	* @todo Add notes as the implementation progresses...
3133	*/
3134

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-darwin.cpp@ 92351

Download in other formats: