NEMR3Native-linux.cpp@ 95843

Last change on this file since 95843 was 95248, checked in by vboxsync, 2 years ago
VMM/CPUM: Started changing the way we advertise SYSCALL, SEP, NX, and others as well as deduplicating the code for enabling 64-bit guest support (long mode). Also, the SYSCALL CPUID bit is now correctly suppressed when not in 64-bit mode on Intel CPUs. bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 116.8 KB

Line
1	/* $Id: NEMR3Native-linux.cpp 95248 2022-06-10 16:40:34Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 Linux backend.
4	*/
5
6	/*
7	* Copyright (C) 2021-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_NEM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <VBox/vmm/nem.h>
25	#include <VBox/vmm/iem.h>
26	#include <VBox/vmm/em.h>
27	#include <VBox/vmm/apic.h>
28	#include <VBox/vmm/pdm.h>
29	#include <VBox/vmm/trpm.h>
30	#include "NEMInternal.h"
31	#include <VBox/vmm/vmcc.h>
32
33	#include <iprt/alloca.h>
34	#include <iprt/string.h>
35	#include <iprt/system.h>
36	#include <iprt/x86.h>
37
38	#include <errno.h>
39	#include <unistd.h>
40	#include <sys/ioctl.h>
41	#include <sys/fcntl.h>
42	#include <sys/mman.h>
43	#include <linux/kvm.h>
44
45	/*
46	* Supply stuff missing from the kvm.h on the build box.
47	*/
48	#ifndef KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON /* since 5.4 */
49	# define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4
50	#endif
51
52
53
54	/**
55	* Worker for nemR3NativeInit that gets the hypervisor capabilities.
56	*
57	* @returns VBox status code.
58	* @param pVM The cross context VM structure.
59	* @param pErrInfo Where to always return error info.
60	*/
61	static int nemR3LnxInitCheckCapabilities(PVM pVM, PRTERRINFO pErrInfo)
62	{
63	AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
64
65	/*
66	* Capabilities.
67	*/
68	static const struct
69	{
70	const char *pszName;
71	int iCap;
72	uint32_t offNem : 24;
73	uint32_t cbNem : 3;
74	uint32_t fReqNonZero : 1;
75	uint32_t uReserved : 4;
76	} s_aCaps[] =
77	{
78	#define CAP_ENTRY__L(a_Define) { #a_Define, a_Define, UINT32_C(0x00ffffff), 0, 0, 0 }
79	#define CAP_ENTRY__S(a_Define, a_Member) { #a_Define, a_Define, RT_UOFFSETOF(NEM, a_Member), RT_SIZEOFMEMB(NEM, a_Member), 0, 0 }
80	#define CAP_ENTRY_MS(a_Define, a_Member) { #a_Define, a_Define, RT_UOFFSETOF(NEM, a_Member), RT_SIZEOFMEMB(NEM, a_Member), 1, 0 }
81	#define CAP_ENTRY__U(a_Number) { "KVM_CAP_" #a_Number, a_Number, UINT32_C(0x00ffffff), 0, 0, 0 }
82	#define CAP_ENTRY_ML(a_Number) { "KVM_CAP_" #a_Number, a_Number, UINT32_C(0x00ffffff), 0, 1, 0 }
83
84	CAP_ENTRY__L(KVM_CAP_IRQCHIP), /* 0 */
85	CAP_ENTRY_ML(KVM_CAP_HLT),
86	CAP_ENTRY__L(KVM_CAP_MMU_SHADOW_CACHE_CONTROL),
87	CAP_ENTRY_ML(KVM_CAP_USER_MEMORY),
88	CAP_ENTRY__L(KVM_CAP_SET_TSS_ADDR),
89	CAP_ENTRY__U(5),
90	CAP_ENTRY__L(KVM_CAP_VAPIC),
91	CAP_ENTRY__L(KVM_CAP_EXT_CPUID),
92	CAP_ENTRY__L(KVM_CAP_CLOCKSOURCE),
93	CAP_ENTRY__L(KVM_CAP_NR_VCPUS),
94	CAP_ENTRY_MS(KVM_CAP_NR_MEMSLOTS, cMaxMemSlots), /* 10 */
95	CAP_ENTRY__L(KVM_CAP_PIT),
96	CAP_ENTRY__L(KVM_CAP_NOP_IO_DELAY),
97	CAP_ENTRY__L(KVM_CAP_PV_MMU),
98	CAP_ENTRY__L(KVM_CAP_MP_STATE),
99	CAP_ENTRY__L(KVM_CAP_COALESCED_MMIO),
100	CAP_ENTRY__L(KVM_CAP_SYNC_MMU),
101	CAP_ENTRY__U(17),
102	CAP_ENTRY__L(KVM_CAP_IOMMU),
103	CAP_ENTRY__U(19), /* Buggy KVM_CAP_JOIN_MEMORY_REGIONS? */
104	CAP_ENTRY__U(20), /* Mon-working KVM_CAP_DESTROY_MEMORY_REGION? */
105	CAP_ENTRY__L(KVM_CAP_DESTROY_MEMORY_REGION_WORKS), /* 21 */
106	CAP_ENTRY__L(KVM_CAP_USER_NMI),
107	#ifdef __KVM_HAVE_GUEST_DEBUG
108	CAP_ENTRY__L(KVM_CAP_SET_GUEST_DEBUG),
109	#endif
110	#ifdef __KVM_HAVE_PIT
111	CAP_ENTRY__L(KVM_CAP_REINJECT_CONTROL),
112	#endif
113	CAP_ENTRY__L(KVM_CAP_IRQ_ROUTING),
114	CAP_ENTRY__L(KVM_CAP_IRQ_INJECT_STATUS),
115	CAP_ENTRY__U(27),
116	CAP_ENTRY__U(28),
117	CAP_ENTRY__L(KVM_CAP_ASSIGN_DEV_IRQ),
118	CAP_ENTRY__L(KVM_CAP_JOIN_MEMORY_REGIONS_WORKS), /* 30 */
119	#ifdef __KVM_HAVE_MCE
120	CAP_ENTRY__L(KVM_CAP_MCE),
121	#endif
122	CAP_ENTRY__L(KVM_CAP_IRQFD),
123	#ifdef __KVM_HAVE_PIT
124	CAP_ENTRY__L(KVM_CAP_PIT2),
125	#endif
126	CAP_ENTRY__L(KVM_CAP_SET_BOOT_CPU_ID),
127	#ifdef __KVM_HAVE_PIT_STATE2
128	CAP_ENTRY__L(KVM_CAP_PIT_STATE2),
129	#endif
130	CAP_ENTRY__L(KVM_CAP_IOEVENTFD),
131	CAP_ENTRY__L(KVM_CAP_SET_IDENTITY_MAP_ADDR),
132	#ifdef __KVM_HAVE_XEN_HVM
133	CAP_ENTRY__L(KVM_CAP_XEN_HVM),
134	#endif
135	CAP_ENTRY_ML(KVM_CAP_ADJUST_CLOCK),
136	CAP_ENTRY__L(KVM_CAP_INTERNAL_ERROR_DATA), /* 40 */
137	#ifdef __KVM_HAVE_VCPU_EVENTS
138	CAP_ENTRY_ML(KVM_CAP_VCPU_EVENTS),
139	#else
140	CAP_ENTRY_MU(41),
141	#endif
142	CAP_ENTRY__L(KVM_CAP_S390_PSW),
143	CAP_ENTRY__L(KVM_CAP_PPC_SEGSTATE),
144	CAP_ENTRY__L(KVM_CAP_HYPERV),
145	CAP_ENTRY__L(KVM_CAP_HYPERV_VAPIC),
146	CAP_ENTRY__L(KVM_CAP_HYPERV_SPIN),
147	CAP_ENTRY__L(KVM_CAP_PCI_SEGMENT),
148	CAP_ENTRY__L(KVM_CAP_PPC_PAIRED_SINGLES),
149	CAP_ENTRY__L(KVM_CAP_INTR_SHADOW),
150	#ifdef __KVM_HAVE_DEBUGREGS
151	CAP_ENTRY__L(KVM_CAP_DEBUGREGS), /* 50 */
152	#endif
153	CAP_ENTRY__S(KVM_CAP_X86_ROBUST_SINGLESTEP, fRobustSingleStep),
154	CAP_ENTRY__L(KVM_CAP_PPC_OSI),
155	CAP_ENTRY__L(KVM_CAP_PPC_UNSET_IRQ),
156	CAP_ENTRY__L(KVM_CAP_ENABLE_CAP),
157	#ifdef __KVM_HAVE_XSAVE
158	CAP_ENTRY_ML(KVM_CAP_XSAVE),
159	#else
160	CAP_ENTRY_MU(55),
161	#endif
162	#ifdef __KVM_HAVE_XCRS
163	CAP_ENTRY_ML(KVM_CAP_XCRS),
164	#else
165	CAP_ENTRY_MU(56),
166	#endif
167	CAP_ENTRY__L(KVM_CAP_PPC_GET_PVINFO),
168	CAP_ENTRY__L(KVM_CAP_PPC_IRQ_LEVEL),
169	CAP_ENTRY__L(KVM_CAP_ASYNC_PF),
170	CAP_ENTRY__L(KVM_CAP_TSC_CONTROL), /* 60 */
171	CAP_ENTRY__L(KVM_CAP_GET_TSC_KHZ),
172	CAP_ENTRY__L(KVM_CAP_PPC_BOOKE_SREGS),
173	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE),
174	CAP_ENTRY__L(KVM_CAP_PPC_SMT),
175	CAP_ENTRY__L(KVM_CAP_PPC_RMA),
176	CAP_ENTRY__L(KVM_CAP_MAX_VCPUS),
177	CAP_ENTRY__L(KVM_CAP_PPC_HIOR),
178	CAP_ENTRY__L(KVM_CAP_PPC_PAPR),
179	CAP_ENTRY__L(KVM_CAP_SW_TLB),
180	CAP_ENTRY__L(KVM_CAP_ONE_REG), /* 70 */
181	CAP_ENTRY__L(KVM_CAP_S390_GMAP),
182	CAP_ENTRY__L(KVM_CAP_TSC_DEADLINE_TIMER),
183	CAP_ENTRY__L(KVM_CAP_S390_UCONTROL),
184	CAP_ENTRY__L(KVM_CAP_SYNC_REGS),
185	CAP_ENTRY__L(KVM_CAP_PCI_2_3),
186	CAP_ENTRY__L(KVM_CAP_KVMCLOCK_CTRL),
187	CAP_ENTRY__L(KVM_CAP_SIGNAL_MSI),
188	CAP_ENTRY__L(KVM_CAP_PPC_GET_SMMU_INFO),
189	CAP_ENTRY__L(KVM_CAP_S390_COW),
190	CAP_ENTRY__L(KVM_CAP_PPC_ALLOC_HTAB), /* 80 */
191	CAP_ENTRY__L(KVM_CAP_READONLY_MEM),
192	CAP_ENTRY__L(KVM_CAP_IRQFD_RESAMPLE),
193	CAP_ENTRY__L(KVM_CAP_PPC_BOOKE_WATCHDOG),
194	CAP_ENTRY__L(KVM_CAP_PPC_HTAB_FD),
195	CAP_ENTRY__L(KVM_CAP_S390_CSS_SUPPORT),
196	CAP_ENTRY__L(KVM_CAP_PPC_EPR),
197	CAP_ENTRY__L(KVM_CAP_ARM_PSCI),
198	CAP_ENTRY__L(KVM_CAP_ARM_SET_DEVICE_ADDR),
199	CAP_ENTRY__L(KVM_CAP_DEVICE_CTRL),
200	CAP_ENTRY__L(KVM_CAP_IRQ_MPIC), /* 90 */
201	CAP_ENTRY__L(KVM_CAP_PPC_RTAS),
202	CAP_ENTRY__L(KVM_CAP_IRQ_XICS),
203	CAP_ENTRY__L(KVM_CAP_ARM_EL1_32BIT),
204	CAP_ENTRY__L(KVM_CAP_SPAPR_MULTITCE),
205	CAP_ENTRY__L(KVM_CAP_EXT_EMUL_CPUID),
206	CAP_ENTRY__L(KVM_CAP_HYPERV_TIME),
207	CAP_ENTRY__L(KVM_CAP_IOAPIC_POLARITY_IGNORED),
208	CAP_ENTRY__L(KVM_CAP_ENABLE_CAP_VM),
209	CAP_ENTRY__L(KVM_CAP_S390_IRQCHIP),
210	CAP_ENTRY__L(KVM_CAP_IOEVENTFD_NO_LENGTH), /* 100 */
211	CAP_ENTRY__L(KVM_CAP_VM_ATTRIBUTES),
212	CAP_ENTRY__L(KVM_CAP_ARM_PSCI_0_2),
213	CAP_ENTRY__L(KVM_CAP_PPC_FIXUP_HCALL),
214	CAP_ENTRY__L(KVM_CAP_PPC_ENABLE_HCALL),
215	CAP_ENTRY__L(KVM_CAP_CHECK_EXTENSION_VM),
216	CAP_ENTRY__L(KVM_CAP_S390_USER_SIGP),
217	CAP_ENTRY__L(KVM_CAP_S390_VECTOR_REGISTERS),
218	CAP_ENTRY__L(KVM_CAP_S390_MEM_OP),
219	CAP_ENTRY__L(KVM_CAP_S390_USER_STSI),
220	CAP_ENTRY__L(KVM_CAP_S390_SKEYS), /* 110 */
221	CAP_ENTRY__L(KVM_CAP_MIPS_FPU),
222	CAP_ENTRY__L(KVM_CAP_MIPS_MSA),
223	CAP_ENTRY__L(KVM_CAP_S390_INJECT_IRQ),
224	CAP_ENTRY__L(KVM_CAP_S390_IRQ_STATE),
225	CAP_ENTRY__L(KVM_CAP_PPC_HWRNG),
226	CAP_ENTRY__L(KVM_CAP_DISABLE_QUIRKS),
227	CAP_ENTRY__L(KVM_CAP_X86_SMM),
228	CAP_ENTRY__L(KVM_CAP_MULTI_ADDRESS_SPACE),
229	CAP_ENTRY__L(KVM_CAP_GUEST_DEBUG_HW_BPS),
230	CAP_ENTRY__L(KVM_CAP_GUEST_DEBUG_HW_WPS), /* 120 */
231	CAP_ENTRY__L(KVM_CAP_SPLIT_IRQCHIP),
232	CAP_ENTRY__L(KVM_CAP_IOEVENTFD_ANY_LENGTH),
233	CAP_ENTRY__L(KVM_CAP_HYPERV_SYNIC),
234	CAP_ENTRY__L(KVM_CAP_S390_RI),
235	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE_64),
236	CAP_ENTRY__L(KVM_CAP_ARM_PMU_V3),
237	CAP_ENTRY__L(KVM_CAP_VCPU_ATTRIBUTES),
238	CAP_ENTRY__L(KVM_CAP_MAX_VCPU_ID),
239	CAP_ENTRY__L(KVM_CAP_X2APIC_API),
240	CAP_ENTRY__L(KVM_CAP_S390_USER_INSTR0), /* 130 */
241	CAP_ENTRY__L(KVM_CAP_MSI_DEVID),
242	CAP_ENTRY__L(KVM_CAP_PPC_HTM),
243	CAP_ENTRY__L(KVM_CAP_SPAPR_RESIZE_HPT),
244	CAP_ENTRY__L(KVM_CAP_PPC_MMU_RADIX),
245	CAP_ENTRY__L(KVM_CAP_PPC_MMU_HASH_V3),
246	CAP_ENTRY__L(KVM_CAP_IMMEDIATE_EXIT),
247	CAP_ENTRY__L(KVM_CAP_MIPS_VZ),
248	CAP_ENTRY__L(KVM_CAP_MIPS_TE),
249	CAP_ENTRY__L(KVM_CAP_MIPS_64BIT),
250	CAP_ENTRY__L(KVM_CAP_S390_GS), /* 140 */
251	CAP_ENTRY__L(KVM_CAP_S390_AIS),
252	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE_VFIO),
253	CAP_ENTRY__L(KVM_CAP_X86_DISABLE_EXITS),
254	CAP_ENTRY__L(KVM_CAP_ARM_USER_IRQ),
255	CAP_ENTRY__L(KVM_CAP_S390_CMMA_MIGRATION),
256	CAP_ENTRY__L(KVM_CAP_PPC_FWNMI),
257	CAP_ENTRY__L(KVM_CAP_PPC_SMT_POSSIBLE),
258	CAP_ENTRY__L(KVM_CAP_HYPERV_SYNIC2),
259	CAP_ENTRY__L(KVM_CAP_HYPERV_VP_INDEX),
260	CAP_ENTRY__L(KVM_CAP_S390_AIS_MIGRATION), /* 150 */
261	CAP_ENTRY__L(KVM_CAP_PPC_GET_CPU_CHAR),
262	CAP_ENTRY__L(KVM_CAP_S390_BPB),
263	CAP_ENTRY__L(KVM_CAP_GET_MSR_FEATURES),
264	CAP_ENTRY__L(KVM_CAP_HYPERV_EVENTFD),
265	CAP_ENTRY__L(KVM_CAP_HYPERV_TLBFLUSH),
266	CAP_ENTRY__L(KVM_CAP_S390_HPAGE_1M),
267	CAP_ENTRY__L(KVM_CAP_NESTED_STATE),
268	CAP_ENTRY__L(KVM_CAP_ARM_INJECT_SERROR_ESR),
269	CAP_ENTRY__L(KVM_CAP_MSR_PLATFORM_INFO),
270	CAP_ENTRY__L(KVM_CAP_PPC_NESTED_HV), /* 160 */
271	CAP_ENTRY__L(KVM_CAP_HYPERV_SEND_IPI),
272	CAP_ENTRY__L(KVM_CAP_COALESCED_PIO),
273	CAP_ENTRY__L(KVM_CAP_HYPERV_ENLIGHTENED_VMCS),
274	CAP_ENTRY__L(KVM_CAP_EXCEPTION_PAYLOAD),
275	CAP_ENTRY__L(KVM_CAP_ARM_VM_IPA_SIZE),
276	CAP_ENTRY__L(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT),
277	CAP_ENTRY__L(KVM_CAP_HYPERV_CPUID),
278	CAP_ENTRY__L(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2),
279	CAP_ENTRY__L(KVM_CAP_PPC_IRQ_XIVE),
280	CAP_ENTRY__L(KVM_CAP_ARM_SVE), /* 170 */
281	CAP_ENTRY__L(KVM_CAP_ARM_PTRAUTH_ADDRESS),
282	CAP_ENTRY__L(KVM_CAP_ARM_PTRAUTH_GENERIC),
283	CAP_ENTRY__L(KVM_CAP_PMU_EVENT_FILTER),
284	CAP_ENTRY__L(KVM_CAP_ARM_IRQ_LINE_LAYOUT_2),
285	CAP_ENTRY__L(KVM_CAP_HYPERV_DIRECT_TLBFLUSH),
286	CAP_ENTRY__L(KVM_CAP_PPC_GUEST_DEBUG_SSTEP),
287	CAP_ENTRY__L(KVM_CAP_ARM_NISV_TO_USER),
288	CAP_ENTRY__L(KVM_CAP_ARM_INJECT_EXT_DABT),
289	CAP_ENTRY__L(KVM_CAP_S390_VCPU_RESETS),
290	CAP_ENTRY__L(KVM_CAP_S390_PROTECTED), /* 180 */
291	CAP_ENTRY__L(KVM_CAP_PPC_SECURE_GUEST),
292	CAP_ENTRY__L(KVM_CAP_HALT_POLL),
293	CAP_ENTRY__L(KVM_CAP_ASYNC_PF_INT),
294	CAP_ENTRY__L(KVM_CAP_LAST_CPU),
295	CAP_ENTRY__L(KVM_CAP_SMALLER_MAXPHYADDR),
296	CAP_ENTRY__L(KVM_CAP_S390_DIAG318),
297	CAP_ENTRY__L(KVM_CAP_STEAL_TIME),
298	CAP_ENTRY_ML(KVM_CAP_X86_USER_SPACE_MSR), /* (since 5.10) */
299	CAP_ENTRY_ML(KVM_CAP_X86_MSR_FILTER),
300	CAP_ENTRY__L(KVM_CAP_ENFORCE_PV_FEATURE_CPUID), /* 190 */
301	CAP_ENTRY__L(KVM_CAP_SYS_HYPERV_CPUID),
302	CAP_ENTRY__L(KVM_CAP_DIRTY_LOG_RING),
303	CAP_ENTRY__L(KVM_CAP_X86_BUS_LOCK_EXIT),
304	CAP_ENTRY__L(KVM_CAP_PPC_DAWR1),
305	CAP_ENTRY__L(KVM_CAP_SET_GUEST_DEBUG2),
306	CAP_ENTRY__L(KVM_CAP_SGX_ATTRIBUTE),
307	CAP_ENTRY__L(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM),
308	CAP_ENTRY__L(KVM_CAP_PTP_KVM),
309	CAP_ENTRY__U(199),
310	CAP_ENTRY__U(200),
311	CAP_ENTRY__U(201),
312	CAP_ENTRY__U(202),
313	CAP_ENTRY__U(203),
314	CAP_ENTRY__U(204),
315	CAP_ENTRY__U(205),
316	CAP_ENTRY__U(206),
317	CAP_ENTRY__U(207),
318	CAP_ENTRY__U(208),
319	CAP_ENTRY__U(209),
320	CAP_ENTRY__U(210),
321	CAP_ENTRY__U(211),
322	CAP_ENTRY__U(212),
323	CAP_ENTRY__U(213),
324	CAP_ENTRY__U(214),
325	CAP_ENTRY__U(215),
326	CAP_ENTRY__U(216),
327	};
328
329	LogRel(("NEM: KVM capabilities (system):\n"));
330	int rcRet = VINF_SUCCESS;
331	for (unsigned i = 0; i < RT_ELEMENTS(s_aCaps); i++)
332	{
333	int rc = ioctl(pVM->nem.s.fdVm, KVM_CHECK_EXTENSION, s_aCaps[i].iCap);
334	if (rc >= 10)
335	LogRel(("NEM: %36s: %#x (%d)\n", s_aCaps[i].pszName, rc, rc));
336	else if (rc >= 0)
337	LogRel(("NEM: %36s: %d\n", s_aCaps[i].pszName, rc));
338	else
339	LogRel(("NEM: %s failed: %d/%d\n", s_aCaps[i].pszName, rc, errno));
340	switch (s_aCaps[i].cbNem)
341	{
342	case 0:
343	break;
344	case 1:
345	{
346	uint8_t puValue = (uint8_t )&pVM->nem.padding[s_aCaps[i].offNem];
347	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
348	*puValue = (uint8_t)rc;
349	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
350	break;
351	}
352	case 2:
353	{
354	uint16_t puValue = (uint16_t )&pVM->nem.padding[s_aCaps[i].offNem];
355	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
356	*puValue = (uint16_t)rc;
357	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
358	break;
359	}
360	case 4:
361	{
362	uint32_t puValue = (uint32_t )&pVM->nem.padding[s_aCaps[i].offNem];
363	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
364	*puValue = (uint32_t)rc;
365	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
366	break;
367	}
368	default:
369	rcRet = RTErrInfoSetF(pErrInfo, VERR_NEM_IPE_0, "s_aCaps[%u] is bad: cbNem=%#x - %s",
370	i, s_aCaps[i].pszName, s_aCaps[i].cbNem);
371	AssertFailedReturn(rcRet);
372	}
373
374	/*
375	* Is a require non-zero entry zero or failing?
376	*/
377	if (s_aCaps[i].fReqNonZero && rc <= 0)
378	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_MISSING_FEATURE,
379	"Required capability '%s' is missing!", s_aCaps[i].pszName);
380	}
381
382	/*
383	* Get per VCpu KVM_RUN MMAP area size.
384	*/
385	int rc = ioctl(pVM->nem.s.fdKvm, KVM_GET_VCPU_MMAP_SIZE, 0UL);
386	if ((unsigned)rc < _64M)
387	{
388	pVM->nem.s.cbVCpuMmap = (uint32_t)rc;
389	LogRel(("NEM: %36s: %#x (%d)\n", "KVM_GET_VCPU_MMAP_SIZE", rc, rc));
390	}
391	else if (rc < 0)
392	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_MISSING_FEATURE, "KVM_GET_VCPU_MMAP_SIZE failed: %d", errno);
393	else
394	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_INIT_FAILED, "Odd KVM_GET_VCPU_MMAP_SIZE value: %#x (%d)", rc, rc);
395
396	/*
397	* Init the slot ID bitmap.
398	*/
399	ASMBitSet(&pVM->nem.s.bmSlotIds[0], 0); /* don't use slot 0 */
400	if (pVM->nem.s.cMaxMemSlots < _32K)
401	ASMBitSetRange(&pVM->nem.s.bmSlotIds[0], pVM->nem.s.cMaxMemSlots, _32K);
402	ASMBitSet(&pVM->nem.s.bmSlotIds[0], _32K - 1); /* don't use the last slot */
403
404	return rcRet;
405	}
406
407
408	/**
409	* Does the early setup of a KVM VM.
410	*
411	* @returns VBox status code.
412	* @param pVM The cross context VM structure.
413	* @param pErrInfo Where to always return error info.
414	*/
415	static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo)
416	{
417	AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
418
419	/*
420	* Enable user space MSRs and let us check everything KVM cannot handle.
421	* We will set up filtering later when ring-3 init has completed.
422	*/
423	struct kvm_enable_cap CapEn =
424	{
425	KVM_CAP_X86_USER_SPACE_MSR, 0,
426	{ KVM_MSR_EXIT_REASON_FILTER \| KVM_MSR_EXIT_REASON_UNKNOWN \| KVM_MSR_EXIT_REASON_INVAL, 0, 0, 0}
427	};
428	int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_ENABLE_CAP, &CapEn);
429	if (rcLnx == -1)
430	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "Failed to enable KVM_CAP_X86_USER_SPACE_MSR failed: %u", errno);
431
432	/*
433	* Create the VCpus.
434	*/
435	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
436	{
437	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
438
439	/* Create it. */
440	pVCpu->nem.s.fdVCpu = ioctl(pVM->nem.s.fdVm, KVM_CREATE_VCPU, (unsigned long)idCpu);
441	if (pVCpu->nem.s.fdVCpu < 0)
442	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VCPU failed for VCpu #%u: %d", idCpu, errno);
443
444	/* Map the KVM_RUN area. */
445	pVCpu->nem.s.pRun = (struct kvm_run *)mmap(NULL, pVM->nem.s.cbVCpuMmap, PROT_READ \| PROT_WRITE, MAP_SHARED,
446	pVCpu->nem.s.fdVCpu, 0 /offset/);
447	if ((void *)pVCpu->nem.s.pRun == MAP_FAILED)
448	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "mmap failed for VCpu #%u: %d", idCpu, errno);
449
450	/* We want all x86 registers and events on each exit. */
451	pVCpu->nem.s.pRun->kvm_valid_regs = KVM_SYNC_X86_REGS \| KVM_SYNC_X86_SREGS \| KVM_SYNC_X86_EVENTS;
452	}
453	return VINF_SUCCESS;
454	}
455
456
457	/** @callback_method_impl{FNVMMEMTRENDEZVOUS} */
458	static DECLCALLBACK(VBOXSTRICTRC) nemR3LnxFixThreadPoke(PVM pVM, PVMCPU pVCpu, void *pvUser)
459	{
460	RT_NOREF(pVM, pvUser);
461	int rc = RTThreadControlPokeSignal(pVCpu->hThread, true /fEnable/);
462	AssertLogRelRC(rc);
463	return VINF_SUCCESS;
464	}
465
466
467	/**
468	* Try initialize the native API.
469	*
470	* This may only do part of the job, more can be done in
471	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
472	*
473	* @returns VBox status code.
474	* @param pVM The cross context VM structure.
475	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
476	* the latter we'll fail if we cannot initialize.
477	* @param fForced Whether the HMForced flag is set and we should
478	* fail if we cannot initialize.
479	*/
480	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
481	{
482	RT_NOREF(pVM, fFallback, fForced);
483	/*
484	* Some state init.
485	*/
486	pVM->nem.s.fdKvm = -1;
487	pVM->nem.s.fdVm = -1;
488	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
489	{
490	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
491	pNemCpu->fdVCpu = -1;
492	}
493
494	/*
495	* Error state.
496	* The error message will be non-empty on failure and 'rc' will be set too.
497	*/
498	RTERRINFOSTATIC ErrInfo;
499	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
500
501	/*
502	* Open kvm subsystem so we can issue system ioctls.
503	*/
504	int rc;
505	int fdKvm = open("/dev/kvm", O_RDWR \| O_CLOEXEC);
506	if (fdKvm >= 0)
507	{
508	pVM->nem.s.fdKvm = fdKvm;
509
510	/*
511	* Create an empty VM since it is recommended we check capabilities on
512	* the VM rather than the system descriptor.
513	*/
514	int fdVm = ioctl(fdKvm, KVM_CREATE_VM, 0UL /* Type must be zero on x86 */);
515	if (fdVm >= 0)
516	{
517	pVM->nem.s.fdVm = fdVm;
518
519	/*
520	* Check capabilities.
521	*/
522	rc = nemR3LnxInitCheckCapabilities(pVM, pErrInfo);
523	if (RT_SUCCESS(rc))
524	{
525	/*
526	* Set up the VM (more on this later).
527	*/
528	rc = nemR3LnxInitSetupVm(pVM, pErrInfo);
529	if (RT_SUCCESS(rc))
530	{
531	/*
532	* Set ourselves as the execution engine and make config adjustments.
533	*/
534	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
535	Log(("NEM: Marked active!\n"));
536	PGMR3EnableNemMode(pVM);
537
538	/*
539	* Register release statistics
540	*/
541	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
542	{
543	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
544	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnDemand, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of on-demand state imports", "/NEM/CPU%u/ImportOnDemand", idCpu);
545	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of state imports on loop return", "/NEM/CPU%u/ImportOnReturn", idCpu);
546	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturnSkipped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of skipped state imports on loop return", "/NEM/CPU%u/ImportOnReturnSkipped", idCpu);
547	STAMR3RegisterF(pVM, &pNemCpu->StatImportPendingInterrupt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times an interrupt was pending when importing from KVM", "/NEM/CPU%u/ImportPendingInterrupt", idCpu);
548	STAMR3RegisterF(pVM, &pNemCpu->StatExportPendingInterrupt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times an interrupt was pending when exporting to KVM", "/NEM/CPU%u/ExportPendingInterrupt", idCpu);
549	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn", idCpu);
550	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn1Loop, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-01-loop", idCpu);
551	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn2Loops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-02-loops", idCpu);
552	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn3Loops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-03-loops", idCpu);
553	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn4PlusLoops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-04-to-7-loops", idCpu);
554	STAMR3RegisterF(pVM, &pNemCpu->StatQueryCpuTick, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of TSC queries", "/NEM/CPU%u/QueryCpuTick", idCpu);
555	STAMR3RegisterF(pVM, &pNemCpu->StatExitTotal, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "All exits", "/NEM/CPU%u/Exit", idCpu);
556	STAMR3RegisterF(pVM, &pNemCpu->StatExitIo, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_IO", "/NEM/CPU%u/Exit/Io", idCpu);
557	STAMR3RegisterF(pVM, &pNemCpu->StatExitMmio, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_MMIO", "/NEM/CPU%u/Exit/Mmio", idCpu);
558	STAMR3RegisterF(pVM, &pNemCpu->StatExitSetTpr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_SET_TRP", "/NEM/CPU%u/Exit/SetTpr", idCpu);
559	STAMR3RegisterF(pVM, &pNemCpu->StatExitTprAccess, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_TPR_ACCESS", "/NEM/CPU%u/Exit/TprAccess", idCpu);
560	STAMR3RegisterF(pVM, &pNemCpu->StatExitRdMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_RDMSR", "/NEM/CPU%u/Exit/RdMsr", idCpu);
561	STAMR3RegisterF(pVM, &pNemCpu->StatExitWrMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_WRMSR", "/NEM/CPU%u/Exit/WrMsr", idCpu);
562	STAMR3RegisterF(pVM, &pNemCpu->StatExitIrqWindowOpen, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_IRQ_WINDOWS_OPEN", "/NEM/CPU%u/Exit/IrqWindowOpen", idCpu);
563	STAMR3RegisterF(pVM, &pNemCpu->StatExitHalt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_HLT", "/NEM/CPU%u/Exit/Hlt", idCpu);
564	STAMR3RegisterF(pVM, &pNemCpu->StatExitIntr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTR", "/NEM/CPU%u/Exit/Intr", idCpu);
565	STAMR3RegisterF(pVM, &pNemCpu->StatExitHypercall, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_HYPERCALL", "/NEM/CPU%u/Exit/Hypercall", idCpu);
566	STAMR3RegisterF(pVM, &pNemCpu->StatExitDebug, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_DEBUG", "/NEM/CPU%u/Exit/Debug", idCpu);
567	STAMR3RegisterF(pVM, &pNemCpu->StatExitBusLock, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_BUS_LOCK", "/NEM/CPU%u/Exit/BusLock", idCpu);
568	STAMR3RegisterF(pVM, &pNemCpu->StatExitInternalErrorEmulation, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTERNAL_ERROR/EMULATION", "/NEM/CPU%u/Exit/InternalErrorEmulation", idCpu);
569	STAMR3RegisterF(pVM, &pNemCpu->StatExitInternalErrorFatal, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTERNAL_ERROR/*", "/NEM/CPU%u/Exit/InternalErrorFatal", idCpu);
570	}
571
572	/*
573	* Success.
574	*/
575	return VINF_SUCCESS;
576	}
577
578	/*
579	* Bail out.
580	*/
581	}
582	close(fdVm);
583	pVM->nem.s.fdVm = -1;
584	}
585	else
586	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VM failed: %u", errno);
587	close(fdKvm);
588	pVM->nem.s.fdKvm = -1;
589	}
590	else if (errno == EACCES)
591	rc = RTErrInfoSet(pErrInfo, VERR_ACCESS_DENIED, "Do not have access to open /dev/kvm for reading & writing.");
592	else if (errno == ENOENT)
593	rc = RTErrInfoSet(pErrInfo, VERR_NOT_SUPPORTED, "KVM is not availble (/dev/kvm does not exist)");
594	else
595	rc = RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno), "Failed to open '/dev/kvm': %u", errno);
596
597	/*
598	* We only fail if in forced mode, otherwise just log the complaint and return.
599	*/
600	Assert(RTErrInfoIsSet(pErrInfo));
601	if ( (fForced \|\| !fFallback)
602	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
603	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
604	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
605	return VINF_SUCCESS;
606	}
607
608
609	/**
610	* This is called after CPUMR3Init is done.
611	*
612	* @returns VBox status code.
613	* @param pVM The VM handle..
614	*/
615	int nemR3NativeInitAfterCPUM(PVM pVM)
616	{
617	/*
618	* Validate sanity.
619	*/
620	AssertReturn(pVM->nem.s.fdKvm >= 0, VERR_WRONG_ORDER);
621	AssertReturn(pVM->nem.s.fdVm >= 0, VERR_WRONG_ORDER);
622	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
623
624	/** @todo */
625
626	return VINF_SUCCESS;
627	}
628
629
630	/**
631	* Update the CPUID leaves for a VCPU.
632	*
633	* The KVM_SET_CPUID2 call replaces any previous leaves, so we have to redo
634	* everything when there really just are single bit changes. That said, it
635	* looks like KVM update the XCR/XSAVE related stuff as well as the APIC enabled
636	* bit(s), so it should suffice if we do this at startup, I hope.
637	*/
638	static int nemR3LnxUpdateCpuIdsLeaves(PVM pVM, PVMCPU pVCpu)
639	{
640	uint32_t cLeaves = 0;
641	PCCPUMCPUIDLEAF const paLeaves = CPUMR3CpuIdGetPtr(pVM, &cLeaves);
642	struct kvm_cpuid2 pReq = (struct kvm_cpuid2 )alloca(RT_UOFFSETOF_DYN(struct kvm_cpuid2, entries[cLeaves + 2]));
643
644	pReq->nent = cLeaves;
645	pReq->padding = 0;
646
647	for (uint32_t i = 0; i < cLeaves; i++)
648	{
649	CPUMGetGuestCpuId(pVCpu, paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, -1 /f64BitMode/,
650	&pReq->entries[i].eax,
651	&pReq->entries[i].ebx,
652	&pReq->entries[i].ecx,
653	&pReq->entries[i].edx);
654	pReq->entries[i].function = paLeaves[i].uLeaf;
655	pReq->entries[i].index = paLeaves[i].uSubLeaf;
656	pReq->entries[i].flags = !paLeaves[i].fSubLeafMask ? 0 : KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
657	pReq->entries[i].padding[0] = 0;
658	pReq->entries[i].padding[1] = 0;
659	pReq->entries[i].padding[2] = 0;
660	}
661
662	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_CPUID2, pReq);
663	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d cLeaves=%#x\n", rcLnx, errno, cLeaves), RTErrConvertFromErrno(errno));
664
665	return VINF_SUCCESS;
666	}
667
668
669	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
670	{
671	/*
672	* Make RTThreadPoke work again (disabled for avoiding unnecessary
673	* critical section issues in ring-0).
674	*/
675	if (enmWhat == VMINITCOMPLETED_RING3)
676	VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, nemR3LnxFixThreadPoke, NULL);
677
678	/*
679	* Configure CPUIDs after ring-3 init has been done.
680	*/
681	if (enmWhat == VMINITCOMPLETED_RING3)
682	{
683	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
684	{
685	int rc = nemR3LnxUpdateCpuIdsLeaves(pVM, pVM->apCpusR3[idCpu]);
686	AssertRCReturn(rc, rc);
687	}
688	}
689
690	/*
691	* Configure MSRs after ring-3 init is done.
692	*
693	* We only need to tell KVM which MSRs it can handle, as we already
694	* requested KVM_MSR_EXIT_REASON_FILTER, KVM_MSR_EXIT_REASON_UNKNOWN
695	* and KVM_MSR_EXIT_REASON_INVAL in nemR3LnxInitSetupVm, and here we
696	* will use KVM_MSR_FILTER_DEFAULT_DENY. So, all MSRs w/o a 1 in the
697	* bitmaps should be deferred to ring-3.
698	*/
699	if (enmWhat == VMINITCOMPLETED_RING3)
700	{
701	struct kvm_msr_filter MsrFilters = {0}; /* Structure with a couple of implicit paddings on 64-bit systems. */
702	MsrFilters.flags = KVM_MSR_FILTER_DEFAULT_DENY;
703
704	unsigned iRange = 0;
705	#define MSR_RANGE_BEGIN(a_uBase, a_uEnd, a_fFlags) \
706	AssertCompile(0x3000 <= KVM_MSR_FILTER_MAX_BITMAP_SIZE * 8); \
707	uint64_t RT_CONCAT(bm, a_uBase)[0x3000 / 64] = {0}; \
708	do { \
709	uint64_t * const pbm = RT_CONCAT(bm, a_uBase); \
710	uint32_t const uBase = UINT32_C(a_uBase); \
711	uint32_t const cMsrs = UINT32_C(a_uEnd) - UINT32_C(a_uBase); \
712	MsrFilters.ranges[iRange].base = UINT32_C(a_uBase); \
713	MsrFilters.ranges[iRange].nmsrs = cMsrs; \
714	MsrFilters.ranges[iRange].flags = (a_fFlags); \
715	MsrFilters.ranges[iRange].bitmap = (uint8_t *)&RT_CONCAT(bm, a_uBase)[0]
716	#define MSR_RANGE_ADD(a_Msr) \
717	do { Assert((uint32_t)(a_Msr) - uBase < cMsrs); ASMBitSet(pbm, (uint32_t)(a_Msr) - uBase); } while (0)
718	#define MSR_RANGE_END(a_cMinMsrs) \
719	/* optimize the range size before closing: */ \
720	uint32_t cBitmap = cMsrs / 64; \
721	while (cBitmap > ((a_cMinMsrs) + 63 / 64) && pbm[cBitmap - 1] == 0) \
722	cBitmap -= 1; \
723	MsrFilters.ranges[iRange].nmsrs = cBitmap * 64; \
724	iRange++; \
725	} while (0)
726
727	/* 1st Intel range: 0000_0000 to 0000_3000. */
728	MSR_RANGE_BEGIN(0x00000000, 0x00003000, KVM_MSR_FILTER_READ \| KVM_MSR_FILTER_WRITE);
729	MSR_RANGE_ADD(MSR_IA32_TSC);
730	MSR_RANGE_ADD(MSR_IA32_SYSENTER_CS);
731	MSR_RANGE_ADD(MSR_IA32_SYSENTER_ESP);
732	MSR_RANGE_ADD(MSR_IA32_SYSENTER_EIP);
733	MSR_RANGE_ADD(MSR_IA32_CR_PAT);
734	/** @todo more? */
735	MSR_RANGE_END(64);
736
737	/* 1st AMD range: c000_0000 to c000_3000 */
738	MSR_RANGE_BEGIN(0xc0000000, 0xc0003000, KVM_MSR_FILTER_READ \| KVM_MSR_FILTER_WRITE);
739	MSR_RANGE_ADD(MSR_K6_EFER);
740	MSR_RANGE_ADD(MSR_K6_STAR);
741	MSR_RANGE_ADD(MSR_K8_GS_BASE);
742	MSR_RANGE_ADD(MSR_K8_KERNEL_GS_BASE);
743	MSR_RANGE_ADD(MSR_K8_LSTAR);
744	MSR_RANGE_ADD(MSR_K8_CSTAR);
745	MSR_RANGE_ADD(MSR_K8_SF_MASK);
746	MSR_RANGE_ADD(MSR_K8_TSC_AUX);
747	/** @todo add more? */
748	MSR_RANGE_END(64);
749
750	/** @todo Specify other ranges too? Like hyper-V and KVM to make sure we get
751	* the MSR requests instead of KVM. */
752
753	int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_X86_SET_MSR_FILTER, &MsrFilters);
754	if (rcLnx == -1)
755	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
756	"Failed to enable KVM_X86_SET_MSR_FILTER failed: %u", errno);
757	}
758
759	return VINF_SUCCESS;
760	}
761
762
763	int nemR3NativeTerm(PVM pVM)
764	{
765	/*
766	* Per-cpu data
767	*/
768	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
769	{
770	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
771
772	if (pVCpu->nem.s.fdVCpu != -1)
773	{
774	close(pVCpu->nem.s.fdVCpu);
775	pVCpu->nem.s.fdVCpu = -1;
776	}
777	if (pVCpu->nem.s.pRun)
778	{
779	munmap(pVCpu->nem.s.pRun, pVM->nem.s.cbVCpuMmap);
780	pVCpu->nem.s.pRun = NULL;
781	}
782	}
783
784	/*
785	* Global data.
786	*/
787	if (pVM->nem.s.fdVm != -1)
788	{
789	close(pVM->nem.s.fdVm);
790	pVM->nem.s.fdVm = -1;
791	}
792
793	if (pVM->nem.s.fdKvm != -1)
794	{
795	close(pVM->nem.s.fdKvm);
796	pVM->nem.s.fdKvm = -1;
797	}
798	return VINF_SUCCESS;
799	}
800
801
802	/**
803	* VM reset notification.
804	*
805	* @param pVM The cross context VM structure.
806	*/
807	void nemR3NativeReset(PVM pVM)
808	{
809	RT_NOREF(pVM);
810	}
811
812
813	/**
814	* Reset CPU due to INIT IPI or hot (un)plugging.
815	*
816	* @param pVCpu The cross context virtual CPU structure of the CPU being
817	* reset.
818	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
819	*/
820	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
821	{
822	RT_NOREF(pVCpu, fInitIpi);
823	}
824
825
826	/*********************************************************************************************************************************
827	* Memory management *
828	*********************************************************************************************************************************/
829
830
831	/**
832	* Allocates a memory slot ID.
833	*
834	* @returns Slot ID on success, UINT16_MAX on failure.
835	*/
836	static uint16_t nemR3LnxMemSlotIdAlloc(PVM pVM)
837	{
838	/* Use the hint first. */
839	uint16_t idHint = pVM->nem.s.idPrevSlot;
840	if (idHint < _32K - 1)
841	{
842	int32_t idx = ASMBitNextClear(&pVM->nem.s.bmSlotIds, _32K, idHint);
843	Assert(idx < _32K);
844	if (idx > 0 && !ASMAtomicBitTestAndSet(&pVM->nem.s.bmSlotIds, idx))
845	return pVM->nem.s.idPrevSlot = (uint16_t)idx;
846	}
847
848	/*
849	* Search the whole map from the start.
850	*/
851	int32_t idx = ASMBitFirstClear(&pVM->nem.s.bmSlotIds, _32K);
852	Assert(idx < _32K);
853	if (idx > 0 && !ASMAtomicBitTestAndSet(&pVM->nem.s.bmSlotIds, idx))
854	return pVM->nem.s.idPrevSlot = (uint16_t)idx;
855
856	Assert(idx < 0 /shouldn't trigger unless there is a race /);
857	return UINT16_MAX; /* caller is expected to assert. */
858	}
859
860
861	/**
862	* Frees a memory slot ID
863	*/
864	static void nemR3LnxMemSlotIdFree(PVM pVM, uint16_t idSlot)
865	{
866	if (RT_LIKELY(idSlot < _32K && ASMAtomicBitTestAndClear(&pVM->nem.s.bmSlotIds, idSlot)))
867	{ /likely/ }
868	else
869	AssertMsgFailed(("idSlot=%u (%#x)\n", idSlot, idSlot));
870	}
871
872
873
874	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
875	uint8_t pu2State, uint32_t puNemRange)
876	{
877	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
878	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
879
880	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p pu2State=%p (%d) puNemRange=%p (%d) - idSlot=%#x\n",
881	GCPhys, cb, pvR3, pu2State, pu2State, puNemRange, *puNemRange, idSlot));
882
883	struct kvm_userspace_memory_region Region;
884	Region.slot = idSlot;
885	Region.flags = 0;
886	Region.guest_phys_addr = GCPhys;
887	Region.memory_size = cb;
888	Region.userspace_addr = (uintptr_t)pvR3;
889
890	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
891	if (rc == 0)
892	{
893	*pu2State = 0;
894	*puNemRange = idSlot;
895	return VINF_SUCCESS;
896	}
897
898	LogRel(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p, idSlot=%#x failed: %u/%u\n", GCPhys, cb, pvR3, idSlot, rc, errno));
899	nemR3LnxMemSlotIdFree(pVM, idSlot);
900	return VERR_NEM_MAP_PAGES_FAILED;
901	}
902
903
904	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
905	{
906	RT_NOREF(pVM);
907	return true;
908	}
909
910
911	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
912	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
913	{
914	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d) puNemRange=%p (%#x)\n",
915	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, pu2State, puNemRange, puNemRange ? puNemRange : UINT32_MAX));
916	RT_NOREF(pvRam);
917
918	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
919	{
920	/** @todo implement splitting and whatnot of ranges if we want to be 100%
921	* conforming (just modify RAM registrations in MM.cpp to test). */
922	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p\n", GCPhys, cb, fFlags, pvRam, pvMmio2),
923	VERR_NEM_MAP_PAGES_FAILED);
924	}
925
926	/*
927	* Register MMIO2.
928	*/
929	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
930	{
931	AssertReturn(pvMmio2, VERR_NEM_MAP_PAGES_FAILED);
932	AssertReturn(puNemRange, VERR_NEM_MAP_PAGES_FAILED);
933
934	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
935	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
936
937	struct kvm_userspace_memory_region Region;
938	Region.slot = idSlot;
939	Region.flags = fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES ? KVM_MEM_LOG_DIRTY_PAGES : 0;
940	Region.guest_phys_addr = GCPhys;
941	Region.memory_size = cb;
942	Region.userspace_addr = (uintptr_t)pvMmio2;
943
944	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
945	if (rc == 0)
946	{
947	*pu2State = 0;
948	*puNemRange = idSlot;
949	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvMmio2=%p - idSlot=%#x\n",
950	GCPhys, cb, fFlags, pvMmio2, idSlot));
951	return VINF_SUCCESS;
952	}
953
954	nemR3LnxMemSlotIdFree(pVM, idSlot);
955	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvMmio2=%p, idSlot=%#x failed: %u/%u\n",
956	GCPhys, cb, fFlags, pvMmio2, idSlot, errno, rc),
957	VERR_NEM_MAP_PAGES_FAILED);
958	}
959
960	/* MMIO, don't care. */
961	*pu2State = 0;
962	*puNemRange = UINT32_MAX;
963	return VINF_SUCCESS;
964	}
965
966
967	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
968	void pvRam, void pvMmio2, uint32_t *puNemRange)
969	{
970	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
971	return VINF_SUCCESS;
972	}
973
974
975	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
976	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
977	{
978	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p puNemRange=%p (%#x)\n",
979	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
980	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
981
982	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
983	{
984	/** @todo implement splitting and whatnot of ranges if we want to be 100%
985	* conforming (just modify RAM registrations in MM.cpp to test). */
986	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p\n", GCPhys, cb, fFlags, pvRam, pvMmio2),
987	VERR_NEM_UNMAP_PAGES_FAILED);
988	}
989
990	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
991	{
992	uint32_t const idSlot = *puNemRange;
993	AssertReturn(idSlot > 0 && idSlot < _32K, VERR_NEM_IPE_4);
994	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, idSlot), VERR_NEM_IPE_4);
995
996	struct kvm_userspace_memory_region Region;
997	Region.slot = idSlot;
998	Region.flags = 0;
999	Region.guest_phys_addr = GCPhys;
1000	Region.memory_size = 0; /* this deregisters it. */
1001	Region.userspace_addr = (uintptr_t)pvMmio2;
1002
1003	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
1004	if (rc == 0)
1005	{
1006	if (pu2State)
1007	*pu2State = 0;
1008	*puNemRange = UINT32_MAX;
1009	nemR3LnxMemSlotIdFree(pVM, idSlot);
1010	return VINF_SUCCESS;
1011	}
1012
1013	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvMmio2=%p, idSlot=%#x failed: %u/%u\n",
1014	GCPhys, cb, fFlags, pvMmio2, idSlot, errno, rc),
1015	VERR_NEM_UNMAP_PAGES_FAILED);
1016	}
1017
1018	if (pu2State)
1019	*pu2State = UINT8_MAX;
1020	return VINF_SUCCESS;
1021	}
1022
1023
1024	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
1025	void *pvBitmap, size_t cbBitmap)
1026	{
1027	AssertReturn(uNemRange > 0 && uNemRange < _32K, VERR_NEM_IPE_4);
1028	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, uNemRange), VERR_NEM_IPE_4);
1029
1030	RT_NOREF(GCPhys, cbBitmap);
1031
1032	struct kvm_dirty_log DirtyLog;
1033	DirtyLog.slot = uNemRange;
1034	DirtyLog.padding1 = 0;
1035	DirtyLog.dirty_bitmap = pvBitmap;
1036
1037	int rc = ioctl(pVM->nem.s.fdVm, KVM_GET_DIRTY_LOG, &DirtyLog);
1038	AssertLogRelMsgReturn(rc == 0, ("%RGp LB %RGp idSlot=%#x failed: %u/%u\n", GCPhys, cb, uNemRange, errno, rc),
1039	VERR_NEM_QUERY_DIRTY_BITMAP_FAILED);
1040
1041	return VINF_SUCCESS;
1042	}
1043
1044
1045	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
1046	uint8_t pu2State, uint32_t puNemRange)
1047	{
1048	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
1049	*pu2State = UINT8_MAX;
1050
1051	/* Don't support puttint ROM where there is already RAM. For
1052	now just shuffle the registrations till it works... */
1053	AssertLogRelMsgReturn(!(fFlags & NEM_NOTIFY_PHYS_ROM_F_REPLACE), ("%RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags),
1054	VERR_NEM_MAP_PAGES_FAILED);
1055
1056	/** @todo figure out how to do shadow ROMs. */
1057
1058	/*
1059	* We only allocate a slot number here in case we need to use it to
1060	* fend of physical handler fun.
1061	*/
1062	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
1063	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
1064
1065	*pu2State = 0;
1066	*puNemRange = idSlot;
1067	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x pvPages=%p - idSlot=%#x\n",
1068	GCPhys, cb, fFlags, pvPages, idSlot));
1069	RT_NOREF(GCPhys, cb, fFlags, pvPages);
1070	return VINF_SUCCESS;
1071	}
1072
1073
1074	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
1075	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
1076	{
1077	Log5(("NEMR3NotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
1078	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
1079
1080	AssertPtrReturn(pvPages, VERR_NEM_IPE_5);
1081
1082	uint32_t const idSlot = *puNemRange;
1083	AssertReturn(idSlot > 0 && idSlot < _32K, VERR_NEM_IPE_4);
1084	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, idSlot), VERR_NEM_IPE_4);
1085
1086	*pu2State = UINT8_MAX;
1087
1088	/*
1089	* Do the actual setting of the user pages here now that we've
1090	* got a valid pvPages (typically isn't available during the early
1091	* notification, unless we're replacing RAM).
1092	*/
1093	struct kvm_userspace_memory_region Region;
1094	Region.slot = idSlot;
1095	Region.flags = 0;
1096	Region.guest_phys_addr = GCPhys;
1097	Region.memory_size = cb;
1098	Region.userspace_addr = (uintptr_t)pvPages;
1099
1100	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
1101	if (rc == 0)
1102	{
1103	*pu2State = 0;
1104	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x pvPages=%p - idSlot=%#x\n",
1105	GCPhys, cb, fFlags, pvPages, idSlot));
1106	return VINF_SUCCESS;
1107	}
1108	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvPages=%p, idSlot=%#x failed: %u/%u\n",
1109	GCPhys, cb, fFlags, pvPages, idSlot, errno, rc),
1110	VERR_NEM_MAP_PAGES_FAILED);
1111	}
1112
1113
1114	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
1115	{
1116	Log(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
1117	Assert(VM_IS_NEM_ENABLED(pVCpu->CTX_SUFF(pVM)));
1118	RT_NOREF(pVCpu, fEnabled);
1119	}
1120
1121
1122	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
1123	RTR3PTR pvMemR3, uint8_t *pu2State)
1124	{
1125	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
1126	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
1127
1128	*pu2State = UINT8_MAX;
1129	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
1130	}
1131
1132
1133	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
1134	{
1135	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
1136	RT_NOREF(pVM, enmKind, GCPhys, cb);
1137	}
1138
1139
1140	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
1141	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
1142	{
1143	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
1144	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
1145	RT_NOREF(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM);
1146	}
1147
1148
1149	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
1150	PGMPAGETYPE enmType, uint8_t *pu2State)
1151	{
1152	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
1153	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
1154	RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
1155	return VINF_SUCCESS;
1156	}
1157
1158
1159	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
1160	PGMPAGETYPE enmType, uint8_t *pu2State)
1161	{
1162	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
1163	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
1164	Assert(VM_IS_NEM_ENABLED(pVM));
1165	RT_NOREF(pVM, GCPhys, HCPhys, pvR3, fPageProt, enmType, pu2State);
1166
1167	}
1168
1169
1170	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
1171	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
1172	{
1173	Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp pvNewR3=%p fPageProt=%#x enmType=%d *pu2State=%d\n",
1174	GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, *pu2State));
1175	Assert(VM_IS_NEM_ENABLED(pVM));
1176	RT_NOREF(pVM, GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, pu2State);
1177	}
1178
1179
1180	/*********************************************************************************************************************************
1181	* CPU State *
1182	*********************************************************************************************************************************/
1183
1184	/**
1185	* Worker that imports selected state from KVM.
1186	*/
1187	static int nemHCLnxImportState(PVMCPUCC pVCpu, uint64_t fWhat, PCPUMCTX pCtx, struct kvm_run *pRun)
1188	{
1189	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
1190	if (!fWhat)
1191	return VINF_SUCCESS;
1192
1193	/*
1194	* Stuff that goes into kvm_run::s.regs.regs:
1195	*/
1196	if (fWhat & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK))
1197	{
1198	if (fWhat & CPUMCTX_EXTRN_RIP)
1199	pCtx->rip = pRun->s.regs.regs.rip;
1200	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
1201	pCtx->rflags.u = pRun->s.regs.regs.rflags;
1202
1203	if (fWhat & CPUMCTX_EXTRN_RAX)
1204	pCtx->rax = pRun->s.regs.regs.rax;
1205	if (fWhat & CPUMCTX_EXTRN_RCX)
1206	pCtx->rcx = pRun->s.regs.regs.rcx;
1207	if (fWhat & CPUMCTX_EXTRN_RDX)
1208	pCtx->rdx = pRun->s.regs.regs.rdx;
1209	if (fWhat & CPUMCTX_EXTRN_RBX)
1210	pCtx->rbx = pRun->s.regs.regs.rbx;
1211	if (fWhat & CPUMCTX_EXTRN_RSP)
1212	pCtx->rsp = pRun->s.regs.regs.rsp;
1213	if (fWhat & CPUMCTX_EXTRN_RBP)
1214	pCtx->rbp = pRun->s.regs.regs.rbp;
1215	if (fWhat & CPUMCTX_EXTRN_RSI)
1216	pCtx->rsi = pRun->s.regs.regs.rsi;
1217	if (fWhat & CPUMCTX_EXTRN_RDI)
1218	pCtx->rdi = pRun->s.regs.regs.rdi;
1219	if (fWhat & CPUMCTX_EXTRN_R8_R15)
1220	{
1221	pCtx->r8 = pRun->s.regs.regs.r8;
1222	pCtx->r9 = pRun->s.regs.regs.r9;
1223	pCtx->r10 = pRun->s.regs.regs.r10;
1224	pCtx->r11 = pRun->s.regs.regs.r11;
1225	pCtx->r12 = pRun->s.regs.regs.r12;
1226	pCtx->r13 = pRun->s.regs.regs.r13;
1227	pCtx->r14 = pRun->s.regs.regs.r14;
1228	pCtx->r15 = pRun->s.regs.regs.r15;
1229	}
1230	}
1231
1232	/*
1233	* Stuff that goes into kvm_run::s.regs.sregs.
1234	*
1235	* Note! The apic_base can be ignored because we gets all MSR writes to it
1236	* and VBox always keeps the correct value.
1237	*/
1238	bool fMaybeChangedMode = false;
1239	bool fUpdateCr3 = false;
1240	if (fWhat & ( CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_CR_MASK
1241	\| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_APIC_TPR))
1242	{
1243	/** @todo what about Attr.n.u4LimitHigh? */
1244	#define NEM_LNX_IMPORT_SEG(a_CtxSeg, a_KvmSeg) do { \
1245	(a_CtxSeg).u64Base = (a_KvmSeg).base; \
1246	(a_CtxSeg).u32Limit = (a_KvmSeg).limit; \
1247	(a_CtxSeg).ValidSel = (a_CtxSeg).Sel = (a_KvmSeg).selector; \
1248	(a_CtxSeg).Attr.n.u4Type = (a_KvmSeg).type; \
1249	(a_CtxSeg).Attr.n.u1DescType = (a_KvmSeg).s; \
1250	(a_CtxSeg).Attr.n.u2Dpl = (a_KvmSeg).dpl; \
1251	(a_CtxSeg).Attr.n.u1Present = (a_KvmSeg).present; \
1252	(a_CtxSeg).Attr.n.u1Available = (a_KvmSeg).avl; \
1253	(a_CtxSeg).Attr.n.u1Long = (a_KvmSeg).l; \
1254	(a_CtxSeg).Attr.n.u1DefBig = (a_KvmSeg).db; \
1255	(a_CtxSeg).Attr.n.u1Granularity = (a_KvmSeg).g; \
1256	(a_CtxSeg).Attr.n.u1Unusable = (a_KvmSeg).unusable; \
1257	(a_CtxSeg).fFlags = CPUMSELREG_FLAGS_VALID; \
1258	CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &(a_CtxSeg)); \
1259	} while (0)
1260
1261	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
1262	{
1263	if (fWhat & CPUMCTX_EXTRN_ES)
1264	NEM_LNX_IMPORT_SEG(pCtx->es, pRun->s.regs.sregs.es);
1265	if (fWhat & CPUMCTX_EXTRN_CS)
1266	NEM_LNX_IMPORT_SEG(pCtx->cs, pRun->s.regs.sregs.cs);
1267	if (fWhat & CPUMCTX_EXTRN_SS)
1268	NEM_LNX_IMPORT_SEG(pCtx->ss, pRun->s.regs.sregs.ss);
1269	if (fWhat & CPUMCTX_EXTRN_DS)
1270	NEM_LNX_IMPORT_SEG(pCtx->ds, pRun->s.regs.sregs.ds);
1271	if (fWhat & CPUMCTX_EXTRN_FS)
1272	NEM_LNX_IMPORT_SEG(pCtx->fs, pRun->s.regs.sregs.fs);
1273	if (fWhat & CPUMCTX_EXTRN_GS)
1274	NEM_LNX_IMPORT_SEG(pCtx->gs, pRun->s.regs.sregs.gs);
1275	}
1276	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
1277	{
1278	if (fWhat & CPUMCTX_EXTRN_GDTR)
1279	{
1280	pCtx->gdtr.pGdt = pRun->s.regs.sregs.gdt.base;
1281	pCtx->gdtr.cbGdt = pRun->s.regs.sregs.gdt.limit;
1282	}
1283	if (fWhat & CPUMCTX_EXTRN_IDTR)
1284	{
1285	pCtx->idtr.pIdt = pRun->s.regs.sregs.idt.base;
1286	pCtx->idtr.cbIdt = pRun->s.regs.sregs.idt.limit;
1287	}
1288	if (fWhat & CPUMCTX_EXTRN_LDTR)
1289	NEM_LNX_IMPORT_SEG(pCtx->ldtr, pRun->s.regs.sregs.ldt);
1290	if (fWhat & CPUMCTX_EXTRN_TR)
1291	NEM_LNX_IMPORT_SEG(pCtx->tr, pRun->s.regs.sregs.tr);
1292	}
1293	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
1294	{
1295	if (fWhat & CPUMCTX_EXTRN_CR0)
1296	{
1297	if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
1298	{
1299	CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
1300	fMaybeChangedMode = true;
1301	}
1302	}
1303	if (fWhat & CPUMCTX_EXTRN_CR2)
1304	pCtx->cr2 = pRun->s.regs.sregs.cr2;
1305	if (fWhat & CPUMCTX_EXTRN_CR3)
1306	{
1307	if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
1308	{
1309	CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
1310	fUpdateCr3 = true;
1311	}
1312	}
1313	if (fWhat & CPUMCTX_EXTRN_CR4)
1314	{
1315	if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
1316	{
1317	CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
1318	fMaybeChangedMode = true;
1319	}
1320	}
1321	}
1322	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1323	APICSetTpr(pVCpu, (uint8_t)pRun->s.regs.sregs.cr8 << 4);
1324	if (fWhat & CPUMCTX_EXTRN_EFER)
1325	{
1326	if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
1327	{
1328	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
1329	if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1330	PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
1331	pCtx->msrEFER = pRun->s.regs.sregs.efer;
1332	fMaybeChangedMode = true;
1333	}
1334	}
1335	#undef NEM_LNX_IMPORT_SEG
1336	}
1337
1338	/*
1339	* Debug registers.
1340	*/
1341	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
1342	{
1343	struct kvm_debugregs DbgRegs = {{0}};
1344	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
1345	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1346
1347	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1348	{
1349	pCtx->dr[0] = DbgRegs.db[0];
1350	pCtx->dr[1] = DbgRegs.db[1];
1351	pCtx->dr[2] = DbgRegs.db[2];
1352	pCtx->dr[3] = DbgRegs.db[3];
1353	}
1354	if (fWhat & CPUMCTX_EXTRN_DR6)
1355	pCtx->dr[6] = DbgRegs.dr6;
1356	if (fWhat & CPUMCTX_EXTRN_DR7)
1357	pCtx->dr[7] = DbgRegs.dr7;
1358	}
1359
1360	/*
1361	* FPU, SSE, AVX, ++.
1362	*/
1363	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx))
1364	{
1365	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1366	{
1367	fWhat \|= CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE; /* we do all or nothing at all */
1368
1369	AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
1370	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XSAVE, &pCtx->XState);
1371	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1372	}
1373
1374	if (fWhat & CPUMCTX_EXTRN_XCRx)
1375	{
1376	struct kvm_xcrs Xcrs =
1377	{ /.nr_xcrs = / 2,
1378	/.flags = / 0,
1379	/.xcrs= / {
1380	{ /.xcr =/ 0, /.reserved=/ 0, /.value=/ pCtx->aXcr[0] },
1381	{ /.xcr =/ 1, /.reserved=/ 0, /.value=/ pCtx->aXcr[1] },
1382	}
1383	};
1384
1385	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XCRS, &Xcrs);
1386	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1387
1388	pCtx->aXcr[0] = Xcrs.xcrs[0].value;
1389	pCtx->aXcr[1] = Xcrs.xcrs[1].value;
1390	}
1391	}
1392
1393	/*
1394	* MSRs.
1395	*/
1396	if (fWhat & ( CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_SYSENTER_MSRS
1397	\| CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
1398	{
1399	union
1400	{
1401	struct kvm_msrs Core;
1402	uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
1403	} uBuf;
1404	uint64_t *pauDsts[32];
1405	uint32_t iMsr = 0;
1406	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1407
1408	#define ADD_MSR(a_Msr, a_uValue) do { \
1409	Assert(iMsr < 32); \
1410	uBuf.Core.entries[iMsr].index = (a_Msr); \
1411	uBuf.Core.entries[iMsr].reserved = 0; \
1412	uBuf.Core.entries[iMsr].data = UINT64_MAX; \
1413	pauDsts[iMsr] = &(a_uValue); \
1414	iMsr += 1; \
1415	} while (0)
1416
1417	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1418	ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1419	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1420	{
1421	ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
1422	ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
1423	ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
1424	ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1425	}
1426	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1427	{
1428	ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
1429	ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
1430	ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
1431	}
1432	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1433	ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1434	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1435	{
1436	ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
1437	/** @todo What do we _have_ to add here?
1438	* We also have: Mttr, MiscEnable, FeatureControl. /
1439	}
1440
1441	uBuf.Core.pad = 0;
1442	uBuf.Core.nmsrs = iMsr;
1443	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MSRS, &uBuf);
1444	AssertMsgReturn(rc == (int)iMsr,
1445	("rc=%d iMsr=%d (->%#x) errno=%d\n",
1446	rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
1447	VERR_NEM_IPE_3);
1448
1449	while (iMsr-- > 0)
1450	*pauDsts[iMsr] = uBuf.Core.entries[iMsr].data;
1451	#undef ADD_MSR
1452	}
1453
1454	/*
1455	* Interruptibility state and pending interrupts.
1456	*/
1457	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1458	{
1459	fWhat \|= CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI; /* always do both, see export and interrupt FF handling */
1460
1461	struct kvm_vcpu_events KvmEvents = {0};
1462	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
1463	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
1464
1465	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
1466	pVCpu->cpum.GstCtx.rip = pRun->s.regs.regs.rip;
1467
1468	if (KvmEvents.interrupt.shadow)
1469	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
1470	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1471	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1472
1473	if (KvmEvents.nmi.masked)
1474	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
1475	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
1476	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
1477
1478	if (KvmEvents.interrupt.injected)
1479	{
1480	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportPendingInterrupt);
1481	TRPMAssertTrap(pVCpu, KvmEvents.interrupt.nr, !KvmEvents.interrupt.soft ? TRPM_HARDWARE_INT : TRPM_SOFTWARE_INT);
1482	}
1483
1484	Assert(KvmEvents.nmi.injected == 0);
1485	Assert(KvmEvents.nmi.pending == 0);
1486	}
1487
1488	/*
1489	* Update the external mask.
1490	*/
1491	pCtx->fExtrn &= ~fWhat;
1492	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1493	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1494	pVCpu->cpum.GstCtx.fExtrn = 0;
1495
1496	/*
1497	* We sometimes need to update PGM on the guest status.
1498	*/
1499	if (!fMaybeChangedMode && !fUpdateCr3)
1500	{ /* likely */ }
1501	else
1502	{
1503	/*
1504	* Make sure we got all the state PGM might need.
1505	*/
1506	Log7(("nemHCLnxImportState: fMaybeChangedMode=%d fUpdateCr3=%d fExtrnNeeded=%#RX64\n", fMaybeChangedMode, fUpdateCr3,
1507	pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER) ));
1508	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER))
1509	{
1510	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR0)
1511	{
1512	if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
1513	{
1514	CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
1515	fMaybeChangedMode = true;
1516	}
1517	}
1518	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR3)
1519	{
1520	if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
1521	{
1522	CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
1523	fUpdateCr3 = true;
1524	}
1525	}
1526	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR4)
1527	{
1528	if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
1529	{
1530	CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
1531	fMaybeChangedMode = true;
1532	}
1533	}
1534	if (fWhat & CPUMCTX_EXTRN_EFER)
1535	{
1536	if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
1537	{
1538	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
1539	if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1540	PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
1541	pCtx->msrEFER = pRun->s.regs.sregs.efer;
1542	fMaybeChangedMode = true;
1543	}
1544	}
1545
1546	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER);
1547	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1548	pVCpu->cpum.GstCtx.fExtrn = 0;
1549	}
1550
1551	/*
1552	* Notify PGM about the changes.
1553	*/
1554	if (fMaybeChangedMode)
1555	{
1556	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4,
1557	pVCpu->cpum.GstCtx.msrEFER, false /fForce/);
1558	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1559	}
1560
1561	if (fUpdateCr3)
1562	{
1563	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
1564	if (rc == VINF_SUCCESS)
1565	{ /* likely */ }
1566	else
1567	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1568	}
1569	}
1570
1571	return VINF_SUCCESS;
1572	}
1573
1574
1575	/**
1576	* Interface for importing state on demand (used by IEM).
1577	*
1578	* @returns VBox status code.
1579	* @param pVCpu The cross context CPU structure.
1580	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1581	*/
1582	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1583	{
1584	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
1585	return nemHCLnxImportState(pVCpu, fWhat, &pVCpu->cpum.GstCtx, pVCpu->nem.s.pRun);
1586	}
1587
1588
1589	/**
1590	* Exports state to KVM.
1591	*/
1592	static int nemHCLnxExportState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, struct kvm_run *pRun)
1593	{
1594	uint64_t const fExtrn = ~pCtx->fExtrn & CPUMCTX_EXTRN_ALL;
1595	Assert((~fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL);
1596
1597	/*
1598	* Stuff that goes into kvm_run::s.regs.regs:
1599	*/
1600	if (fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK))
1601	{
1602	if (fExtrn & CPUMCTX_EXTRN_RIP)
1603	pRun->s.regs.regs.rip = pCtx->rip;
1604	if (fExtrn & CPUMCTX_EXTRN_RFLAGS)
1605	pRun->s.regs.regs.rflags = pCtx->rflags.u;
1606
1607	if (fExtrn & CPUMCTX_EXTRN_RAX)
1608	pRun->s.regs.regs.rax = pCtx->rax;
1609	if (fExtrn & CPUMCTX_EXTRN_RCX)
1610	pRun->s.regs.regs.rcx = pCtx->rcx;
1611	if (fExtrn & CPUMCTX_EXTRN_RDX)
1612	pRun->s.regs.regs.rdx = pCtx->rdx;
1613	if (fExtrn & CPUMCTX_EXTRN_RBX)
1614	pRun->s.regs.regs.rbx = pCtx->rbx;
1615	if (fExtrn & CPUMCTX_EXTRN_RSP)
1616	pRun->s.regs.regs.rsp = pCtx->rsp;
1617	if (fExtrn & CPUMCTX_EXTRN_RBP)
1618	pRun->s.regs.regs.rbp = pCtx->rbp;
1619	if (fExtrn & CPUMCTX_EXTRN_RSI)
1620	pRun->s.regs.regs.rsi = pCtx->rsi;
1621	if (fExtrn & CPUMCTX_EXTRN_RDI)
1622	pRun->s.regs.regs.rdi = pCtx->rdi;
1623	if (fExtrn & CPUMCTX_EXTRN_R8_R15)
1624	{
1625	pRun->s.regs.regs.r8 = pCtx->r8;
1626	pRun->s.regs.regs.r9 = pCtx->r9;
1627	pRun->s.regs.regs.r10 = pCtx->r10;
1628	pRun->s.regs.regs.r11 = pCtx->r11;
1629	pRun->s.regs.regs.r12 = pCtx->r12;
1630	pRun->s.regs.regs.r13 = pCtx->r13;
1631	pRun->s.regs.regs.r14 = pCtx->r14;
1632	pRun->s.regs.regs.r15 = pCtx->r15;
1633	}
1634	pRun->kvm_dirty_regs \|= KVM_SYNC_X86_REGS;
1635	}
1636
1637	/*
1638	* Stuff that goes into kvm_run::s.regs.sregs:
1639	*
1640	* The APIC base register updating is a little suboptimal... But at least
1641	* VBox always has the right base register value, so it's one directional.
1642	*/
1643	uint64_t const uApicBase = APICGetBaseMsrNoCheck(pVCpu);
1644	if ( (fExtrn & ( CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_CR_MASK
1645	\| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_APIC_TPR))
1646	\|\| uApicBase != pVCpu->nem.s.uKvmApicBase)
1647	{
1648	if ((pVCpu->nem.s.uKvmApicBase ^ uApicBase) & MSR_IA32_APICBASE_EN)
1649	Log(("NEM/%u: APICBASE_EN changed %#010RX64 -> %#010RX64\n", pVCpu->idCpu, pVCpu->nem.s.uKvmApicBase, uApicBase));
1650	pRun->s.regs.sregs.apic_base = uApicBase;
1651	pVCpu->nem.s.uKvmApicBase = uApicBase;
1652
1653	if (fExtrn & CPUMCTX_EXTRN_APIC_TPR)
1654	pRun->s.regs.sregs.cr8 = CPUMGetGuestCR8(pVCpu);
1655
1656	#define NEM_LNX_EXPORT_SEG(a_KvmSeg, a_CtxSeg) do { \
1657	(a_KvmSeg).base = (a_CtxSeg).u64Base; \
1658	(a_KvmSeg).limit = (a_CtxSeg).u32Limit; \
1659	(a_KvmSeg).selector = (a_CtxSeg).Sel; \
1660	(a_KvmSeg).type = (a_CtxSeg).Attr.n.u4Type; \
1661	(a_KvmSeg).s = (a_CtxSeg).Attr.n.u1DescType; \
1662	(a_KvmSeg).dpl = (a_CtxSeg).Attr.n.u2Dpl; \
1663	(a_KvmSeg).present = (a_CtxSeg).Attr.n.u1Present; \
1664	(a_KvmSeg).avl = (a_CtxSeg).Attr.n.u1Available; \
1665	(a_KvmSeg).l = (a_CtxSeg).Attr.n.u1Long; \
1666	(a_KvmSeg).db = (a_CtxSeg).Attr.n.u1DefBig; \
1667	(a_KvmSeg).g = (a_CtxSeg).Attr.n.u1Granularity; \
1668	(a_KvmSeg).unusable = (a_CtxSeg).Attr.n.u1Unusable; \
1669	(a_KvmSeg).padding = 0; \
1670	} while (0)
1671
1672	if (fExtrn & CPUMCTX_EXTRN_SREG_MASK)
1673	{
1674	if (fExtrn & CPUMCTX_EXTRN_ES)
1675	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.es, pCtx->es);
1676	if (fExtrn & CPUMCTX_EXTRN_CS)
1677	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.cs, pCtx->cs);
1678	if (fExtrn & CPUMCTX_EXTRN_SS)
1679	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ss, pCtx->ss);
1680	if (fExtrn & CPUMCTX_EXTRN_DS)
1681	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ds, pCtx->ds);
1682	if (fExtrn & CPUMCTX_EXTRN_FS)
1683	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.fs, pCtx->fs);
1684	if (fExtrn & CPUMCTX_EXTRN_GS)
1685	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.gs, pCtx->gs);
1686	}
1687	if (fExtrn & CPUMCTX_EXTRN_TABLE_MASK)
1688	{
1689	if (fExtrn & CPUMCTX_EXTRN_GDTR)
1690	{
1691	pRun->s.regs.sregs.gdt.base = pCtx->gdtr.pGdt;
1692	pRun->s.regs.sregs.gdt.limit = pCtx->gdtr.cbGdt;
1693	pRun->s.regs.sregs.gdt.padding[0] = 0;
1694	pRun->s.regs.sregs.gdt.padding[1] = 0;
1695	pRun->s.regs.sregs.gdt.padding[2] = 0;
1696	}
1697	if (fExtrn & CPUMCTX_EXTRN_IDTR)
1698	{
1699	pRun->s.regs.sregs.idt.base = pCtx->idtr.pIdt;
1700	pRun->s.regs.sregs.idt.limit = pCtx->idtr.cbIdt;
1701	pRun->s.regs.sregs.idt.padding[0] = 0;
1702	pRun->s.regs.sregs.idt.padding[1] = 0;
1703	pRun->s.regs.sregs.idt.padding[2] = 0;
1704	}
1705	if (fExtrn & CPUMCTX_EXTRN_LDTR)
1706	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ldt, pCtx->ldtr);
1707	if (fExtrn & CPUMCTX_EXTRN_TR)
1708	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.tr, pCtx->tr);
1709	}
1710	if (fExtrn & CPUMCTX_EXTRN_CR_MASK)
1711	{
1712	if (fExtrn & CPUMCTX_EXTRN_CR0)
1713	pRun->s.regs.sregs.cr0 = pCtx->cr0;
1714	if (fExtrn & CPUMCTX_EXTRN_CR2)
1715	pRun->s.regs.sregs.cr2 = pCtx->cr2;
1716	if (fExtrn & CPUMCTX_EXTRN_CR3)
1717	pRun->s.regs.sregs.cr3 = pCtx->cr3;
1718	if (fExtrn & CPUMCTX_EXTRN_CR4)
1719	pRun->s.regs.sregs.cr4 = pCtx->cr4;
1720	}
1721	if (fExtrn & CPUMCTX_EXTRN_EFER)
1722	pRun->s.regs.sregs.efer = pCtx->msrEFER;
1723
1724	RT_ZERO(pRun->s.regs.sregs.interrupt_bitmap); /* this is an alternative interrupt injection interface */
1725
1726	pRun->kvm_dirty_regs \|= KVM_SYNC_X86_SREGS;
1727	}
1728
1729	/*
1730	* Debug registers.
1731	*/
1732	if (fExtrn & CPUMCTX_EXTRN_DR_MASK)
1733	{
1734	struct kvm_debugregs DbgRegs = {{0}};
1735
1736	if ((fExtrn & CPUMCTX_EXTRN_DR_MASK) != CPUMCTX_EXTRN_DR_MASK)
1737	{
1738	/* Partial debug state, we must get DbgRegs first so we can merge: */
1739	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
1740	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1741	}
1742
1743	if (fExtrn & CPUMCTX_EXTRN_DR0_DR3)
1744	{
1745	DbgRegs.db[0] = pCtx->dr[0];
1746	DbgRegs.db[1] = pCtx->dr[1];
1747	DbgRegs.db[2] = pCtx->dr[2];
1748	DbgRegs.db[3] = pCtx->dr[3];
1749	}
1750	if (fExtrn & CPUMCTX_EXTRN_DR6)
1751	DbgRegs.dr6 = pCtx->dr[6];
1752	if (fExtrn & CPUMCTX_EXTRN_DR7)
1753	DbgRegs.dr7 = pCtx->dr[7];
1754
1755	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_DEBUGREGS, &DbgRegs);
1756	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1757	}
1758
1759	/*
1760	* FPU, SSE, AVX, ++.
1761	*/
1762	if (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx))
1763	{
1764	if (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1765	{
1766	/** @todo could IEM just grab state partial control in some situations? */
1767	Assert( (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1768	== (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE)); /* no partial states */
1769
1770	AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
1771	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XSAVE, &pCtx->XState);
1772	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1773	}
1774
1775	if (fExtrn & CPUMCTX_EXTRN_XCRx)
1776	{
1777	struct kvm_xcrs Xcrs =
1778	{ /.nr_xcrs = / 2,
1779	/.flags = / 0,
1780	/.xcrs= / {
1781	{ /.xcr =/ 0, /.reserved=/ 0, /.value=/ pCtx->aXcr[0] },
1782	{ /.xcr =/ 1, /.reserved=/ 0, /.value=/ pCtx->aXcr[1] },
1783	}
1784	};
1785
1786	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XCRS, &Xcrs);
1787	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1788	}
1789	}
1790
1791	/*
1792	* MSRs.
1793	*/
1794	if (fExtrn & ( CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_SYSENTER_MSRS
1795	\| CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
1796	{
1797	union
1798	{
1799	struct kvm_msrs Core;
1800	uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
1801	} uBuf;
1802	uint32_t iMsr = 0;
1803	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1804
1805	#define ADD_MSR(a_Msr, a_uValue) do { \
1806	Assert(iMsr < 32); \
1807	uBuf.Core.entries[iMsr].index = (a_Msr); \
1808	uBuf.Core.entries[iMsr].reserved = 0; \
1809	uBuf.Core.entries[iMsr].data = (a_uValue); \
1810	iMsr += 1; \
1811	} while (0)
1812
1813	if (fExtrn & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1814	ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1815	if (fExtrn & CPUMCTX_EXTRN_SYSCALL_MSRS)
1816	{
1817	ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
1818	ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
1819	ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
1820	ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1821	}
1822	if (fExtrn & CPUMCTX_EXTRN_SYSENTER_MSRS)
1823	{
1824	ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
1825	ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
1826	ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
1827	}
1828	if (fExtrn & CPUMCTX_EXTRN_TSC_AUX)
1829	ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1830	if (fExtrn & CPUMCTX_EXTRN_OTHER_MSRS)
1831	{
1832	ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
1833	/** @todo What do we _have_ to add here?
1834	* We also have: Mttr, MiscEnable, FeatureControl. /
1835	}
1836
1837	uBuf.Core.pad = 0;
1838	uBuf.Core.nmsrs = iMsr;
1839	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_MSRS, &uBuf);
1840	AssertMsgReturn(rc == (int)iMsr,
1841	("rc=%d iMsr=%d (->%#x) errno=%d\n",
1842	rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
1843	VERR_NEM_IPE_3);
1844	}
1845
1846	/*
1847	* Interruptibility state.
1848	*
1849	* Note! This I/O control function sets most fields passed in, so when
1850	* raising an interrupt, NMI, SMI or exception, this must be done
1851	* by the code doing the rasing or we'll overwrite it here.
1852	*/
1853	if (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1854	{
1855	Assert( (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1856	== (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI));
1857
1858	struct kvm_vcpu_events KvmEvents = {0};
1859
1860	KvmEvents.flags = KVM_VCPUEVENT_VALID_SHADOW;
1861	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1862	{
1863	if (pRun->s.regs.regs.rip == EMGetInhibitInterruptsPC(pVCpu))
1864	KvmEvents.interrupt.shadow = KVM_X86_SHADOW_INT_MOV_SS \| KVM_X86_SHADOW_INT_STI;
1865	else
1866	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1867	}
1868
1869	/* No flag - this is updated unconditionally. */
1870	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
1871	KvmEvents.nmi.masked = 1;
1872
1873	if (TRPMHasTrap(pVCpu))
1874	{
1875	TRPMEVENT enmType = TRPM_32BIT_HACK;
1876	uint8_t bTrapNo = 0;
1877	TRPMQueryTrap(pVCpu, &bTrapNo, &enmType);
1878	Log(("nemHCLnxExportState: Pending trap: bTrapNo=%#x enmType=%d\n", bTrapNo, enmType));
1879	if ( enmType == TRPM_HARDWARE_INT
1880	\|\| enmType == TRPM_SOFTWARE_INT)
1881	{
1882	KvmEvents.interrupt.soft = enmType == TRPM_SOFTWARE_INT;
1883	KvmEvents.interrupt.nr = bTrapNo;
1884	KvmEvents.interrupt.injected = 1;
1885	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExportPendingInterrupt);
1886	TRPMResetTrap(pVCpu);
1887	}
1888	else
1889	AssertFailed();
1890	}
1891
1892	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
1893	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
1894	}
1895
1896	/*
1897	* KVM now owns all the state.
1898	*/
1899	pCtx->fExtrn = CPUMCTX_EXTRN_KEEPER_NEM \| CPUMCTX_EXTRN_ALL;
1900
1901	RT_NOREF(pVM);
1902	return VINF_SUCCESS;
1903	}
1904
1905
1906	/**
1907	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
1908	*
1909	* @returns VBox status code.
1910	* @param pVCpu The cross context CPU structure.
1911	* @param pcTicks Where to return the CPU tick count.
1912	* @param puAux Where to return the TSC_AUX register value.
1913	*/
1914	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
1915	{
1916	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
1917	// KVM_GET_CLOCK?
1918	RT_NOREF(pVCpu, pcTicks, puAux);
1919	return VINF_SUCCESS;
1920	}
1921
1922
1923	/**
1924	* Resumes CPU clock (TSC) on all virtual CPUs.
1925	*
1926	* This is called by TM when the VM is started, restored, resumed or similar.
1927	*
1928	* @returns VBox status code.
1929	* @param pVM The cross context VM structure.
1930	* @param pVCpu The cross context CPU structure of the calling EMT.
1931	* @param uPausedTscValue The TSC value at the time of pausing.
1932	*/
1933	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
1934	{
1935	// KVM_SET_CLOCK?
1936	RT_NOREF(pVM, pVCpu, uPausedTscValue);
1937	return VINF_SUCCESS;
1938	}
1939
1940
1941	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
1942	{
1943	RT_NOREF(pVM);
1944	return NEM_FEAT_F_NESTED_PAGING
1945	\| NEM_FEAT_F_FULL_GST_EXEC
1946	\| NEM_FEAT_F_XSAVE_XRSTOR;
1947	}
1948
1949
1950
1951	/*********************************************************************************************************************************
1952	* Execution *
1953	*********************************************************************************************************************************/
1954
1955
1956	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
1957	{
1958	/*
1959	* Only execute when the A20 gate is enabled as I cannot immediately
1960	* spot any A20 support in KVM.
1961	*/
1962	RT_NOREF(pVM);
1963	Assert(VM_IS_NEM_ENABLED(pVM));
1964	return PGMPhysIsA20Enabled(pVCpu);
1965	}
1966
1967
1968	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
1969	{
1970	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
1971	return false;
1972	}
1973
1974
1975	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
1976	{
1977	int rc = RTThreadPoke(pVCpu->hThread);
1978	LogFlow(("nemR3NativeNotifyFF: #%u -> %Rrc\n", pVCpu->idCpu, rc));
1979	AssertRC(rc);
1980	RT_NOREF(pVM, fFlags);
1981	}
1982
1983
1984	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
1985	{
1986	RT_NOREF(pVM, fUseDebugLoop);
1987	return false;
1988	}
1989
1990
1991	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
1992	{
1993	RT_NOREF(pVM, pVCpu, fUseDebugLoop);
1994	return false;
1995	}
1996
1997
1998	/**
1999	* Deals with pending interrupt FFs prior to executing guest code.
2000	*/
2001	static VBOXSTRICTRC nemHCLnxHandleInterruptFF(PVM pVM, PVMCPU pVCpu, struct kvm_run *pRun)
2002	{
2003	RT_NOREF_PV(pVM);
2004
2005	/*
2006	* Do not doing anything if TRPM has something pending already as we can
2007	* only inject one event per KVM_RUN call. This can only happend if we
2008	* can directly from the loop in EM, so the inhibit bits must be internal.
2009	*/
2010	if (!TRPMHasTrap(pVCpu))
2011	{ /* semi likely */ }
2012	else
2013	{
2014	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI)));
2015	Log8(("nemHCLnxHandleInterruptFF: TRPM has an pending event already\n"));
2016	return VINF_SUCCESS;
2017	}
2018
2019	/*
2020	* First update APIC. We ASSUME this won't need TPR/CR8.
2021	*/
2022	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
2023	{
2024	APICUpdatePendingInterrupts(pVCpu);
2025	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
2026	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2027	return VINF_SUCCESS;
2028	}
2029
2030	/*
2031	* We don't currently implement SMIs.
2032	*/
2033	AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
2034
2035	/*
2036	* In KVM the CPUMCTX_EXTRN_INHIBIT_INT and CPUMCTX_EXTRN_INHIBIT_NMI states
2037	* are tied together with interrupt and NMI delivery, so we must get and
2038	* synchronize these all in one go and set both CPUMCTX_EXTRN_INHIBIT_XXX flags.
2039	* If we don't we may lose the interrupt/NMI we marked pending here when the
2040	* state is exported again before execution.
2041	*/
2042	struct kvm_vcpu_events KvmEvents = {0};
2043	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
2044	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2045
2046	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
2047	pRun->s.regs.regs.rip = pVCpu->cpum.GstCtx.rip;
2048
2049	KvmEvents.flags \|= KVM_VCPUEVENT_VALID_SHADOW;
2050	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_INT))
2051	{
2052	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2053	KvmEvents.interrupt.shadow = 0;
2054	else if (EMGetInhibitInterruptsPC(pVCpu) == pRun->s.regs.regs.rip)
2055	KvmEvents.interrupt.shadow = KVM_X86_SHADOW_INT_MOV_SS \| KVM_X86_SHADOW_INT_STI;
2056	else
2057	{
2058	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2059	KvmEvents.interrupt.shadow = 0;
2060	}
2061	}
2062	else if (KvmEvents.interrupt.shadow)
2063	EMSetInhibitInterruptsPC(pVCpu, pRun->s.regs.regs.rip);
2064	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2065	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2066
2067	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_NMI))
2068	KvmEvents.nmi.masked = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS) ? 1 : 0;
2069	else if (KvmEvents.nmi.masked)
2070	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
2071	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
2072	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
2073
2074	/* KVM will own the INT + NMI inhibit state soon: */
2075	pVCpu->cpum.GstCtx.fExtrn = (pVCpu->cpum.GstCtx.fExtrn & ~CPUMCTX_EXTRN_KEEPER_MASK)
2076	\| CPUMCTX_EXTRN_KEEPER_NEM \| CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI;
2077
2078	/*
2079	* NMI? Try deliver it first.
2080	*/
2081	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
2082	{
2083	#if 0
2084	int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_NMI, 0UL);
2085	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2086	#else
2087	KvmEvents.flags \|= KVM_VCPUEVENT_VALID_NMI_PENDING;
2088	KvmEvents.nmi.pending = 1;
2089	#endif
2090	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
2091	Log8(("Queuing NMI on %u\n", pVCpu->idCpu));
2092	}
2093
2094	/*
2095	* APIC or PIC interrupt?
2096	*/
2097	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
2098	{
2099	if (pRun->s.regs.regs.rflags & X86_EFL_IF)
2100	{
2101	if (KvmEvents.interrupt.shadow == 0)
2102	{
2103	/*
2104	* If CR8 is in KVM, update the VBox copy so PDMGetInterrupt will
2105	* work correctly.
2106	*/
2107	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_APIC_TPR)
2108	APICSetTpr(pVCpu, (uint8_t)pRun->cr8 << 4);
2109
2110	uint8_t bInterrupt;
2111	int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
2112	if (RT_SUCCESS(rc))
2113	{
2114	Assert(KvmEvents.interrupt.injected == false);
2115	#if 0
2116	int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_INTERRUPT, (unsigned long)bInterrupt);
2117	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2118	#else
2119	KvmEvents.interrupt.nr = bInterrupt;
2120	KvmEvents.interrupt.soft = false;
2121	KvmEvents.interrupt.injected = true;
2122	#endif
2123	Log8(("Queuing interrupt %#x on %u: %04x:%08RX64 efl=%#x\n", bInterrupt, pVCpu->idCpu,
2124	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags));
2125	}
2126	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR) /** @todo this isn't extremely efficient if we get a lot of exits... */
2127	Log8(("VERR_APIC_INTR_MASKED_BY_TPR\n")); /* We'll get a TRP exit - no interrupt window needed. */
2128	else
2129	Log8(("PDMGetInterrupt failed -> %Rrc\n", rc));
2130	}
2131	else
2132	{
2133	pRun->request_interrupt_window = 1;
2134	Log8(("Interrupt window pending on %u (#2)\n", pVCpu->idCpu));
2135	}
2136	}
2137	else
2138	{
2139	pRun->request_interrupt_window = 1;
2140	Log8(("Interrupt window pending on %u (#1)\n", pVCpu->idCpu));
2141	}
2142	}
2143
2144	/*
2145	* Now, update the state.
2146	*/
2147	/** @todo skip when possible... */
2148	rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
2149	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2150
2151	return VINF_SUCCESS;
2152	}
2153
2154
2155	/**
2156	* Handles KVM_EXIT_INTERNAL_ERROR.
2157	*/
2158	static VBOXSTRICTRC nemR3LnxHandleInternalError(PVMCPU pVCpu, struct kvm_run *pRun)
2159	{
2160	Log(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%d) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror,
2161	pRun->internal.suberror, pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
2162
2163	/*
2164	* Deal with each suberror, returning if we don't want IEM to handle it.
2165	*/
2166	switch (pRun->internal.suberror)
2167	{
2168	case KVM_INTERNAL_ERROR_EMULATION:
2169	{
2170	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_EMULATION),
2171	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2172	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorEmulation);
2173	break;
2174	}
2175
2176	case KVM_INTERNAL_ERROR_SIMUL_EX:
2177	case KVM_INTERNAL_ERROR_DELIVERY_EV:
2178	case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON:
2179	default:
2180	{
2181	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_FATAL),
2182	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2183	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorFatal);
2184	const char *pszName;
2185	switch (pRun->internal.suberror)
2186	{
2187	case KVM_INTERNAL_ERROR_EMULATION: pszName = "KVM_INTERNAL_ERROR_EMULATION"; break;
2188	case KVM_INTERNAL_ERROR_SIMUL_EX: pszName = "KVM_INTERNAL_ERROR_SIMUL_EX"; break;
2189	case KVM_INTERNAL_ERROR_DELIVERY_EV: pszName = "KVM_INTERNAL_ERROR_DELIVERY_EV"; break;
2190	case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON: pszName = "KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON"; break;
2191	default: pszName = "unknown"; break;
2192	}
2193	LogRel(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%s) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror, pszName,
2194	pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
2195	return VERR_NEM_IPE_0;
2196	}
2197	}
2198
2199	/*
2200	* Execute instruction in IEM and try get on with it.
2201	*/
2202	Log2(("nemR3LnxHandleInternalError: Executing instruction at %04x:%08RX64 in IEM\n",
2203	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip));
2204	VBOXSTRICTRC rcStrict = nemHCLnxImportState(pVCpu,
2205	IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_INHIBIT_INT
2206	\| CPUMCTX_EXTRN_INHIBIT_NMI,
2207	&pVCpu->cpum.GstCtx, pRun);
2208	if (RT_SUCCESS(rcStrict))
2209	rcStrict = IEMExecOne(pVCpu);
2210	return rcStrict;
2211	}
2212
2213
2214	/**
2215	* Handles KVM_EXIT_IO.
2216	*/
2217	static VBOXSTRICTRC nemHCLnxHandleExitIo(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
2218	{
2219	/*
2220	* Input validation.
2221	*/
2222	Assert(pRun->io.count > 0);
2223	Assert(pRun->io.size == 1 \|\| pRun->io.size == 2 \|\| pRun->io.size == 4);
2224	Assert(pRun->io.direction == KVM_EXIT_IO_IN \|\| pRun->io.direction == KVM_EXIT_IO_OUT);
2225	Assert(pRun->io.data_offset < pVM->nem.s.cbVCpuMmap);
2226	Assert(pRun->io.data_offset + pRun->io.size * pRun->io.count <= pVM->nem.s.cbVCpuMmap);
2227
2228	/*
2229	* We cannot easily act on the exit history here, because the I/O port
2230	* exit is stateful and the instruction will be completed in the next
2231	* KVM_RUN call. There seems no way to avoid this.
2232	*/
2233	EMHistoryAddExit(pVCpu,
2234	pRun->io.count == 1
2235	? ( pRun->io.direction == KVM_EXIT_IO_IN
2236	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ)
2237	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE))
2238	: ( pRun->io.direction == KVM_EXIT_IO_IN
2239	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)
2240	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)),
2241	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2242
2243	/*
2244	* Do the requested job.
2245	*/
2246	VBOXSTRICTRC rcStrict;
2247	RTPTRUNION uPtrData;
2248	uPtrData.pu8 = (uint8_t *)pRun + pRun->io.data_offset;
2249	if (pRun->io.count == 1)
2250	{
2251	if (pRun->io.direction == KVM_EXIT_IO_IN)
2252	{
2253	uint32_t uValue = 0;
2254	rcStrict = IOMIOPortRead(pVM, pVCpu, pRun->io.port, &uValue, pRun->io.size);
2255	Log4(("IOExit/%u: %04x:%08RX64: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2256	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2257	pRun->io.port, pRun->io.size, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2258	if (IOM_SUCCESS(rcStrict))
2259	{
2260	if (pRun->io.size == 4)
2261	*uPtrData.pu32 = uValue;
2262	else if (pRun->io.size == 2)
2263	*uPtrData.pu16 = (uint16_t)uValue;
2264	else
2265	*uPtrData.pu8 = (uint8_t)uValue;
2266	}
2267	}
2268	else
2269	{
2270	uint32_t const uValue = pRun->io.size == 4 ? *uPtrData.pu32
2271	: pRun->io.size == 2 ? *uPtrData.pu16
2272	: *uPtrData.pu8;
2273	rcStrict = IOMIOPortWrite(pVM, pVCpu, pRun->io.port, uValue, pRun->io.size);
2274	Log4(("IOExit/%u: %04x:%08RX64: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2275	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2276	pRun->io.port, uValue, pRun->io.size, VBOXSTRICTRC_VAL(rcStrict) ));
2277	}
2278	}
2279	else
2280	{
2281	uint32_t cTransfers = pRun->io.count;
2282	if (pRun->io.direction == KVM_EXIT_IO_IN)
2283	{
2284	rcStrict = IOMIOPortReadString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
2285	Log4(("IOExit/%u: %04x:%08RX64: REP INS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
2286	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2287	pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
2288	}
2289	else
2290	{
2291	rcStrict = IOMIOPortWriteString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
2292	Log4(("IOExit/%u: %04x:%08RX64: REP OUTS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
2293	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2294	pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
2295	}
2296	Assert(cTransfers == 0);
2297	}
2298	return rcStrict;
2299	}
2300
2301
2302	/**
2303	* Handles KVM_EXIT_MMIO.
2304	*/
2305	static VBOXSTRICTRC nemHCLnxHandleExitMmio(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
2306	{
2307	/*
2308	* Input validation.
2309	*/
2310	Assert(pRun->mmio.len <= sizeof(pRun->mmio.data));
2311	Assert(pRun->mmio.is_write <= 1);
2312
2313	/*
2314	* We cannot easily act on the exit history here, because the MMIO port
2315	* exit is stateful and the instruction will be completed in the next
2316	* KVM_RUN call. There seems no way to circumvent this.
2317	*/
2318	EMHistoryAddExit(pVCpu,
2319	pRun->mmio.is_write
2320	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2321	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2322	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2323
2324	/*
2325	* Do the requested job.
2326	*/
2327	VBOXSTRICTRC rcStrict;
2328	if (pRun->mmio.is_write)
2329	{
2330	rcStrict = PGMPhysWrite(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
2331	Log4(("MmioExit/%u: %04x:%08RX64: WRITE %#x LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
2332	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2333	pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
2334	}
2335	else
2336	{
2337	rcStrict = PGMPhysRead(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
2338	Log4(("MmioExit/%u: %04x:%08RX64: READ %#x LB %u -> %.*Rhxs rcStrict=%Rrc\n",
2339	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2340	pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
2341	}
2342	return rcStrict;
2343	}
2344
2345
2346	/**
2347	* Handles KVM_EXIT_RDMSR
2348	*/
2349	static VBOXSTRICTRC nemHCLnxHandleExitRdMsr(PVMCPUCC pVCpu, struct kvm_run *pRun)
2350	{
2351	/*
2352	* Input validation.
2353	*/
2354	Assert( pRun->msr.reason == KVM_MSR_EXIT_REASON_INVAL
2355	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_UNKNOWN
2356	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
2357
2358	/*
2359	* We cannot easily act on the exit history here, because the MSR exit is
2360	* stateful and the instruction will be completed in the next KVM_RUN call.
2361	* There seems no way to circumvent this.
2362	*/
2363	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2364	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2365
2366	/*
2367	* Do the requested job.
2368	*/
2369	uint64_t uValue = 0;
2370	VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pRun->msr.index, &uValue);
2371	pRun->msr.data = uValue;
2372	if (rcStrict != VERR_CPUM_RAISE_GP_0)
2373	{
2374	Log3(("MsrRead/%u: %04x:%08RX64: msr=%#010x (reason=%#x) -> %#RX64 rcStrict=%Rrc\n", pVCpu->idCpu,
2375	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.reason, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2376	pRun->msr.error = 0;
2377	}
2378	else
2379	{
2380	Log3(("MsrRead/%u: %04x:%08RX64: msr=%#010x (reason%#x)-> %#RX64 rcStrict=#GP!\n", pVCpu->idCpu,
2381	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.reason, uValue));
2382	pRun->msr.error = 1;
2383	rcStrict = VINF_SUCCESS;
2384	}
2385	return rcStrict;
2386	}
2387
2388
2389	/**
2390	* Handles KVM_EXIT_WRMSR
2391	*/
2392	static VBOXSTRICTRC nemHCLnxHandleExitWrMsr(PVMCPUCC pVCpu, struct kvm_run *pRun)
2393	{
2394	/*
2395	* Input validation.
2396	*/
2397	Assert( pRun->msr.reason == KVM_MSR_EXIT_REASON_INVAL
2398	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_UNKNOWN
2399	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
2400
2401	/*
2402	* We cannot easily act on the exit history here, because the MSR exit is
2403	* stateful and the instruction will be completed in the next KVM_RUN call.
2404	* There seems no way to circumvent this.
2405	*/
2406	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE),
2407	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2408
2409	/*
2410	* Do the requested job.
2411	*/
2412	VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pRun->msr.index, pRun->msr.data);
2413	if (rcStrict != VERR_CPUM_RAISE_GP_0)
2414	{
2415	Log3(("MsrWrite/%u: %04x:%08RX64: msr=%#010x := %#RX64 (reason=%#x) -> rcStrict=%Rrc\n", pVCpu->idCpu,
2416	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.data, pRun->msr.reason, VBOXSTRICTRC_VAL(rcStrict) ));
2417	pRun->msr.error = 0;
2418	}
2419	else
2420	{
2421	Log3(("MsrWrite/%u: %04x:%08RX64: msr=%#010x := %#RX64 (reason%#x)-> rcStrict=#GP!\n", pVCpu->idCpu,
2422	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.data, pRun->msr.reason));
2423	pRun->msr.error = 1;
2424	rcStrict = VINF_SUCCESS;
2425	}
2426	return rcStrict;
2427	}
2428
2429
2430
2431	static VBOXSTRICTRC nemHCLnxHandleExit(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run pRun, bool pfStatefulExit)
2432	{
2433	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTotal);
2434	switch (pRun->exit_reason)
2435	{
2436	case KVM_EXIT_EXCEPTION:
2437	AssertFailed();
2438	break;
2439
2440	case KVM_EXIT_IO:
2441	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIo);
2442	*pfStatefulExit = true;
2443	return nemHCLnxHandleExitIo(pVM, pVCpu, pRun);
2444
2445	case KVM_EXIT_MMIO:
2446	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMmio);
2447	*pfStatefulExit = true;
2448	return nemHCLnxHandleExitMmio(pVM, pVCpu, pRun);
2449
2450	case KVM_EXIT_IRQ_WINDOW_OPEN:
2451	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2452	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2453	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIrqWindowOpen);
2454	Log5(("IrqWinOpen/%u: %d\n", pVCpu->idCpu, pRun->request_interrupt_window));
2455	pRun->request_interrupt_window = 0;
2456	return VINF_SUCCESS;
2457
2458	case KVM_EXIT_SET_TPR:
2459	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitSetTpr);
2460	AssertFailed();
2461	break;
2462
2463	case KVM_EXIT_TPR_ACCESS:
2464	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTprAccess);
2465	AssertFailed();
2466	break;
2467
2468	case KVM_EXIT_X86_RDMSR:
2469	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitRdMsr);
2470	*pfStatefulExit = true;
2471	return nemHCLnxHandleExitRdMsr(pVCpu, pRun);
2472
2473	case KVM_EXIT_X86_WRMSR:
2474	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitWrMsr);
2475	*pfStatefulExit = true;
2476	return nemHCLnxHandleExitWrMsr(pVCpu, pRun);
2477
2478	case KVM_EXIT_HLT:
2479	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
2480	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2481	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
2482	Log5(("Halt/%u\n", pVCpu->idCpu));
2483	return VINF_EM_HALT;
2484
2485	case KVM_EXIT_INTR: /* EINTR */
2486	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERRUPTED),
2487	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2488	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIntr);
2489	Log5(("Intr/%u\n", pVCpu->idCpu));
2490	return VINF_SUCCESS;
2491
2492	case KVM_EXIT_HYPERCALL:
2493	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHypercall);
2494	AssertFailed();
2495	break;
2496
2497	case KVM_EXIT_DEBUG:
2498	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitDebug);
2499	AssertFailed();
2500	break;
2501
2502	case KVM_EXIT_SYSTEM_EVENT:
2503	AssertFailed();
2504	break;
2505	case KVM_EXIT_IOAPIC_EOI:
2506	AssertFailed();
2507	break;
2508	case KVM_EXIT_HYPERV:
2509	AssertFailed();
2510	break;
2511
2512	case KVM_EXIT_DIRTY_RING_FULL:
2513	AssertFailed();
2514	break;
2515	case KVM_EXIT_AP_RESET_HOLD:
2516	AssertFailed();
2517	break;
2518	case KVM_EXIT_X86_BUS_LOCK:
2519	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitBusLock);
2520	AssertFailed();
2521	break;
2522
2523
2524	case KVM_EXIT_SHUTDOWN:
2525	AssertFailed();
2526	break;
2527
2528	case KVM_EXIT_FAIL_ENTRY:
2529	LogRel(("NEM: KVM_EXIT_FAIL_ENTRY! hardware_entry_failure_reason=%#x cpu=%#x\n",
2530	pRun->fail_entry.hardware_entry_failure_reason, pRun->fail_entry.cpu));
2531	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_FAILED_ENTRY),
2532	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2533	return VERR_NEM_IPE_1;
2534
2535	case KVM_EXIT_INTERNAL_ERROR:
2536	/* we're counting sub-reasons inside the function. */
2537	return nemR3LnxHandleInternalError(pVCpu, pRun);
2538
2539	/*
2540	* Foreign and unknowns.
2541	*/
2542	case KVM_EXIT_NMI:
2543	AssertLogRelMsgFailedReturn(("KVM_EXIT_NMI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2544	case KVM_EXIT_EPR:
2545	AssertLogRelMsgFailedReturn(("KVM_EXIT_EPR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2546	case KVM_EXIT_WATCHDOG:
2547	AssertLogRelMsgFailedReturn(("KVM_EXIT_WATCHDOG on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2548	case KVM_EXIT_ARM_NISV:
2549	AssertLogRelMsgFailedReturn(("KVM_EXIT_ARM_NISV on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2550	case KVM_EXIT_S390_STSI:
2551	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_STSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2552	case KVM_EXIT_S390_TSCH:
2553	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_TSCH on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2554	case KVM_EXIT_OSI:
2555	AssertLogRelMsgFailedReturn(("KVM_EXIT_OSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2556	case KVM_EXIT_PAPR_HCALL:
2557	AssertLogRelMsgFailedReturn(("KVM_EXIT_PAPR_HCALL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2558	case KVM_EXIT_S390_UCONTROL:
2559	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_UCONTROL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2560	case KVM_EXIT_DCR:
2561	AssertLogRelMsgFailedReturn(("KVM_EXIT_DCR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2562	case KVM_EXIT_S390_SIEIC:
2563	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_SIEIC on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2564	case KVM_EXIT_S390_RESET:
2565	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_RESET on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2566	case KVM_EXIT_UNKNOWN:
2567	AssertLogRelMsgFailedReturn(("KVM_EXIT_UNKNOWN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2568	case KVM_EXIT_XEN:
2569	AssertLogRelMsgFailedReturn(("KVM_EXIT_XEN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2570	default:
2571	AssertLogRelMsgFailedReturn(("Unknown exit reason %u on VCpu #%u at %04x:%RX64!\n", pRun->exit_reason, pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2572	}
2573
2574	RT_NOREF(pVM, pVCpu, pRun);
2575	return VERR_NOT_IMPLEMENTED;
2576	}
2577
2578
2579	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2580	{
2581	/*
2582	* Try switch to NEM runloop state.
2583	*/
2584	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2585	{ /* likely */ }
2586	else
2587	{
2588	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2589	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
2590	return VINF_SUCCESS;
2591	}
2592
2593	/*
2594	* The run loop.
2595	*/
2596	struct kvm_run * const pRun = pVCpu->nem.s.pRun;
2597	const bool fSingleStepping = DBGFIsStepping(pVCpu);
2598	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2599	bool fStatefulExit = false; /* For MMIO and IO exits. */
2600	for (unsigned iLoop = 0;; iLoop++)
2601	{
2602	/*
2603	* Pending interrupts or such? Need to check and deal with this prior
2604	* to the state syncing.
2605	*/
2606	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_UPDATE_APIC \| VMCPU_FF_INTERRUPT_PIC
2607	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2608	{
2609	/* Try inject interrupt. */
2610	rcStrict = nemHCLnxHandleInterruptFF(pVM, pVCpu, pRun);
2611	if (rcStrict == VINF_SUCCESS)
2612	{ /* likely */ }
2613	else
2614	{
2615	LogFlow(("NEM/%u: breaking: nemHCLnxHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2616	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2617	break;
2618	}
2619	}
2620
2621	/*
2622	* Do not execute in KVM if the A20 isn't enabled.
2623	*/
2624	if (PGMPhysIsA20Enabled(pVCpu))
2625	{ /* likely */ }
2626	else
2627	{
2628	rcStrict = VINF_EM_RESCHEDULE_REM;
2629	LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
2630	break;
2631	}
2632
2633	/*
2634	* Ensure KVM has the whole state.
2635	*/
2636	if ((pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL)
2637	{
2638	int rc2 = nemHCLnxExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx, pRun);
2639	AssertRCReturn(rc2, rc2);
2640	}
2641
2642	/*
2643	* Poll timers and run for a bit.
2644	*
2645	* With the VID approach (ring-0 or ring-3) we can specify a timeout here,
2646	* so we take the time of the next timer event and uses that as a deadline.
2647	* The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
2648	*/
2649	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
2650	* the whole polling job when timers have changed... */
2651	uint64_t offDeltaIgnored;
2652	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
2653	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
2654	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2655	{
2656	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
2657	{
2658	LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
2659	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2660	!!(pRun->s.regs.regs.rflags & X86_EFL_IF), pRun->s.regs.regs.rflags,
2661	pRun->s.regs.sregs.ss.selector, pRun->s.regs.regs.rsp, pRun->s.regs.sregs.cr0));
2662	TMNotifyStartOfExecution(pVM, pVCpu);
2663
2664	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
2665
2666	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
2667	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
2668
2669	#ifdef LOG_ENABLED
2670	if (LogIsFlowEnabled())
2671	{
2672	struct kvm_mp_state MpState = {UINT32_MAX};
2673	ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MP_STATE, &MpState);
2674	LogFlow(("NEM/%u: Exit @ %04x:%08RX64 IF=%d EFL=%#RX64 CR8=%#x Reason=%#x IrqReady=%d Flags=%#x %#lx\n", pVCpu->idCpu,
2675	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->if_flag,
2676	pRun->s.regs.regs.rflags, pRun->s.regs.sregs.cr8, pRun->exit_reason,
2677	pRun->ready_for_interrupt_injection, pRun->flags, MpState.mp_state));
2678	}
2679	#endif
2680	fStatefulExit = false;
2681	if (RT_LIKELY(rcLnx == 0 \|\| errno == EINTR))
2682	{
2683	/*
2684	* Deal with the exit.
2685	*/
2686	rcStrict = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
2687	if (rcStrict == VINF_SUCCESS)
2688	{ /* hopefully likely */ }
2689	else
2690	{
2691	LogFlow(("NEM/%u: breaking: nemHCLnxHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2692	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2693	break;
2694	}
2695	}
2696	else
2697	{
2698	int rc2 = RTErrConvertFromErrno(errno);
2699	AssertLogRelMsgFailedReturn(("KVM_RUN failed: rcLnx=%d errno=%u rc=%Rrc\n", rcLnx, errno, rc2), rc2);
2700	}
2701
2702	/*
2703	* If no relevant FFs are pending, loop.
2704	*/
2705	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2706	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2707	{ /* likely */ }
2708	else
2709	{
2710
2711	/** @todo Try handle pending flags, not just return to EM loops. Take care
2712	* not to set important RCs here unless we've handled an exit. */
2713	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
2714	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
2715	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
2716	break;
2717	}
2718	}
2719	else
2720	{
2721	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
2722	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
2723	break;
2724	}
2725	}
2726	else
2727	{
2728	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
2729	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
2730	break;
2731	}
2732	} /* the run loop */
2733
2734
2735	/*
2736	* If the last exit was stateful, commit the state we provided before
2737	* returning to the EM loop so we have a consistent state and can safely
2738	* be rescheduled and whatnot. This may require us to make multiple runs
2739	* for larger MMIO and I/O operations. Sigh^3.
2740	*
2741	* Note! There is no 'ing way to reset the kernel side completion callback
2742	* for these stateful i/o exits. Very annoying interface.
2743	*/
2744	/** @todo check how this works with string I/O and string MMIO. */
2745	if (fStatefulExit && RT_SUCCESS(rcStrict))
2746	{
2747	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn);
2748	uint32_t const uOrgExit = pRun->exit_reason;
2749	for (uint32_t i = 0; ; i++)
2750	{
2751	pRun->immediate_exit = 1;
2752	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
2753	Log(("NEM/%u: Flushed stateful exit -> %d/%d exit_reason=%d\n", pVCpu->idCpu, rcLnx, errno, pRun->exit_reason));
2754	if (rcLnx == -1 && errno == EINTR)
2755	{
2756	switch (i)
2757	{
2758	case 0: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn1Loop); break;
2759	case 1: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn2Loops); break;
2760	case 2: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn3Loops); break;
2761	default: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn4PlusLoops); break;
2762	}
2763	break;
2764	}
2765	AssertLogRelMsgBreakStmt(rcLnx == 0 && pRun->exit_reason == uOrgExit,
2766	("rcLnx=%d errno=%d exit_reason=%d uOrgExit=%d\n", rcLnx, errno, pRun->exit_reason, uOrgExit),
2767	rcStrict = VERR_NEM_IPE_6);
2768	VBOXSTRICTRC rcStrict2 = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
2769	if (rcStrict2 == VINF_SUCCESS \|\| rcStrict2 == rcStrict)
2770	{ /* likely */ }
2771	else if (RT_FAILURE(rcStrict2))
2772	{
2773	rcStrict = rcStrict2;
2774	break;
2775	}
2776	else
2777	{
2778	AssertLogRelMsgBreakStmt(rcStrict == VINF_SUCCESS,
2779	("rcStrict=%Rrc rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2)),
2780	rcStrict = VERR_NEM_IPE_7);
2781	rcStrict = rcStrict2;
2782	}
2783	}
2784	pRun->immediate_exit = 0;
2785	}
2786
2787	/*
2788	* If the CPU is running, make sure to stop it before we try sync back the
2789	* state and return to EM. We don't sync back the whole state if we can help it.
2790	*/
2791	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
2792	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2793
2794	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
2795	{
2796	/* Try anticipate what we might need. */
2797	uint64_t fImport = CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI /* Required for processing APIC,PIC,NMI & SMI FFs. */
2798	\| IEM_CPUMCTX_EXTRN_MUST_MASK /?/;
2799	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
2800	\|\| RT_FAILURE(rcStrict))
2801	fImport = CPUMCTX_EXTRN_ALL;
2802	# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
2803	else if ( rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
2804	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
2805	fImport = CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS;
2806	else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
2807	fImport = CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS;
2808	# endif
2809	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
2810	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2811	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
2812
2813	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
2814	{
2815	int rc2 = nemHCLnxImportState(pVCpu, fImport, &pVCpu->cpum.GstCtx, pRun);
2816	if (RT_SUCCESS(rc2))
2817	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
2818	else if (RT_SUCCESS(rcStrict))
2819	rcStrict = rc2;
2820	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
2821	pVCpu->cpum.GstCtx.fExtrn = 0;
2822	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
2823	}
2824	else
2825	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2826	}
2827	else
2828	{
2829	pVCpu->cpum.GstCtx.fExtrn = 0;
2830	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2831	}
2832
2833	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
2834	pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
2835	return rcStrict;
2836	}
2837
2838
2839	/** @page pg_nem_linux NEM/linux - Native Execution Manager, Linux.
2840	*
2841	* This is using KVM.
2842	*
2843	*/
2844

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-linux.cpp@ 95843

Download in other formats: