NEMR3Native-linux.cpp@ 92556

Last change on this file since 92556 was 92555, checked in by vboxsync, 3 years ago
VMM/NEM-linux: Fixed the flushing of stateful exits for larger MMIO operations. bugref:9044
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 109.7 KB

Line
1	/* $Id: NEMR3Native-linux.cpp 92555 2021-11-22 18:51:43Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 Linux backend.
4	*/
5
6	/*
7	* Copyright (C) 2021 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_NEM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <VBox/vmm/nem.h>
25	#include <VBox/vmm/iem.h>
26	#include <VBox/vmm/em.h>
27	#include <VBox/vmm/apic.h>
28	#include <VBox/vmm/pdm.h>
29	#include <VBox/vmm/trpm.h>
30	#include "NEMInternal.h"
31	#include <VBox/vmm/vmcc.h>
32
33	#include <iprt/alloca.h>
34	#include <iprt/string.h>
35	#include <iprt/system.h>
36
37	#include <errno.h>
38	#include <unistd.h>
39	#include <sys/ioctl.h>
40	#include <sys/fcntl.h>
41	#include <sys/mman.h>
42	#include <linux/kvm.h>
43
44	/*
45	* Supply stuff missing from the kvm.h on the build box.
46	*/
47	#ifndef KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON /* since 5.4 */
48	# define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4
49	#endif
50
51
52
53	/**
54	* Worker for nemR3NativeInit that gets the hypervisor capabilities.
55	*
56	* @returns VBox status code.
57	* @param pVM The cross context VM structure.
58	* @param pErrInfo Where to always return error info.
59	*/
60	static int nemR3LnxInitCheckCapabilities(PVM pVM, PRTERRINFO pErrInfo)
61	{
62	AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
63
64	/*
65	* Capabilities.
66	*/
67	static const struct
68	{
69	const char *pszName;
70	int iCap;
71	uint32_t offNem : 24;
72	uint32_t cbNem : 3;
73	uint32_t fReqNonZero : 1;
74	uint32_t uReserved : 4;
75	} s_aCaps[] =
76	{
77	#define CAP_ENTRY__L(a_Define) { #a_Define, a_Define, UINT32_C(0x00ffffff), 0, 0, 0 }
78	#define CAP_ENTRY__S(a_Define, a_Member) { #a_Define, a_Define, RT_UOFFSETOF(NEM, a_Member), RT_SIZEOFMEMB(NEM, a_Member), 0, 0 }
79	#define CAP_ENTRY_MS(a_Define, a_Member) { #a_Define, a_Define, RT_UOFFSETOF(NEM, a_Member), RT_SIZEOFMEMB(NEM, a_Member), 1, 0 }
80	#define CAP_ENTRY__U(a_Number) { "KVM_CAP_" #a_Number, a_Number, UINT32_C(0x00ffffff), 0, 0, 0 }
81	#define CAP_ENTRY_ML(a_Number) { "KVM_CAP_" #a_Number, a_Number, UINT32_C(0x00ffffff), 0, 1, 0 }
82
83	CAP_ENTRY__L(KVM_CAP_IRQCHIP), /* 0 */
84	CAP_ENTRY_ML(KVM_CAP_HLT),
85	CAP_ENTRY__L(KVM_CAP_MMU_SHADOW_CACHE_CONTROL),
86	CAP_ENTRY_ML(KVM_CAP_USER_MEMORY),
87	CAP_ENTRY__L(KVM_CAP_SET_TSS_ADDR),
88	CAP_ENTRY__U(5),
89	CAP_ENTRY__L(KVM_CAP_VAPIC),
90	CAP_ENTRY__L(KVM_CAP_EXT_CPUID),
91	CAP_ENTRY__L(KVM_CAP_CLOCKSOURCE),
92	CAP_ENTRY__L(KVM_CAP_NR_VCPUS),
93	CAP_ENTRY_MS(KVM_CAP_NR_MEMSLOTS, cMaxMemSlots), /* 10 */
94	CAP_ENTRY__L(KVM_CAP_PIT),
95	CAP_ENTRY__L(KVM_CAP_NOP_IO_DELAY),
96	CAP_ENTRY__L(KVM_CAP_PV_MMU),
97	CAP_ENTRY__L(KVM_CAP_MP_STATE),
98	CAP_ENTRY__L(KVM_CAP_COALESCED_MMIO),
99	CAP_ENTRY__L(KVM_CAP_SYNC_MMU),
100	CAP_ENTRY__U(17),
101	CAP_ENTRY__L(KVM_CAP_IOMMU),
102	CAP_ENTRY__U(19), /* Buggy KVM_CAP_JOIN_MEMORY_REGIONS? */
103	CAP_ENTRY__U(20), /* Mon-working KVM_CAP_DESTROY_MEMORY_REGION? */
104	CAP_ENTRY__L(KVM_CAP_DESTROY_MEMORY_REGION_WORKS), /* 21 */
105	CAP_ENTRY__L(KVM_CAP_USER_NMI),
106	#ifdef __KVM_HAVE_GUEST_DEBUG
107	CAP_ENTRY__L(KVM_CAP_SET_GUEST_DEBUG),
108	#endif
109	#ifdef __KVM_HAVE_PIT
110	CAP_ENTRY__L(KVM_CAP_REINJECT_CONTROL),
111	#endif
112	CAP_ENTRY__L(KVM_CAP_IRQ_ROUTING),
113	CAP_ENTRY__L(KVM_CAP_IRQ_INJECT_STATUS),
114	CAP_ENTRY__U(27),
115	CAP_ENTRY__U(28),
116	CAP_ENTRY__L(KVM_CAP_ASSIGN_DEV_IRQ),
117	CAP_ENTRY__L(KVM_CAP_JOIN_MEMORY_REGIONS_WORKS), /* 30 */
118	#ifdef __KVM_HAVE_MCE
119	CAP_ENTRY__L(KVM_CAP_MCE),
120	#endif
121	CAP_ENTRY__L(KVM_CAP_IRQFD),
122	#ifdef __KVM_HAVE_PIT
123	CAP_ENTRY__L(KVM_CAP_PIT2),
124	#endif
125	CAP_ENTRY__L(KVM_CAP_SET_BOOT_CPU_ID),
126	#ifdef __KVM_HAVE_PIT_STATE2
127	CAP_ENTRY__L(KVM_CAP_PIT_STATE2),
128	#endif
129	CAP_ENTRY__L(KVM_CAP_IOEVENTFD),
130	CAP_ENTRY__L(KVM_CAP_SET_IDENTITY_MAP_ADDR),
131	#ifdef __KVM_HAVE_XEN_HVM
132	CAP_ENTRY__L(KVM_CAP_XEN_HVM),
133	#endif
134	CAP_ENTRY_ML(KVM_CAP_ADJUST_CLOCK),
135	CAP_ENTRY__L(KVM_CAP_INTERNAL_ERROR_DATA), /* 40 */
136	#ifdef __KVM_HAVE_VCPU_EVENTS
137	CAP_ENTRY_ML(KVM_CAP_VCPU_EVENTS),
138	#else
139	CAP_ENTRY_MU(41),
140	#endif
141	CAP_ENTRY__L(KVM_CAP_S390_PSW),
142	CAP_ENTRY__L(KVM_CAP_PPC_SEGSTATE),
143	CAP_ENTRY__L(KVM_CAP_HYPERV),
144	CAP_ENTRY__L(KVM_CAP_HYPERV_VAPIC),
145	CAP_ENTRY__L(KVM_CAP_HYPERV_SPIN),
146	CAP_ENTRY__L(KVM_CAP_PCI_SEGMENT),
147	CAP_ENTRY__L(KVM_CAP_PPC_PAIRED_SINGLES),
148	CAP_ENTRY__L(KVM_CAP_INTR_SHADOW),
149	#ifdef __KVM_HAVE_DEBUGREGS
150	CAP_ENTRY__L(KVM_CAP_DEBUGREGS), /* 50 */
151	#endif
152	CAP_ENTRY__S(KVM_CAP_X86_ROBUST_SINGLESTEP, fRobustSingleStep),
153	CAP_ENTRY__L(KVM_CAP_PPC_OSI),
154	CAP_ENTRY__L(KVM_CAP_PPC_UNSET_IRQ),
155	CAP_ENTRY__L(KVM_CAP_ENABLE_CAP),
156	#ifdef __KVM_HAVE_XSAVE
157	CAP_ENTRY_ML(KVM_CAP_XSAVE),
158	#else
159	CAP_ENTRY_MU(55),
160	#endif
161	#ifdef __KVM_HAVE_XCRS
162	CAP_ENTRY_ML(KVM_CAP_XCRS),
163	#else
164	CAP_ENTRY_MU(56),
165	#endif
166	CAP_ENTRY__L(KVM_CAP_PPC_GET_PVINFO),
167	CAP_ENTRY__L(KVM_CAP_PPC_IRQ_LEVEL),
168	CAP_ENTRY__L(KVM_CAP_ASYNC_PF),
169	CAP_ENTRY__L(KVM_CAP_TSC_CONTROL), /* 60 */
170	CAP_ENTRY__L(KVM_CAP_GET_TSC_KHZ),
171	CAP_ENTRY__L(KVM_CAP_PPC_BOOKE_SREGS),
172	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE),
173	CAP_ENTRY__L(KVM_CAP_PPC_SMT),
174	CAP_ENTRY__L(KVM_CAP_PPC_RMA),
175	CAP_ENTRY__L(KVM_CAP_MAX_VCPUS),
176	CAP_ENTRY__L(KVM_CAP_PPC_HIOR),
177	CAP_ENTRY__L(KVM_CAP_PPC_PAPR),
178	CAP_ENTRY__L(KVM_CAP_SW_TLB),
179	CAP_ENTRY__L(KVM_CAP_ONE_REG), /* 70 */
180	CAP_ENTRY__L(KVM_CAP_S390_GMAP),
181	CAP_ENTRY__L(KVM_CAP_TSC_DEADLINE_TIMER),
182	CAP_ENTRY__L(KVM_CAP_S390_UCONTROL),
183	CAP_ENTRY__L(KVM_CAP_SYNC_REGS),
184	CAP_ENTRY__L(KVM_CAP_PCI_2_3),
185	CAP_ENTRY__L(KVM_CAP_KVMCLOCK_CTRL),
186	CAP_ENTRY__L(KVM_CAP_SIGNAL_MSI),
187	CAP_ENTRY__L(KVM_CAP_PPC_GET_SMMU_INFO),
188	CAP_ENTRY__L(KVM_CAP_S390_COW),
189	CAP_ENTRY__L(KVM_CAP_PPC_ALLOC_HTAB), /* 80 */
190	CAP_ENTRY__L(KVM_CAP_READONLY_MEM),
191	CAP_ENTRY__L(KVM_CAP_IRQFD_RESAMPLE),
192	CAP_ENTRY__L(KVM_CAP_PPC_BOOKE_WATCHDOG),
193	CAP_ENTRY__L(KVM_CAP_PPC_HTAB_FD),
194	CAP_ENTRY__L(KVM_CAP_S390_CSS_SUPPORT),
195	CAP_ENTRY__L(KVM_CAP_PPC_EPR),
196	CAP_ENTRY__L(KVM_CAP_ARM_PSCI),
197	CAP_ENTRY__L(KVM_CAP_ARM_SET_DEVICE_ADDR),
198	CAP_ENTRY__L(KVM_CAP_DEVICE_CTRL),
199	CAP_ENTRY__L(KVM_CAP_IRQ_MPIC), /* 90 */
200	CAP_ENTRY__L(KVM_CAP_PPC_RTAS),
201	CAP_ENTRY__L(KVM_CAP_IRQ_XICS),
202	CAP_ENTRY__L(KVM_CAP_ARM_EL1_32BIT),
203	CAP_ENTRY__L(KVM_CAP_SPAPR_MULTITCE),
204	CAP_ENTRY__L(KVM_CAP_EXT_EMUL_CPUID),
205	CAP_ENTRY__L(KVM_CAP_HYPERV_TIME),
206	CAP_ENTRY__L(KVM_CAP_IOAPIC_POLARITY_IGNORED),
207	CAP_ENTRY__L(KVM_CAP_ENABLE_CAP_VM),
208	CAP_ENTRY__L(KVM_CAP_S390_IRQCHIP),
209	CAP_ENTRY__L(KVM_CAP_IOEVENTFD_NO_LENGTH), /* 100 */
210	CAP_ENTRY__L(KVM_CAP_VM_ATTRIBUTES),
211	CAP_ENTRY__L(KVM_CAP_ARM_PSCI_0_2),
212	CAP_ENTRY__L(KVM_CAP_PPC_FIXUP_HCALL),
213	CAP_ENTRY__L(KVM_CAP_PPC_ENABLE_HCALL),
214	CAP_ENTRY__L(KVM_CAP_CHECK_EXTENSION_VM),
215	CAP_ENTRY__L(KVM_CAP_S390_USER_SIGP),
216	CAP_ENTRY__L(KVM_CAP_S390_VECTOR_REGISTERS),
217	CAP_ENTRY__L(KVM_CAP_S390_MEM_OP),
218	CAP_ENTRY__L(KVM_CAP_S390_USER_STSI),
219	CAP_ENTRY__L(KVM_CAP_S390_SKEYS), /* 110 */
220	CAP_ENTRY__L(KVM_CAP_MIPS_FPU),
221	CAP_ENTRY__L(KVM_CAP_MIPS_MSA),
222	CAP_ENTRY__L(KVM_CAP_S390_INJECT_IRQ),
223	CAP_ENTRY__L(KVM_CAP_S390_IRQ_STATE),
224	CAP_ENTRY__L(KVM_CAP_PPC_HWRNG),
225	CAP_ENTRY__L(KVM_CAP_DISABLE_QUIRKS),
226	CAP_ENTRY__L(KVM_CAP_X86_SMM),
227	CAP_ENTRY__L(KVM_CAP_MULTI_ADDRESS_SPACE),
228	CAP_ENTRY__L(KVM_CAP_GUEST_DEBUG_HW_BPS),
229	CAP_ENTRY__L(KVM_CAP_GUEST_DEBUG_HW_WPS), /* 120 */
230	CAP_ENTRY__L(KVM_CAP_SPLIT_IRQCHIP),
231	CAP_ENTRY__L(KVM_CAP_IOEVENTFD_ANY_LENGTH),
232	CAP_ENTRY__L(KVM_CAP_HYPERV_SYNIC),
233	CAP_ENTRY__L(KVM_CAP_S390_RI),
234	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE_64),
235	CAP_ENTRY__L(KVM_CAP_ARM_PMU_V3),
236	CAP_ENTRY__L(KVM_CAP_VCPU_ATTRIBUTES),
237	CAP_ENTRY__L(KVM_CAP_MAX_VCPU_ID),
238	CAP_ENTRY__L(KVM_CAP_X2APIC_API),
239	CAP_ENTRY__L(KVM_CAP_S390_USER_INSTR0), /* 130 */
240	CAP_ENTRY__L(KVM_CAP_MSI_DEVID),
241	CAP_ENTRY__L(KVM_CAP_PPC_HTM),
242	CAP_ENTRY__L(KVM_CAP_SPAPR_RESIZE_HPT),
243	CAP_ENTRY__L(KVM_CAP_PPC_MMU_RADIX),
244	CAP_ENTRY__L(KVM_CAP_PPC_MMU_HASH_V3),
245	CAP_ENTRY__L(KVM_CAP_IMMEDIATE_EXIT),
246	CAP_ENTRY__L(KVM_CAP_MIPS_VZ),
247	CAP_ENTRY__L(KVM_CAP_MIPS_TE),
248	CAP_ENTRY__L(KVM_CAP_MIPS_64BIT),
249	CAP_ENTRY__L(KVM_CAP_S390_GS), /* 140 */
250	CAP_ENTRY__L(KVM_CAP_S390_AIS),
251	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE_VFIO),
252	CAP_ENTRY__L(KVM_CAP_X86_DISABLE_EXITS),
253	CAP_ENTRY__L(KVM_CAP_ARM_USER_IRQ),
254	CAP_ENTRY__L(KVM_CAP_S390_CMMA_MIGRATION),
255	CAP_ENTRY__L(KVM_CAP_PPC_FWNMI),
256	CAP_ENTRY__L(KVM_CAP_PPC_SMT_POSSIBLE),
257	CAP_ENTRY__L(KVM_CAP_HYPERV_SYNIC2),
258	CAP_ENTRY__L(KVM_CAP_HYPERV_VP_INDEX),
259	CAP_ENTRY__L(KVM_CAP_S390_AIS_MIGRATION), /* 150 */
260	CAP_ENTRY__L(KVM_CAP_PPC_GET_CPU_CHAR),
261	CAP_ENTRY__L(KVM_CAP_S390_BPB),
262	CAP_ENTRY__L(KVM_CAP_GET_MSR_FEATURES),
263	CAP_ENTRY__L(KVM_CAP_HYPERV_EVENTFD),
264	CAP_ENTRY__L(KVM_CAP_HYPERV_TLBFLUSH),
265	CAP_ENTRY__L(KVM_CAP_S390_HPAGE_1M),
266	CAP_ENTRY__L(KVM_CAP_NESTED_STATE),
267	CAP_ENTRY__L(KVM_CAP_ARM_INJECT_SERROR_ESR),
268	CAP_ENTRY__L(KVM_CAP_MSR_PLATFORM_INFO),
269	CAP_ENTRY__L(KVM_CAP_PPC_NESTED_HV), /* 160 */
270	CAP_ENTRY__L(KVM_CAP_HYPERV_SEND_IPI),
271	CAP_ENTRY__L(KVM_CAP_COALESCED_PIO),
272	CAP_ENTRY__L(KVM_CAP_HYPERV_ENLIGHTENED_VMCS),
273	CAP_ENTRY__L(KVM_CAP_EXCEPTION_PAYLOAD),
274	CAP_ENTRY__L(KVM_CAP_ARM_VM_IPA_SIZE),
275	CAP_ENTRY__L(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT),
276	CAP_ENTRY__L(KVM_CAP_HYPERV_CPUID),
277	CAP_ENTRY__L(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2),
278	CAP_ENTRY__L(KVM_CAP_PPC_IRQ_XIVE),
279	CAP_ENTRY__L(KVM_CAP_ARM_SVE), /* 170 */
280	CAP_ENTRY__L(KVM_CAP_ARM_PTRAUTH_ADDRESS),
281	CAP_ENTRY__L(KVM_CAP_ARM_PTRAUTH_GENERIC),
282	CAP_ENTRY__L(KVM_CAP_PMU_EVENT_FILTER),
283	CAP_ENTRY__L(KVM_CAP_ARM_IRQ_LINE_LAYOUT_2),
284	CAP_ENTRY__L(KVM_CAP_HYPERV_DIRECT_TLBFLUSH),
285	CAP_ENTRY__L(KVM_CAP_PPC_GUEST_DEBUG_SSTEP),
286	CAP_ENTRY__L(KVM_CAP_ARM_NISV_TO_USER),
287	CAP_ENTRY__L(KVM_CAP_ARM_INJECT_EXT_DABT),
288	CAP_ENTRY__L(KVM_CAP_S390_VCPU_RESETS),
289	CAP_ENTRY__L(KVM_CAP_S390_PROTECTED), /* 180 */
290	CAP_ENTRY__L(KVM_CAP_PPC_SECURE_GUEST),
291	CAP_ENTRY__L(KVM_CAP_HALT_POLL),
292	CAP_ENTRY__L(KVM_CAP_ASYNC_PF_INT),
293	CAP_ENTRY__L(KVM_CAP_LAST_CPU),
294	CAP_ENTRY__L(KVM_CAP_SMALLER_MAXPHYADDR),
295	CAP_ENTRY__L(KVM_CAP_S390_DIAG318),
296	CAP_ENTRY__L(KVM_CAP_STEAL_TIME),
297	CAP_ENTRY_ML(KVM_CAP_X86_USER_SPACE_MSR), /* (since 5.10) */
298	CAP_ENTRY_ML(KVM_CAP_X86_MSR_FILTER),
299	CAP_ENTRY__L(KVM_CAP_ENFORCE_PV_FEATURE_CPUID), /* 190 */
300	CAP_ENTRY__L(KVM_CAP_SYS_HYPERV_CPUID),
301	CAP_ENTRY__L(KVM_CAP_DIRTY_LOG_RING),
302	CAP_ENTRY__L(KVM_CAP_X86_BUS_LOCK_EXIT),
303	CAP_ENTRY__L(KVM_CAP_PPC_DAWR1),
304	CAP_ENTRY__L(KVM_CAP_SET_GUEST_DEBUG2),
305	CAP_ENTRY__L(KVM_CAP_SGX_ATTRIBUTE),
306	CAP_ENTRY__L(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM),
307	CAP_ENTRY__L(KVM_CAP_PTP_KVM),
308	CAP_ENTRY__U(199),
309	CAP_ENTRY__U(200),
310	CAP_ENTRY__U(201),
311	CAP_ENTRY__U(202),
312	CAP_ENTRY__U(203),
313	CAP_ENTRY__U(204),
314	CAP_ENTRY__U(205),
315	CAP_ENTRY__U(206),
316	CAP_ENTRY__U(207),
317	CAP_ENTRY__U(208),
318	CAP_ENTRY__U(209),
319	CAP_ENTRY__U(210),
320	CAP_ENTRY__U(211),
321	CAP_ENTRY__U(212),
322	CAP_ENTRY__U(213),
323	CAP_ENTRY__U(214),
324	CAP_ENTRY__U(215),
325	CAP_ENTRY__U(216),
326	};
327
328	LogRel(("NEM: KVM capabilities (system):\n"));
329	int rcRet = VINF_SUCCESS;
330	for (unsigned i = 0; i < RT_ELEMENTS(s_aCaps); i++)
331	{
332	int rc = ioctl(pVM->nem.s.fdVm, KVM_CHECK_EXTENSION, s_aCaps[i].iCap);
333	if (rc >= 10)
334	LogRel(("NEM: %36s: %#x (%d)\n", s_aCaps[i].pszName, rc, rc));
335	else if (rc >= 0)
336	LogRel(("NEM: %36s: %d\n", s_aCaps[i].pszName, rc));
337	else
338	LogRel(("NEM: %s failed: %d/%d\n", s_aCaps[i].pszName, rc, errno));
339	switch (s_aCaps[i].cbNem)
340	{
341	case 0:
342	break;
343	case 1:
344	{
345	uint8_t puValue = (uint8_t )&pVM->nem.padding[s_aCaps[i].offNem];
346	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
347	*puValue = (uint8_t)rc;
348	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
349	break;
350	}
351	case 2:
352	{
353	uint16_t puValue = (uint16_t )&pVM->nem.padding[s_aCaps[i].offNem];
354	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
355	*puValue = (uint16_t)rc;
356	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
357	break;
358	}
359	case 4:
360	{
361	uint32_t puValue = (uint32_t )&pVM->nem.padding[s_aCaps[i].offNem];
362	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
363	*puValue = (uint32_t)rc;
364	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
365	break;
366	}
367	default:
368	rcRet = RTErrInfoSetF(pErrInfo, VERR_NEM_IPE_0, "s_aCaps[%u] is bad: cbNem=%#x - %s",
369	i, s_aCaps[i].pszName, s_aCaps[i].cbNem);
370	AssertFailedReturn(rcRet);
371	}
372
373	/*
374	* Is a require non-zero entry zero or failing?
375	*/
376	if (s_aCaps[i].fReqNonZero && rc <= 0)
377	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_MISSING_FEATURE,
378	"Required capability '%s' is missing!", s_aCaps[i].pszName);
379	}
380
381	/*
382	* Get per VCpu KVM_RUN MMAP area size.
383	*/
384	int rc = ioctl(pVM->nem.s.fdKvm, KVM_GET_VCPU_MMAP_SIZE, 0UL);
385	if ((unsigned)rc < _64M)
386	{
387	pVM->nem.s.cbVCpuMmap = (uint32_t)rc;
388	LogRel(("NEM: %36s: %#x (%d)\n", "KVM_GET_VCPU_MMAP_SIZE", rc, rc));
389	}
390	else if (rc < 0)
391	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_MISSING_FEATURE, "KVM_GET_VCPU_MMAP_SIZE failed: %d", errno);
392	else
393	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_INIT_FAILED, "Odd KVM_GET_VCPU_MMAP_SIZE value: %#x (%d)", rc, rc);
394
395	/*
396	* Init the slot ID bitmap.
397	*/
398	ASMBitSet(&pVM->nem.s.bmSlotIds[0], 0); /* don't use slot 0 */
399	if (pVM->nem.s.cMaxMemSlots < _32K)
400	ASMBitSetRange(&pVM->nem.s.bmSlotIds[0], pVM->nem.s.cMaxMemSlots, _32K);
401	ASMBitSet(&pVM->nem.s.bmSlotIds[0], _32K - 1); /* don't use the last slot */
402
403	return rcRet;
404	}
405
406
407	/**
408	* Does the early setup of a KVM VM.
409	*
410	* @returns VBox status code.
411	* @param pVM The cross context VM structure.
412	* @param pErrInfo Where to always return error info.
413	*/
414	static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo)
415	{
416	AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
417
418	/*
419	* Create the VCpus.
420	*/
421	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
422	{
423	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
424
425	/* Create it. */
426	pVCpu->nem.s.fdVCpu = ioctl(pVM->nem.s.fdVm, KVM_CREATE_VCPU, (unsigned long)idCpu);
427	if (pVCpu->nem.s.fdVCpu < 0)
428	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
429	"KVM_CREATE_VCPU failed for VCpu #%u: %d", idCpu, errno);
430
431	/* Map the KVM_RUN area. */
432	pVCpu->nem.s.pRun = (struct kvm_run *)mmap(NULL, pVM->nem.s.cbVCpuMmap, PROT_READ \| PROT_WRITE, MAP_SHARED,
433	pVCpu->nem.s.fdVCpu, 0 /offset/);
434	if ((void *)pVCpu->nem.s.pRun == MAP_FAILED)
435	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "mmap failed for VCpu #%u: %d", idCpu, errno);
436
437	/* We want all x86 registers and events on each exit. */
438	pVCpu->nem.s.pRun->kvm_valid_regs = KVM_SYNC_X86_REGS \| KVM_SYNC_X86_SREGS \| KVM_SYNC_X86_EVENTS;
439	}
440	return VINF_SUCCESS;
441	}
442
443
444	/** @callback_method_impl{FNVMMEMTRENDEZVOUS} */
445	static DECLCALLBACK(VBOXSTRICTRC) nemR3LnxFixThreadPoke(PVM pVM, PVMCPU pVCpu, void *pvUser)
446	{
447	RT_NOREF(pVM, pvUser);
448	int rc = RTThreadControlPokeSignal(pVCpu->hThread, true /fEnable/);
449	AssertLogRelRC(rc);
450	return VINF_SUCCESS;
451	}
452
453
454	/**
455	* Try initialize the native API.
456	*
457	* This may only do part of the job, more can be done in
458	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
459	*
460	* @returns VBox status code.
461	* @param pVM The cross context VM structure.
462	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
463	* the latter we'll fail if we cannot initialize.
464	* @param fForced Whether the HMForced flag is set and we should
465	* fail if we cannot initialize.
466	*/
467	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
468	{
469	RT_NOREF(pVM, fFallback, fForced);
470	/*
471	* Some state init.
472	*/
473	pVM->nem.s.fdKvm = -1;
474	pVM->nem.s.fdVm = -1;
475	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
476	{
477	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
478	pNemCpu->fdVCpu = -1;
479	}
480
481	/*
482	* Error state.
483	* The error message will be non-empty on failure and 'rc' will be set too.
484	*/
485	RTERRINFOSTATIC ErrInfo;
486	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
487
488	/*
489	* Open kvm subsystem so we can issue system ioctls.
490	*/
491	int rc;
492	int fdKvm = open("/dev/kvm", O_RDWR \| O_CLOEXEC);
493	if (fdKvm >= 0)
494	{
495	pVM->nem.s.fdKvm = fdKvm;
496
497	/*
498	* Create an empty VM since it is recommended we check capabilities on
499	* the VM rather than the system descriptor.
500	*/
501	int fdVm = ioctl(fdKvm, KVM_CREATE_VM, 0UL /* Type must be zero on x86 */);
502	if (fdVm >= 0)
503	{
504	pVM->nem.s.fdVm = fdVm;
505
506	/*
507	* Check capabilities.
508	*/
509	rc = nemR3LnxInitCheckCapabilities(pVM, pErrInfo);
510	if (RT_SUCCESS(rc))
511	{
512	/*
513	* Set up the VM (more on this later).
514	*/
515	rc = nemR3LnxInitSetupVm(pVM, pErrInfo);
516	if (RT_SUCCESS(rc))
517	{
518	/*
519	* Set ourselves as the execution engine and make config adjustments.
520	*/
521	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
522	Log(("NEM: Marked active!\n"));
523	PGMR3EnableNemMode(pVM);
524
525	/*
526	* Register release statistics
527	*/
528	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
529	{
530	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
531	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnDemand, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of on-demand state imports", "/NEM/CPU%u/ImportOnDemand", idCpu);
532	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of state imports on loop return", "/NEM/CPU%u/ImportOnReturn", idCpu);
533	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturnSkipped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of skipped state imports on loop return", "/NEM/CPU%u/ImportOnReturnSkipped", idCpu);
534	STAMR3RegisterF(pVM, &pNemCpu->StatImportPendingInterrupt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times an interrupt was pending when importing from KVM", "/NEM/CPU%u/ImportPendingInterrupt", idCpu);
535	STAMR3RegisterF(pVM, &pNemCpu->StatExportPendingInterrupt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times an interrupt was pending when exporting to KVM", "/NEM/CPU%u/ExportPendingInterrupt", idCpu);
536	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn", idCpu);
537	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn1Loop, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-01-loop", idCpu);
538	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn2Loops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-02-loops", idCpu);
539	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn3Loops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-03-loops", idCpu);
540	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn4PlusLoops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-04-to-7-loops", idCpu);
541	STAMR3RegisterF(pVM, &pNemCpu->StatQueryCpuTick, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of TSC queries", "/NEM/CPU%u/QueryCpuTick", idCpu);
542	STAMR3RegisterF(pVM, &pNemCpu->StatExitTotal, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "All exits", "/NEM/CPU%u/Exit", idCpu);
543	STAMR3RegisterF(pVM, &pNemCpu->StatExitIo, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_IO", "/NEM/CPU%u/Exit/Io", idCpu);
544	STAMR3RegisterF(pVM, &pNemCpu->StatExitMmio, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_MMIO", "/NEM/CPU%u/Exit/Mmio", idCpu);
545	STAMR3RegisterF(pVM, &pNemCpu->StatExitSetTpr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_SET_TRP", "/NEM/CPU%u/Exit/SetTpr", idCpu);
546	STAMR3RegisterF(pVM, &pNemCpu->StatExitTprAccess, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_TPR_ACCESS", "/NEM/CPU%u/Exit/TprAccess", idCpu);
547	STAMR3RegisterF(pVM, &pNemCpu->StatExitRdMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_RDMSR", "/NEM/CPU%u/Exit/RdMsr", idCpu);
548	STAMR3RegisterF(pVM, &pNemCpu->StatExitWrMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_WRMSR", "/NEM/CPU%u/Exit/WrMsr", idCpu);
549	STAMR3RegisterF(pVM, &pNemCpu->StatExitIrqWindowOpen, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_IRQ_WINDOWS_OPEN", "/NEM/CPU%u/Exit/IrqWindowOpen", idCpu);
550	STAMR3RegisterF(pVM, &pNemCpu->StatExitHalt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_HLT", "/NEM/CPU%u/Exit/Hlt", idCpu);
551	STAMR3RegisterF(pVM, &pNemCpu->StatExitIntr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTR", "/NEM/CPU%u/Exit/Intr", idCpu);
552	STAMR3RegisterF(pVM, &pNemCpu->StatExitHypercall, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_HYPERCALL", "/NEM/CPU%u/Exit/Hypercall", idCpu);
553	STAMR3RegisterF(pVM, &pNemCpu->StatExitDebug, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_DEBUG", "/NEM/CPU%u/Exit/Debug", idCpu);
554	STAMR3RegisterF(pVM, &pNemCpu->StatExitBusLock, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_BUS_LOCK", "/NEM/CPU%u/Exit/BusLock", idCpu);
555	STAMR3RegisterF(pVM, &pNemCpu->StatExitInternalErrorEmulation, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTERNAL_ERROR/EMULATION", "/NEM/CPU%u/Exit/InternalErrorEmulation", idCpu);
556	STAMR3RegisterF(pVM, &pNemCpu->StatExitInternalErrorFatal, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTERNAL_ERROR/*", "/NEM/CPU%u/Exit/InternalErrorFatal", idCpu);
557	}
558
559	/*
560	* Make RTThreadPoke work again (disabled for avoiding unnecessary
561	* critical section issues in ring-0).
562	*/
563	VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, nemR3LnxFixThreadPoke, NULL);
564
565	/*
566	* Success.
567	*/
568	return VINF_SUCCESS;
569	}
570
571	/*
572	* Bail out.
573	*/
574	}
575	close(fdVm);
576	pVM->nem.s.fdVm = -1;
577	}
578	else
579	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VM failed: %u", errno);
580	close(fdKvm);
581	pVM->nem.s.fdKvm = -1;
582	}
583	else if (errno == EACCES)
584	rc = RTErrInfoSet(pErrInfo, VERR_ACCESS_DENIED, "Do not have access to open /dev/kvm for reading & writing.");
585	else if (errno == ENOENT)
586	rc = RTErrInfoSet(pErrInfo, VERR_NOT_SUPPORTED, "KVM is not availble (/dev/kvm does not exist)");
587	else
588	rc = RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno), "Failed to open '/dev/kvm': %u", errno);
589
590	/*
591	* We only fail if in forced mode, otherwise just log the complaint and return.
592	*/
593	Assert(RTErrInfoIsSet(pErrInfo));
594	if ( (fForced \|\| !fFallback)
595	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
596	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
597	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
598	return VINF_SUCCESS;
599	}
600
601
602	/**
603	* This is called after CPUMR3Init is done.
604	*
605	* @returns VBox status code.
606	* @param pVM The VM handle..
607	*/
608	int nemR3NativeInitAfterCPUM(PVM pVM)
609	{
610	/*
611	* Validate sanity.
612	*/
613	AssertReturn(pVM->nem.s.fdKvm >= 0, VERR_WRONG_ORDER);
614	AssertReturn(pVM->nem.s.fdVm >= 0, VERR_WRONG_ORDER);
615	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
616
617	/** @todo */
618
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Update the CPUID leaves for a VCPU.
625	*
626	* The KVM_SET_CPUID2 call replaces any previous leaves, so we have to redo
627	* everything when there really just are single bit changes. That said, it
628	* looks like KVM update the XCR/XSAVE related stuff as well as the APIC enabled
629	* bit(s), so it should suffice if we do this at startup, I hope.
630	*/
631	static int nemR3LnxUpdateCpuIdsLeaves(PVM pVM, PVMCPU pVCpu)
632	{
633	uint32_t cLeaves = 0;
634	PCCPUMCPUIDLEAF const paLeaves = CPUMR3CpuIdGetPtr(pVM, &cLeaves);
635	struct kvm_cpuid2 pReq = (struct kvm_cpuid2 )alloca(RT_UOFFSETOF_DYN(struct kvm_cpuid2, entries[cLeaves + 2]));
636
637	pReq->nent = cLeaves;
638	pReq->padding = 0;
639
640	for (uint32_t i = 0; i < cLeaves; i++)
641	{
642	CPUMGetGuestCpuId(pVCpu, paLeaves[i].uLeaf, paLeaves[i].uSubLeaf,
643	&pReq->entries[i].eax,
644	&pReq->entries[i].ebx,
645	&pReq->entries[i].ecx,
646	&pReq->entries[i].edx);
647	pReq->entries[i].function = paLeaves[i].uLeaf;
648	pReq->entries[i].index = paLeaves[i].uSubLeaf;
649	pReq->entries[i].flags = !paLeaves[i].fSubLeafMask ? 0 : KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
650	pReq->entries[i].padding[0] = 0;
651	pReq->entries[i].padding[1] = 0;
652	pReq->entries[i].padding[2] = 0;
653	}
654
655	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_CPUID2, pReq);
656	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d cLeaves=%#x\n", rcLnx, errno, cLeaves), RTErrConvertFromErrno(errno));
657
658	return VINF_SUCCESS;
659	}
660
661
662	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
663	{
664	/*
665	* Configure CPUIDs after ring-3 init has been done.
666	*/
667	if (enmWhat == VMINITCOMPLETED_RING3)
668	{
669	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
670	{
671	int rc = nemR3LnxUpdateCpuIdsLeaves(pVM, pVM->apCpusR3[idCpu]);
672	AssertRCReturn(rc, rc);
673	}
674	}
675
676	return VINF_SUCCESS;
677	}
678
679
680	int nemR3NativeTerm(PVM pVM)
681	{
682	/*
683	* Per-cpu data
684	*/
685	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
686	{
687	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
688
689	if (pVCpu->nem.s.fdVCpu != -1)
690	{
691	close(pVCpu->nem.s.fdVCpu);
692	pVCpu->nem.s.fdVCpu = -1;
693	}
694	if (pVCpu->nem.s.pRun)
695	{
696	munmap(pVCpu->nem.s.pRun, pVM->nem.s.cbVCpuMmap);
697	pVCpu->nem.s.pRun = NULL;
698	}
699	}
700
701	/*
702	* Global data.
703	*/
704	if (pVM->nem.s.fdVm != -1)
705	{
706	close(pVM->nem.s.fdVm);
707	pVM->nem.s.fdVm = -1;
708	}
709
710	if (pVM->nem.s.fdKvm != -1)
711	{
712	close(pVM->nem.s.fdKvm);
713	pVM->nem.s.fdKvm = -1;
714	}
715	return VINF_SUCCESS;
716	}
717
718
719	/**
720	* VM reset notification.
721	*
722	* @param pVM The cross context VM structure.
723	*/
724	void nemR3NativeReset(PVM pVM)
725	{
726	RT_NOREF(pVM);
727	}
728
729
730	/**
731	* Reset CPU due to INIT IPI or hot (un)plugging.
732	*
733	* @param pVCpu The cross context virtual CPU structure of the CPU being
734	* reset.
735	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
736	*/
737	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
738	{
739	RT_NOREF(pVCpu, fInitIpi);
740	}
741
742
743	/*********************************************************************************************************************************
744	* Memory management *
745	*********************************************************************************************************************************/
746
747
748	/**
749	* Allocates a memory slot ID.
750	*
751	* @returns Slot ID on success, UINT16_MAX on failure.
752	*/
753	static uint16_t nemR3LnxMemSlotIdAlloc(PVM pVM)
754	{
755	/* Use the hint first. */
756	uint16_t idHint = pVM->nem.s.idPrevSlot;
757	if (idHint < _32K - 1)
758	{
759	int32_t idx = ASMBitNextClear(&pVM->nem.s.bmSlotIds, _32K, idHint);
760	Assert(idx < _32K);
761	if (idx > 0 && !ASMAtomicBitTestAndSet(&pVM->nem.s.bmSlotIds, idx))
762	return pVM->nem.s.idPrevSlot = (uint16_t)idx;
763	}
764
765	/*
766	* Search the whole map from the start.
767	*/
768	int32_t idx = ASMBitFirstClear(&pVM->nem.s.bmSlotIds, _32K);
769	Assert(idx < _32K);
770	if (idx > 0 && !ASMAtomicBitTestAndSet(&pVM->nem.s.bmSlotIds, idx))
771	return pVM->nem.s.idPrevSlot = (uint16_t)idx;
772
773	Assert(idx < 0 /shouldn't trigger unless there is a race /);
774	return UINT16_MAX; /* caller is expected to assert. */
775	}
776
777
778	/**
779	* Frees a memory slot ID
780	*/
781	static void nemR3LnxMemSlotIdFree(PVM pVM, uint16_t idSlot)
782	{
783	if (RT_LIKELY(idSlot < _32K && ASMAtomicBitTestAndClear(&pVM->nem.s.bmSlotIds, idSlot)))
784	{ /likely/ }
785	else
786	AssertMsgFailed(("idSlot=%u (%#x)\n", idSlot, idSlot));
787	}
788
789
790
791	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
792	uint8_t pu2State, uint32_t puNemRange)
793	{
794	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
795	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
796
797	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p pu2State=%p (%d) puNemRange=%p (%d) - idSlot=%#x\n",
798	GCPhys, cb, pvR3, pu2State, pu2State, puNemRange, *puNemRange, idSlot));
799
800	struct kvm_userspace_memory_region Region;
801	Region.slot = idSlot;
802	Region.flags = 0;
803	Region.guest_phys_addr = GCPhys;
804	Region.memory_size = cb;
805	Region.userspace_addr = (uintptr_t)pvR3;
806
807	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
808	if (rc == 0)
809	{
810	*pu2State = 0;
811	*puNemRange = idSlot;
812	return VINF_SUCCESS;
813	}
814
815	LogRel(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p, idSlot=%#x failed: %u/%u\n", GCPhys, cb, pvR3, idSlot, rc, errno));
816	nemR3LnxMemSlotIdFree(pVM, idSlot);
817	return VERR_NEM_MAP_PAGES_FAILED;
818	}
819
820
821	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
822	{
823	RT_NOREF(pVM);
824	return true;
825	}
826
827
828	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
829	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
830	{
831	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d) puNemRange=%p (%#x)\n",
832	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, pu2State, puNemRange, puNemRange ? puNemRange : UINT32_MAX));
833	RT_NOREF(pvRam);
834
835	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
836	{
837	/** @todo implement splitting and whatnot of ranges if we want to be 100%
838	* conforming (just modify RAM registrations in MM.cpp to test). */
839	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p\n", GCPhys, cb, fFlags, pvRam, pvMmio2),
840	VERR_NEM_MAP_PAGES_FAILED);
841	}
842
843	/*
844	* Register MMIO2.
845	*/
846	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
847	{
848	AssertReturn(pvMmio2, VERR_NEM_MAP_PAGES_FAILED);
849	AssertReturn(puNemRange, VERR_NEM_MAP_PAGES_FAILED);
850
851	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
852	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
853
854	struct kvm_userspace_memory_region Region;
855	Region.slot = idSlot;
856	Region.flags = fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES ? KVM_MEM_LOG_DIRTY_PAGES : 0;
857	Region.guest_phys_addr = GCPhys;
858	Region.memory_size = cb;
859	Region.userspace_addr = (uintptr_t)pvMmio2;
860
861	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
862	if (rc == 0)
863	{
864	*pu2State = 0;
865	*puNemRange = idSlot;
866	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvMmio2=%p - idSlot=%#x\n",
867	GCPhys, cb, fFlags, pvMmio2, idSlot));
868	return VINF_SUCCESS;
869	}
870
871	nemR3LnxMemSlotIdFree(pVM, idSlot);
872	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvMmio2=%p, idSlot=%#x failed: %u/%u\n",
873	GCPhys, cb, fFlags, pvMmio2, idSlot, errno, rc),
874	VERR_NEM_MAP_PAGES_FAILED);
875	}
876
877	/* MMIO, don't care. */
878	*pu2State = 0;
879	*puNemRange = UINT32_MAX;
880	return VINF_SUCCESS;
881	}
882
883
884	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
885	void pvRam, void pvMmio2, uint32_t *puNemRange)
886	{
887	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
888	return VINF_SUCCESS;
889	}
890
891
892	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
893	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
894	{
895	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p puNemRange=%p (%#x)\n",
896	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
897	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
898
899	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
900	{
901	/** @todo implement splitting and whatnot of ranges if we want to be 100%
902	* conforming (just modify RAM registrations in MM.cpp to test). */
903	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p\n", GCPhys, cb, fFlags, pvRam, pvMmio2),
904	VERR_NEM_UNMAP_PAGES_FAILED);
905	}
906
907	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
908	{
909	uint32_t const idSlot = *puNemRange;
910	AssertReturn(idSlot > 0 && idSlot < _32K, VERR_NEM_IPE_4);
911	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, idSlot), VERR_NEM_IPE_4);
912
913	struct kvm_userspace_memory_region Region;
914	Region.slot = idSlot;
915	Region.flags = 0;
916	Region.guest_phys_addr = GCPhys;
917	Region.memory_size = 0; /* this deregisters it. */
918	Region.userspace_addr = (uintptr_t)pvMmio2;
919
920	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
921	if (rc == 0)
922	{
923	if (pu2State)
924	*pu2State = 0;
925	*puNemRange = UINT32_MAX;
926	nemR3LnxMemSlotIdFree(pVM, idSlot);
927	return VINF_SUCCESS;
928	}
929
930	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvMmio2=%p, idSlot=%#x failed: %u/%u\n",
931	GCPhys, cb, fFlags, pvMmio2, idSlot, errno, rc),
932	VERR_NEM_UNMAP_PAGES_FAILED);
933	}
934
935	if (pu2State)
936	*pu2State = UINT8_MAX;
937	return VINF_SUCCESS;
938	}
939
940
941	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
942	void *pvBitmap, size_t cbBitmap)
943	{
944	AssertReturn(uNemRange > 0 && uNemRange < _32K, VERR_NEM_IPE_4);
945	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, uNemRange), VERR_NEM_IPE_4);
946
947	RT_NOREF(GCPhys, cbBitmap);
948
949	struct kvm_dirty_log DirtyLog;
950	DirtyLog.slot = uNemRange;
951	DirtyLog.padding1 = 0;
952	DirtyLog.dirty_bitmap = pvBitmap;
953
954	int rc = ioctl(pVM->nem.s.fdVm, KVM_GET_DIRTY_LOG, &DirtyLog);
955	AssertLogRelMsgReturn(rc == 0, ("%RGp LB %RGp idSlot=%#x failed: %u/%u\n", GCPhys, cb, uNemRange, errno, rc),
956	VERR_NEM_QUERY_DIRTY_BITMAP_FAILED);
957
958	return VINF_SUCCESS;
959	}
960
961
962	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
963	uint8_t pu2State, uint32_t puNemRange)
964	{
965	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
966	*pu2State = UINT8_MAX;
967
968	/* Don't support puttint ROM where there is already RAM. For
969	now just shuffle the registrations till it works... */
970	AssertLogRelMsgReturn(!(fFlags & NEM_NOTIFY_PHYS_ROM_F_REPLACE), ("%RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags),
971	VERR_NEM_MAP_PAGES_FAILED);
972
973	/** @todo figure out how to do shadow ROMs. */
974
975	/*
976	* We only allocate a slot number here in case we need to use it to
977	* fend of physical handler fun.
978	*/
979	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
980	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
981
982	*pu2State = 0;
983	*puNemRange = idSlot;
984	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x pvPages=%p - idSlot=%#x\n",
985	GCPhys, cb, fFlags, pvPages, idSlot));
986	RT_NOREF(GCPhys, cb, fFlags, pvPages);
987	return VINF_SUCCESS;
988	}
989
990
991	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
992	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
993	{
994	Log5(("NEMR3NotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
995	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
996
997	AssertPtrReturn(pvPages, VERR_NEM_IPE_5);
998
999	uint32_t const idSlot = *puNemRange;
1000	AssertReturn(idSlot > 0 && idSlot < _32K, VERR_NEM_IPE_4);
1001	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, idSlot), VERR_NEM_IPE_4);
1002
1003	*pu2State = UINT8_MAX;
1004
1005	/*
1006	* Do the actual setting of the user pages here now that we've
1007	* got a valid pvPages (typically isn't available during the early
1008	* notification, unless we're replacing RAM).
1009	*/
1010	struct kvm_userspace_memory_region Region;
1011	Region.slot = idSlot;
1012	Region.flags = 0;
1013	Region.guest_phys_addr = GCPhys;
1014	Region.memory_size = cb;
1015	Region.userspace_addr = (uintptr_t)pvPages;
1016
1017	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
1018	if (rc == 0)
1019	{
1020	*pu2State = 0;
1021	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x pvPages=%p - idSlot=%#x\n",
1022	GCPhys, cb, fFlags, pvPages, idSlot));
1023	return VINF_SUCCESS;
1024	}
1025	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvPages=%p, idSlot=%#x failed: %u/%u\n",
1026	GCPhys, cb, fFlags, pvPages, idSlot, errno, rc),
1027	VERR_NEM_MAP_PAGES_FAILED);
1028	}
1029
1030
1031	/**
1032	* Called when the A20 state changes.
1033	*
1034	* @param pVCpu The CPU the A20 state changed on.
1035	* @param fEnabled Whether it was enabled (true) or disabled.
1036	*/
1037	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
1038	{
1039	Log(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
1040	Assert(VM_IS_NEM_ENABLED(pVCpu->CTX_SUFF(pVM)));
1041	RT_NOREF(pVCpu, fEnabled);
1042	}
1043
1044
1045	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
1046	RTR3PTR pvMemR3, uint8_t *pu2State)
1047	{
1048	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
1049	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
1050
1051	*pu2State = UINT8_MAX;
1052	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
1053	}
1054
1055
1056	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
1057	{
1058	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
1059	RT_NOREF(pVM, enmKind, GCPhys, cb);
1060	}
1061
1062
1063	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
1064	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
1065	{
1066	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
1067	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
1068	RT_NOREF(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM);
1069	}
1070
1071
1072	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
1073	PGMPAGETYPE enmType, uint8_t *pu2State)
1074	{
1075	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
1076	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
1077	RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
1078	return VINF_SUCCESS;
1079	}
1080
1081
1082	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
1083	PGMPAGETYPE enmType, uint8_t *pu2State)
1084	{
1085	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
1086	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
1087	Assert(VM_IS_NEM_ENABLED(pVM));
1088	RT_NOREF(pVM, GCPhys, HCPhys, pvR3, fPageProt, enmType, pu2State);
1089
1090	}
1091
1092
1093	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
1094	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
1095	{
1096	Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp pvNewR3=%p fPageProt=%#x enmType=%d *pu2State=%d\n",
1097	GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, *pu2State));
1098	Assert(VM_IS_NEM_ENABLED(pVM));
1099	RT_NOREF(pVM, GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, pu2State);
1100	}
1101
1102
1103	/*********************************************************************************************************************************
1104	* CPU State *
1105	*********************************************************************************************************************************/
1106
1107	/**
1108	* Worker that imports selected state from KVM.
1109	*/
1110	static int nemHCLnxImportState(PVMCPUCC pVCpu, uint64_t fWhat, PCPUMCTX pCtx, struct kvm_run *pRun)
1111	{
1112	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
1113	if (!fWhat)
1114	return VINF_SUCCESS;
1115
1116	/*
1117	* Stuff that goes into kvm_run::s.regs.regs:
1118	*/
1119	if (fWhat & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK))
1120	{
1121	if (fWhat & CPUMCTX_EXTRN_RIP)
1122	pCtx->rip = pRun->s.regs.regs.rip;
1123	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
1124	pCtx->rflags.u = pRun->s.regs.regs.rflags;
1125
1126	if (fWhat & CPUMCTX_EXTRN_RAX)
1127	pCtx->rax = pRun->s.regs.regs.rax;
1128	if (fWhat & CPUMCTX_EXTRN_RCX)
1129	pCtx->rcx = pRun->s.regs.regs.rcx;
1130	if (fWhat & CPUMCTX_EXTRN_RDX)
1131	pCtx->rdx = pRun->s.regs.regs.rdx;
1132	if (fWhat & CPUMCTX_EXTRN_RBX)
1133	pCtx->rbx = pRun->s.regs.regs.rbx;
1134	if (fWhat & CPUMCTX_EXTRN_RSP)
1135	pCtx->rsp = pRun->s.regs.regs.rsp;
1136	if (fWhat & CPUMCTX_EXTRN_RBP)
1137	pCtx->rbp = pRun->s.regs.regs.rbp;
1138	if (fWhat & CPUMCTX_EXTRN_RSI)
1139	pCtx->rsi = pRun->s.regs.regs.rsi;
1140	if (fWhat & CPUMCTX_EXTRN_RDI)
1141	pCtx->rdi = pRun->s.regs.regs.rdi;
1142	if (fWhat & CPUMCTX_EXTRN_R8_R15)
1143	{
1144	pCtx->r8 = pRun->s.regs.regs.r8;
1145	pCtx->r9 = pRun->s.regs.regs.r9;
1146	pCtx->r10 = pRun->s.regs.regs.r10;
1147	pCtx->r11 = pRun->s.regs.regs.r11;
1148	pCtx->r12 = pRun->s.regs.regs.r12;
1149	pCtx->r13 = pRun->s.regs.regs.r13;
1150	pCtx->r14 = pRun->s.regs.regs.r14;
1151	pCtx->r15 = pRun->s.regs.regs.r15;
1152	}
1153	}
1154
1155	/*
1156	* Stuff that goes into kvm_run::s.regs.sregs:
1157	*/
1158	/** @todo apic_base */
1159
1160	bool fMaybeChangedMode = false;
1161	bool fUpdateCr3 = false;
1162	if (fWhat & ( CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_CR_MASK
1163	\| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_APIC_TPR))
1164	{
1165	/** @todo what about Attr.n.u4LimitHigh? */
1166	#define NEM_LNX_IMPORT_SEG(a_CtxSeg, a_KvmSeg) do { \
1167	(a_CtxSeg).u64Base = (a_KvmSeg).base; \
1168	(a_CtxSeg).u32Limit = (a_KvmSeg).limit; \
1169	(a_CtxSeg).ValidSel = (a_CtxSeg).Sel = (a_KvmSeg).selector; \
1170	(a_CtxSeg).Attr.n.u4Type = (a_KvmSeg).type; \
1171	(a_CtxSeg).Attr.n.u1DescType = (a_KvmSeg).s; \
1172	(a_CtxSeg).Attr.n.u2Dpl = (a_KvmSeg).dpl; \
1173	(a_CtxSeg).Attr.n.u1Present = (a_KvmSeg).present; \
1174	(a_CtxSeg).Attr.n.u1Available = (a_KvmSeg).avl; \
1175	(a_CtxSeg).Attr.n.u1Long = (a_KvmSeg).l; \
1176	(a_CtxSeg).Attr.n.u1DefBig = (a_KvmSeg).db; \
1177	(a_CtxSeg).Attr.n.u1Granularity = (a_KvmSeg).g; \
1178	(a_CtxSeg).Attr.n.u1Unusable = (a_KvmSeg).unusable; \
1179	(a_CtxSeg).fFlags = CPUMSELREG_FLAGS_VALID; \
1180	CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &(a_CtxSeg)); \
1181	} while (0)
1182
1183	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
1184	{
1185	if (fWhat & CPUMCTX_EXTRN_ES)
1186	NEM_LNX_IMPORT_SEG(pCtx->es, pRun->s.regs.sregs.es);
1187	if (fWhat & CPUMCTX_EXTRN_CS)
1188	NEM_LNX_IMPORT_SEG(pCtx->cs, pRun->s.regs.sregs.cs);
1189	if (fWhat & CPUMCTX_EXTRN_SS)
1190	NEM_LNX_IMPORT_SEG(pCtx->ss, pRun->s.regs.sregs.ss);
1191	if (fWhat & CPUMCTX_EXTRN_DS)
1192	NEM_LNX_IMPORT_SEG(pCtx->ds, pRun->s.regs.sregs.ds);
1193	if (fWhat & CPUMCTX_EXTRN_FS)
1194	NEM_LNX_IMPORT_SEG(pCtx->fs, pRun->s.regs.sregs.fs);
1195	if (fWhat & CPUMCTX_EXTRN_GS)
1196	NEM_LNX_IMPORT_SEG(pCtx->gs, pRun->s.regs.sregs.gs);
1197	}
1198	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
1199	{
1200	if (fWhat & CPUMCTX_EXTRN_GDTR)
1201	{
1202	pCtx->gdtr.pGdt = pRun->s.regs.sregs.gdt.base;
1203	pCtx->gdtr.cbGdt = pRun->s.regs.sregs.gdt.limit;
1204	}
1205	if (fWhat & CPUMCTX_EXTRN_IDTR)
1206	{
1207	pCtx->idtr.pIdt = pRun->s.regs.sregs.idt.base;
1208	pCtx->idtr.cbIdt = pRun->s.regs.sregs.idt.limit;
1209	}
1210	if (fWhat & CPUMCTX_EXTRN_LDTR)
1211	NEM_LNX_IMPORT_SEG(pCtx->ldtr, pRun->s.regs.sregs.ldt);
1212	if (fWhat & CPUMCTX_EXTRN_TR)
1213	NEM_LNX_IMPORT_SEG(pCtx->tr, pRun->s.regs.sregs.tr);
1214	}
1215	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
1216	{
1217	if (fWhat & CPUMCTX_EXTRN_CR0)
1218	{
1219	if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
1220	{
1221	CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
1222	fMaybeChangedMode = true;
1223	}
1224	}
1225	if (fWhat & CPUMCTX_EXTRN_CR2)
1226	pCtx->cr2 = pRun->s.regs.sregs.cr2;
1227	if (fWhat & CPUMCTX_EXTRN_CR3)
1228	{
1229	if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
1230	{
1231	CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
1232	fUpdateCr3 = true;
1233	}
1234	}
1235	if (fWhat & CPUMCTX_EXTRN_CR4)
1236	{
1237	if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
1238	{
1239	CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
1240	fMaybeChangedMode = true;
1241	}
1242	}
1243	}
1244	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1245	APICSetTpr(pVCpu, (uint8_t)pRun->s.regs.sregs.cr8 << 4);
1246	if (fWhat & CPUMCTX_EXTRN_EFER)
1247	{
1248	if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
1249	{
1250	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
1251	if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1252	PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
1253	pCtx->msrEFER = pRun->s.regs.sregs.efer;
1254	fMaybeChangedMode = true;
1255	}
1256	}
1257
1258	/** @todo apic_base */
1259	#undef NEM_LNX_IMPORT_SEG
1260	}
1261
1262	/*
1263	* Debug registers.
1264	*/
1265	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
1266	{
1267	struct kvm_debugregs DbgRegs = {{0}};
1268	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
1269	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1270
1271	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1272	{
1273	pCtx->dr[0] = DbgRegs.db[0];
1274	pCtx->dr[1] = DbgRegs.db[1];
1275	pCtx->dr[2] = DbgRegs.db[2];
1276	pCtx->dr[3] = DbgRegs.db[3];
1277	}
1278	if (fWhat & CPUMCTX_EXTRN_DR6)
1279	pCtx->dr[6] = DbgRegs.dr6;
1280	if (fWhat & CPUMCTX_EXTRN_DR7)
1281	pCtx->dr[7] = DbgRegs.dr7;
1282	}
1283
1284	/*
1285	* FPU, SSE, AVX, ++.
1286	*/
1287	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx))
1288	{
1289	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1290	{
1291	fWhat \|= CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE; /* we do all or nothing at all */
1292
1293	AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
1294	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XSAVE, &pCtx->XState);
1295	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1296	}
1297
1298	if (fWhat & CPUMCTX_EXTRN_XCRx)
1299	{
1300	struct kvm_xcrs Xcrs =
1301	{ /.nr_xcrs = / 2,
1302	/.flags = / 0,
1303	/.xcrs= / {
1304	{ /.xcr =/ 0, /.reserved=/ 0, /.value=/ pCtx->aXcr[0] },
1305	{ /.xcr =/ 1, /.reserved=/ 0, /.value=/ pCtx->aXcr[1] },
1306	}
1307	};
1308
1309	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XCRS, &Xcrs);
1310	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1311
1312	pCtx->aXcr[0] = Xcrs.xcrs[0].value;
1313	pCtx->aXcr[1] = Xcrs.xcrs[1].value;
1314	}
1315	}
1316
1317	/*
1318	* MSRs.
1319	*/
1320	if (fWhat & ( CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_SYSENTER_MSRS
1321	\| CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
1322	{
1323	union
1324	{
1325	struct kvm_msrs Core;
1326	uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
1327	} uBuf;
1328	uint64_t *pauDsts[32];
1329	uint32_t iMsr = 0;
1330	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1331
1332	#define ADD_MSR(a_Msr, a_uValue) do { \
1333	Assert(iMsr < 32); \
1334	uBuf.Core.entries[iMsr].index = (a_Msr); \
1335	uBuf.Core.entries[iMsr].reserved = 0; \
1336	uBuf.Core.entries[iMsr].data = UINT64_MAX; \
1337	pauDsts[iMsr] = &(a_uValue); \
1338	iMsr += 1; \
1339	} while (0)
1340
1341	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1342	ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1343	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1344	{
1345	ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
1346	ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
1347	ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
1348	ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1349	}
1350	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1351	{
1352	ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
1353	ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
1354	ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
1355	}
1356	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1357	ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1358	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1359	{
1360	ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
1361	/** @todo What do we _have_ to add here?
1362	* We also have: Mttr, MiscEnable, FeatureControl. /
1363	}
1364
1365	uBuf.Core.pad = 0;
1366	uBuf.Core.nmsrs = iMsr;
1367	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MSRS, &uBuf);
1368	AssertMsgReturn(rc == (int)iMsr,
1369	("rc=%d iMsr=%d (->%#x) errno=%d\n",
1370	rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
1371	VERR_NEM_IPE_3);
1372
1373	while (iMsr-- > 0)
1374	*pauDsts[iMsr] = uBuf.Core.entries[iMsr].data;
1375	#undef ADD_MSR
1376	}
1377
1378	/*
1379	* Interruptibility state and pending interrupts.
1380	*/
1381	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1382	{
1383	fWhat \|= CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI; /* always do both, see export and interrupt FF handling */
1384
1385	struct kvm_vcpu_events KvmEvents = {0};
1386	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
1387	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
1388
1389	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
1390	pVCpu->cpum.GstCtx.rip = pRun->s.regs.regs.rip;
1391
1392	if (KvmEvents.interrupt.shadow)
1393	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
1394	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1395	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1396
1397	if (KvmEvents.nmi.masked)
1398	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
1399	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
1400	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
1401
1402	if (KvmEvents.interrupt.injected)
1403	{
1404	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportPendingInterrupt);
1405	TRPMAssertTrap(pVCpu, KvmEvents.interrupt.nr, !KvmEvents.interrupt.soft ? TRPM_HARDWARE_INT : TRPM_SOFTWARE_INT);
1406	}
1407
1408	Assert(KvmEvents.nmi.injected == 0);
1409	Assert(KvmEvents.nmi.pending == 0);
1410	}
1411
1412	/*
1413	* Update the external mask.
1414	*/
1415	pCtx->fExtrn &= ~fWhat;
1416	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1417	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1418	pVCpu->cpum.GstCtx.fExtrn = 0;
1419
1420	/*
1421	* We sometimes need to update PGM on the guest status.
1422	*/
1423	if (!fMaybeChangedMode && !fUpdateCr3)
1424	{ /* likely */ }
1425	else
1426	{
1427	/*
1428	* Make sure we got all the state PGM might need.
1429	*/
1430	Log7(("nemHCLnxImportState: fMaybeChangedMode=%d fUpdateCr3=%d fExtrnNeeded=%#RX64\n", fMaybeChangedMode, fUpdateCr3,
1431	pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER) ));
1432	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER))
1433	{
1434	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR0)
1435	{
1436	if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
1437	{
1438	CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
1439	fMaybeChangedMode = true;
1440	}
1441	}
1442	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR3)
1443	{
1444	if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
1445	{
1446	CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
1447	fUpdateCr3 = true;
1448	}
1449	}
1450	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR4)
1451	{
1452	if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
1453	{
1454	CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
1455	fMaybeChangedMode = true;
1456	}
1457	}
1458	if (fWhat & CPUMCTX_EXTRN_EFER)
1459	{
1460	if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
1461	{
1462	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
1463	if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1464	PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
1465	pCtx->msrEFER = pRun->s.regs.sregs.efer;
1466	fMaybeChangedMode = true;
1467	}
1468	}
1469
1470	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER);
1471	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1472	pVCpu->cpum.GstCtx.fExtrn = 0;
1473	}
1474
1475	/*
1476	* Notify PGM about the changes.
1477	*/
1478	if (fMaybeChangedMode)
1479	{
1480	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4,
1481	pVCpu->cpum.GstCtx.msrEFER, false /fForce/);
1482	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1483	}
1484
1485	if (fUpdateCr3)
1486	{
1487	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3, false /fPdpesMapped/);
1488	if (rc == VINF_SUCCESS)
1489	{ /* likely */ }
1490	else
1491	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1492	}
1493	}
1494
1495	return VINF_SUCCESS;
1496	}
1497
1498
1499	/**
1500	* Interface for importing state on demand (used by IEM).
1501	*
1502	* @returns VBox status code.
1503	* @param pVCpu The cross context CPU structure.
1504	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1505	*/
1506	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1507	{
1508	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
1509	return nemHCLnxImportState(pVCpu, fWhat, &pVCpu->cpum.GstCtx, pVCpu->nem.s.pRun);
1510	}
1511
1512
1513	/**
1514	* Exports state to KVM.
1515	*/
1516	static int nemHCLnxExportState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, struct kvm_run *pRun)
1517	{
1518	uint64_t const fExtrn = ~pCtx->fExtrn & CPUMCTX_EXTRN_ALL;
1519	Assert((~fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL);
1520
1521	/*
1522	* Stuff that goes into kvm_run::s.regs.regs:
1523	*/
1524	if (fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK))
1525	{
1526	if (fExtrn & CPUMCTX_EXTRN_RIP)
1527	pRun->s.regs.regs.rip = pCtx->rip;
1528	if (fExtrn & CPUMCTX_EXTRN_RFLAGS)
1529	pRun->s.regs.regs.rflags = pCtx->rflags.u;
1530
1531	if (fExtrn & CPUMCTX_EXTRN_RAX)
1532	pRun->s.regs.regs.rax = pCtx->rax;
1533	if (fExtrn & CPUMCTX_EXTRN_RCX)
1534	pRun->s.regs.regs.rcx = pCtx->rcx;
1535	if (fExtrn & CPUMCTX_EXTRN_RDX)
1536	pRun->s.regs.regs.rdx = pCtx->rdx;
1537	if (fExtrn & CPUMCTX_EXTRN_RBX)
1538	pRun->s.regs.regs.rbx = pCtx->rbx;
1539	if (fExtrn & CPUMCTX_EXTRN_RSP)
1540	pRun->s.regs.regs.rsp = pCtx->rsp;
1541	if (fExtrn & CPUMCTX_EXTRN_RBP)
1542	pRun->s.regs.regs.rbp = pCtx->rbp;
1543	if (fExtrn & CPUMCTX_EXTRN_RSI)
1544	pRun->s.regs.regs.rsi = pCtx->rsi;
1545	if (fExtrn & CPUMCTX_EXTRN_RDI)
1546	pRun->s.regs.regs.rdi = pCtx->rdi;
1547	if (fExtrn & CPUMCTX_EXTRN_R8_R15)
1548	{
1549	pRun->s.regs.regs.r8 = pCtx->r8;
1550	pRun->s.regs.regs.r9 = pCtx->r9;
1551	pRun->s.regs.regs.r10 = pCtx->r10;
1552	pRun->s.regs.regs.r11 = pCtx->r11;
1553	pRun->s.regs.regs.r12 = pCtx->r12;
1554	pRun->s.regs.regs.r13 = pCtx->r13;
1555	pRun->s.regs.regs.r14 = pCtx->r14;
1556	pRun->s.regs.regs.r15 = pCtx->r15;
1557	}
1558	pRun->kvm_dirty_regs \|= KVM_SYNC_X86_REGS;
1559	}
1560
1561	/*
1562	* Stuff that goes into kvm_run::s.regs.sregs:
1563	*/
1564	/** @todo apic_base */
1565	if (fExtrn & ( CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_CR_MASK
1566	\| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_APIC_TPR))
1567	{
1568	#define NEM_LNX_EXPORT_SEG(a_KvmSeg, a_CtxSeg) do { \
1569	(a_KvmSeg).base = (a_CtxSeg).u64Base; \
1570	(a_KvmSeg).limit = (a_CtxSeg).u32Limit; \
1571	(a_KvmSeg).selector = (a_CtxSeg).Sel; \
1572	(a_KvmSeg).type = (a_CtxSeg).Attr.n.u4Type; \
1573	(a_KvmSeg).s = (a_CtxSeg).Attr.n.u1DescType; \
1574	(a_KvmSeg).dpl = (a_CtxSeg).Attr.n.u2Dpl; \
1575	(a_KvmSeg).present = (a_CtxSeg).Attr.n.u1Present; \
1576	(a_KvmSeg).avl = (a_CtxSeg).Attr.n.u1Available; \
1577	(a_KvmSeg).l = (a_CtxSeg).Attr.n.u1Long; \
1578	(a_KvmSeg).db = (a_CtxSeg).Attr.n.u1DefBig; \
1579	(a_KvmSeg).g = (a_CtxSeg).Attr.n.u1Granularity; \
1580	(a_KvmSeg).unusable = (a_CtxSeg).Attr.n.u1Unusable; \
1581	(a_KvmSeg).padding = 0; \
1582	} while (0)
1583
1584	if (fExtrn & CPUMCTX_EXTRN_SREG_MASK)
1585	{
1586	if (fExtrn & CPUMCTX_EXTRN_ES)
1587	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.es, pCtx->es);
1588	if (fExtrn & CPUMCTX_EXTRN_CS)
1589	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.cs, pCtx->cs);
1590	if (fExtrn & CPUMCTX_EXTRN_SS)
1591	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ss, pCtx->ss);
1592	if (fExtrn & CPUMCTX_EXTRN_DS)
1593	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ds, pCtx->ds);
1594	if (fExtrn & CPUMCTX_EXTRN_FS)
1595	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.fs, pCtx->fs);
1596	if (fExtrn & CPUMCTX_EXTRN_GS)
1597	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.gs, pCtx->gs);
1598	}
1599	if (fExtrn & CPUMCTX_EXTRN_TABLE_MASK)
1600	{
1601	if (fExtrn & CPUMCTX_EXTRN_GDTR)
1602	{
1603	pRun->s.regs.sregs.gdt.base = pCtx->gdtr.pGdt;
1604	pRun->s.regs.sregs.gdt.limit = pCtx->gdtr.cbGdt;
1605	pRun->s.regs.sregs.gdt.padding[0] = 0;
1606	pRun->s.regs.sregs.gdt.padding[1] = 0;
1607	pRun->s.regs.sregs.gdt.padding[2] = 0;
1608	}
1609	if (fExtrn & CPUMCTX_EXTRN_IDTR)
1610	{
1611	pRun->s.regs.sregs.idt.base = pCtx->idtr.pIdt;
1612	pRun->s.regs.sregs.idt.limit = pCtx->idtr.cbIdt;
1613	pRun->s.regs.sregs.idt.padding[0] = 0;
1614	pRun->s.regs.sregs.idt.padding[1] = 0;
1615	pRun->s.regs.sregs.idt.padding[2] = 0;
1616	}
1617	if (fExtrn & CPUMCTX_EXTRN_LDTR)
1618	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ldt, pCtx->ldtr);
1619	if (fExtrn & CPUMCTX_EXTRN_TR)
1620	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.tr, pCtx->tr);
1621	}
1622	if (fExtrn & CPUMCTX_EXTRN_CR_MASK)
1623	{
1624	if (fExtrn & CPUMCTX_EXTRN_CR0)
1625	pRun->s.regs.sregs.cr0 = pCtx->cr0;
1626	if (fExtrn & CPUMCTX_EXTRN_CR2)
1627	pRun->s.regs.sregs.cr2 = pCtx->cr2;
1628	if (fExtrn & CPUMCTX_EXTRN_CR3)
1629	pRun->s.regs.sregs.cr3 = pCtx->cr3;
1630	if (fExtrn & CPUMCTX_EXTRN_CR4)
1631	pRun->s.regs.sregs.cr4 = pCtx->cr4;
1632	}
1633	if (fExtrn & CPUMCTX_EXTRN_APIC_TPR)
1634	pRun->s.regs.sregs.cr8 = CPUMGetGuestCR8(pVCpu);
1635	if (fExtrn & CPUMCTX_EXTRN_EFER)
1636	pRun->s.regs.sregs.efer = pCtx->msrEFER;
1637
1638	/** @todo apic_base */
1639
1640	RT_ZERO(pRun->s.regs.sregs.interrupt_bitmap); /* this is an alternative interrupt injection interface */
1641
1642	pRun->kvm_dirty_regs \|= KVM_SYNC_X86_SREGS;
1643	}
1644
1645	/*
1646	* Debug registers.
1647	*/
1648	if (fExtrn & CPUMCTX_EXTRN_DR_MASK)
1649	{
1650	struct kvm_debugregs DbgRegs = {{0}};
1651
1652	if ((fExtrn & CPUMCTX_EXTRN_DR_MASK) != CPUMCTX_EXTRN_DR_MASK)
1653	{
1654	/* Partial debug state, we must get DbgRegs first so we can merge: */
1655	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
1656	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1657	}
1658
1659	if (fExtrn & CPUMCTX_EXTRN_DR0_DR3)
1660	{
1661	DbgRegs.db[0] = pCtx->dr[0];
1662	DbgRegs.db[1] = pCtx->dr[1];
1663	DbgRegs.db[2] = pCtx->dr[2];
1664	DbgRegs.db[3] = pCtx->dr[3];
1665	}
1666	if (fExtrn & CPUMCTX_EXTRN_DR6)
1667	DbgRegs.dr6 = pCtx->dr[6];
1668	if (fExtrn & CPUMCTX_EXTRN_DR7)
1669	DbgRegs.dr7 = pCtx->dr[7];
1670
1671	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_DEBUGREGS, &DbgRegs);
1672	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1673	}
1674
1675	/*
1676	* FPU, SSE, AVX, ++.
1677	*/
1678	if (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx))
1679	{
1680	if (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1681	{
1682	/** @todo could IEM just grab state partial control in some situations? */
1683	Assert( (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1684	== (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE)); /* no partial states */
1685
1686	AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
1687	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XSAVE, &pCtx->XState);
1688	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1689	}
1690
1691	if (fExtrn & CPUMCTX_EXTRN_XCRx)
1692	{
1693	struct kvm_xcrs Xcrs =
1694	{ /.nr_xcrs = / 2,
1695	/.flags = / 0,
1696	/.xcrs= / {
1697	{ /.xcr =/ 0, /.reserved=/ 0, /.value=/ pCtx->aXcr[0] },
1698	{ /.xcr =/ 1, /.reserved=/ 0, /.value=/ pCtx->aXcr[1] },
1699	}
1700	};
1701
1702	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XCRS, &Xcrs);
1703	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1704	}
1705	}
1706
1707	/*
1708	* MSRs.
1709	*/
1710	if (fExtrn & ( CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_SYSENTER_MSRS
1711	\| CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
1712	{
1713	union
1714	{
1715	struct kvm_msrs Core;
1716	uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
1717	} uBuf;
1718	uint32_t iMsr = 0;
1719	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1720
1721	#define ADD_MSR(a_Msr, a_uValue) do { \
1722	Assert(iMsr < 32); \
1723	uBuf.Core.entries[iMsr].index = (a_Msr); \
1724	uBuf.Core.entries[iMsr].reserved = 0; \
1725	uBuf.Core.entries[iMsr].data = (a_uValue); \
1726	iMsr += 1; \
1727	} while (0)
1728
1729	if (fExtrn & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1730	ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1731	if (fExtrn & CPUMCTX_EXTRN_SYSCALL_MSRS)
1732	{
1733	ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
1734	ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
1735	ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
1736	ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1737	}
1738	if (fExtrn & CPUMCTX_EXTRN_SYSENTER_MSRS)
1739	{
1740	ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
1741	ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
1742	ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
1743	}
1744	if (fExtrn & CPUMCTX_EXTRN_TSC_AUX)
1745	ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1746	if (fExtrn & CPUMCTX_EXTRN_OTHER_MSRS)
1747	{
1748	ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
1749	/** @todo What do we _have_ to add here?
1750	* We also have: Mttr, MiscEnable, FeatureControl. /
1751	}
1752
1753	uBuf.Core.pad = 0;
1754	uBuf.Core.nmsrs = iMsr;
1755	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_MSRS, &uBuf);
1756	AssertMsgReturn(rc == (int)iMsr,
1757	("rc=%d iMsr=%d (->%#x) errno=%d\n",
1758	rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
1759	VERR_NEM_IPE_3);
1760	}
1761
1762	/*
1763	* Interruptibility state.
1764	*
1765	* Note! This I/O control function sets most fields passed in, so when
1766	* raising an interrupt, NMI, SMI or exception, this must be done
1767	* by the code doing the rasing or we'll overwrite it here.
1768	*/
1769	if (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1770	{
1771	Assert( (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1772	== (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI));
1773
1774	struct kvm_vcpu_events KvmEvents = {0};
1775
1776	KvmEvents.flags = KVM_VCPUEVENT_VALID_SHADOW;
1777	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1778	{
1779	if (pRun->s.regs.regs.rip == EMGetInhibitInterruptsPC(pVCpu))
1780	KvmEvents.interrupt.shadow = KVM_X86_SHADOW_INT_MOV_SS \| KVM_X86_SHADOW_INT_STI;
1781	else
1782	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1783	}
1784
1785	/* No flag - this is updated unconditionally. */
1786	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
1787	KvmEvents.nmi.masked = 1;
1788
1789	if (TRPMHasTrap(pVCpu))
1790	{
1791	TRPMEVENT enmType = TRPM_32BIT_HACK;
1792	uint8_t bTrapNo = 0;
1793	TRPMQueryTrap(pVCpu, &bTrapNo, &enmType);
1794	Log(("nemHCLnxExportState: Pending trap: bTrapNo=%#x enmType=%d\n", bTrapNo, enmType));
1795	if ( enmType == TRPM_HARDWARE_INT
1796	\|\| enmType == TRPM_SOFTWARE_INT)
1797	{
1798	KvmEvents.interrupt.soft = enmType == TRPM_SOFTWARE_INT;
1799	KvmEvents.interrupt.nr = bTrapNo;
1800	KvmEvents.interrupt.injected = 1;
1801	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExportPendingInterrupt);
1802	TRPMResetTrap(pVCpu);
1803	}
1804	else
1805	AssertFailed();
1806	}
1807
1808	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
1809	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
1810	}
1811
1812	/*
1813	* KVM now owns all the state.
1814	*/
1815	pCtx->fExtrn = CPUMCTX_EXTRN_KEEPER_NEM \| CPUMCTX_EXTRN_ALL;
1816
1817	RT_NOREF(pVM);
1818	return VINF_SUCCESS;
1819	}
1820
1821
1822	/**
1823	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
1824	*
1825	* @returns VBox status code.
1826	* @param pVCpu The cross context CPU structure.
1827	* @param pcTicks Where to return the CPU tick count.
1828	* @param puAux Where to return the TSC_AUX register value.
1829	*/
1830	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
1831	{
1832	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
1833	// KVM_GET_CLOCK?
1834	RT_NOREF(pVCpu, pcTicks, puAux);
1835	return VINF_SUCCESS;
1836	}
1837
1838
1839	/**
1840	* Resumes CPU clock (TSC) on all virtual CPUs.
1841	*
1842	* This is called by TM when the VM is started, restored, resumed or similar.
1843	*
1844	* @returns VBox status code.
1845	* @param pVM The cross context VM structure.
1846	* @param pVCpu The cross context CPU structure of the calling EMT.
1847	* @param uPausedTscValue The TSC value at the time of pausing.
1848	*/
1849	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
1850	{
1851	// KVM_SET_CLOCK?
1852	RT_NOREF(pVM, pVCpu, uPausedTscValue);
1853	return VINF_SUCCESS;
1854	}
1855
1856
1857	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
1858	{
1859	RT_NOREF(pVM);
1860	return NEM_FEAT_F_NESTED_PAGING
1861	\| NEM_FEAT_F_FULL_GST_EXEC
1862	\| NEM_FEAT_F_XSAVE_XRSTOR;
1863	}
1864
1865
1866
1867	/*********************************************************************************************************************************
1868	* Execution *
1869	*********************************************************************************************************************************/
1870
1871
1872	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
1873	{
1874	/*
1875	* Only execute when the A20 gate is enabled as I cannot immediately
1876	* spot any A20 support in KVM.
1877	*/
1878	RT_NOREF(pVM);
1879	Assert(VM_IS_NEM_ENABLED(pVM));
1880	return PGMPhysIsA20Enabled(pVCpu);
1881	}
1882
1883
1884	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
1885	{
1886	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
1887	return false;
1888	}
1889
1890
1891	/**
1892	* Forced flag notification call from VMEmt.cpp.
1893	*
1894	* This is only called when pVCpu is in the VMCPUSTATE_STARTED_EXEC_NEM state.
1895	*
1896	* @param pVM The cross context VM structure.
1897	* @param pVCpu The cross context virtual CPU structure of the CPU
1898	* to be notified.
1899	* @param fFlags Notification flags, VMNOTIFYFF_FLAGS_XXX.
1900	*/
1901	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
1902	{
1903	int rc = RTThreadPoke(pVCpu->hThread);
1904	LogFlow(("nemR3NativeNotifyFF: #%u -> %Rrc\n", pVCpu->idCpu, rc));
1905	AssertRC(rc);
1906	RT_NOREF(pVM, fFlags);
1907	}
1908
1909
1910	/**
1911	* Deals with pending interrupt FFs prior to executing guest code.
1912	*/
1913	static VBOXSTRICTRC nemHCLnxHandleInterruptFF(PVM pVM, PVMCPU pVCpu, struct kvm_run *pRun)
1914	{
1915	RT_NOREF_PV(pVM);
1916
1917	/*
1918	* Do not doing anything if TRPM has something pending already as we can
1919	* only inject one event per KVM_RUN call. This can only happend if we
1920	* can directly from the loop in EM, so the inhibit bits must be internal.
1921	*/
1922	if (!TRPMHasTrap(pVCpu))
1923	{ /* semi likely */ }
1924	else
1925	{
1926	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI)));
1927	Log8(("nemHCLnxHandleInterruptFF: TRPM has an pending event already\n"));
1928	return VINF_SUCCESS;
1929	}
1930
1931	/*
1932	* First update APIC. We ASSUME this won't need TPR/CR8.
1933	*/
1934	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
1935	{
1936	APICUpdatePendingInterrupts(pVCpu);
1937	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
1938	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
1939	return VINF_SUCCESS;
1940	}
1941
1942	/*
1943	* We don't currently implement SMIs.
1944	*/
1945	AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
1946
1947	/*
1948	* In KVM the CPUMCTX_EXTRN_INHIBIT_INT and CPUMCTX_EXTRN_INHIBIT_NMI states
1949	* are tied together with interrupt and NMI delivery, so we must get and
1950	* synchronize these all in one go and set both CPUMCTX_EXTRN_INHIBIT_XXX flags.
1951	* If we don't we may lose the interrupt/NMI we marked pending here when the
1952	* state is exported again before execution.
1953	*/
1954	struct kvm_vcpu_events KvmEvents = {0};
1955	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
1956	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
1957
1958	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
1959	pRun->s.regs.regs.rip = pVCpu->cpum.GstCtx.rip;
1960
1961	KvmEvents.flags \|= KVM_VCPUEVENT_VALID_SHADOW;
1962	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_INT))
1963	{
1964	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1965	KvmEvents.interrupt.shadow = 0;
1966	else if (EMGetInhibitInterruptsPC(pVCpu) == pRun->s.regs.regs.rip)
1967	KvmEvents.interrupt.shadow = KVM_X86_SHADOW_INT_MOV_SS \| KVM_X86_SHADOW_INT_STI;
1968	else
1969	{
1970	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1971	KvmEvents.interrupt.shadow = 0;
1972	}
1973	}
1974	else if (KvmEvents.interrupt.shadow)
1975	EMSetInhibitInterruptsPC(pVCpu, pRun->s.regs.regs.rip);
1976	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1977	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1978
1979	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_NMI))
1980	KvmEvents.nmi.masked = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS) ? 1 : 0;
1981	else if (KvmEvents.nmi.masked)
1982	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
1983	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
1984	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
1985
1986	/* KVM will own the INT + NMI inhibit state soon: */
1987	pVCpu->cpum.GstCtx.fExtrn = (pVCpu->cpum.GstCtx.fExtrn & ~CPUMCTX_EXTRN_KEEPER_MASK)
1988	\| CPUMCTX_EXTRN_KEEPER_NEM \| CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI;
1989
1990	/*
1991	* NMI? Try deliver it first.
1992	*/
1993	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1994	{
1995	#if 0
1996	int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_NMI, 0UL);
1997	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
1998	#else
1999	KvmEvents.flags \|= KVM_VCPUEVENT_VALID_NMI_PENDING;
2000	KvmEvents.nmi.pending = 1;
2001	#endif
2002	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
2003	Log8(("Queuing NMI on %u\n", pVCpu->idCpu));
2004	}
2005
2006	/*
2007	* APIC or PIC interrupt?
2008	*/
2009	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
2010	{
2011	if (pRun->s.regs.regs.rflags & X86_EFL_IF)
2012	{
2013	if (KvmEvents.interrupt.shadow == 0)
2014	{
2015	/*
2016	* If CR8 is in KVM, update the VBox copy so PDMGetInterrupt will
2017	* work correctly.
2018	*/
2019	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_APIC_TPR)
2020	APICSetTpr(pVCpu, (uint8_t)pRun->cr8 << 4);
2021
2022	uint8_t bInterrupt;
2023	int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
2024	if (RT_SUCCESS(rc))
2025	{
2026	Assert(KvmEvents.interrupt.injected == false);
2027	#if 0
2028	int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_INTERRUPT, (unsigned long)bInterrupt);
2029	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2030	#else
2031	KvmEvents.interrupt.nr = bInterrupt;
2032	KvmEvents.interrupt.soft = false;
2033	KvmEvents.interrupt.injected = true;
2034	#endif
2035	Log8(("Queuing interrupt %#x on %u: %04x:%08RX64 efl=%#x\n", bInterrupt, pVCpu->idCpu,
2036	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags));
2037	}
2038	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR) /** @todo this isn't extremely efficient if we get a lot of exits... */
2039	Log8(("VERR_APIC_INTR_MASKED_BY_TPR\n")); /* We'll get a TRP exit - no interrupt window needed. */
2040	else
2041	Log8(("PDMGetInterrupt failed -> %Rrc\n", rc));
2042	}
2043	else
2044	{
2045	pRun->request_interrupt_window = 1;
2046	Log8(("Interrupt window pending on %u (#2)\n", pVCpu->idCpu));
2047	}
2048	}
2049	else
2050	{
2051	pRun->request_interrupt_window = 1;
2052	Log8(("Interrupt window pending on %u (#1)\n", pVCpu->idCpu));
2053	}
2054	}
2055
2056	/*
2057	* Now, update the state.
2058	*/
2059	/** @todo skip when possible... */
2060	rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
2061	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2062
2063	return VINF_SUCCESS;
2064	}
2065
2066
2067	/**
2068	* Handles KVM_EXIT_INTERNAL_ERROR.
2069	*/
2070	static VBOXSTRICTRC nemR3LnxHandleInternalError(PVMCPU pVCpu, struct kvm_run *pRun)
2071	{
2072	Log(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%d) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror,
2073	pRun->internal.suberror, pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
2074
2075	/*
2076	* Deal with each suberror, returning if we don't want IEM to handle it.
2077	*/
2078	switch (pRun->internal.suberror)
2079	{
2080	case KVM_INTERNAL_ERROR_EMULATION:
2081	{
2082	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_EMULATION),
2083	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2084	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorEmulation);
2085	break;
2086	}
2087
2088	case KVM_INTERNAL_ERROR_SIMUL_EX:
2089	case KVM_INTERNAL_ERROR_DELIVERY_EV:
2090	case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON:
2091	default:
2092	{
2093	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_FATAL),
2094	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2095	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorFatal);
2096	const char *pszName;
2097	switch (pRun->internal.suberror)
2098	{
2099	case KVM_INTERNAL_ERROR_EMULATION: pszName = "KVM_INTERNAL_ERROR_EMULATION"; break;
2100	case KVM_INTERNAL_ERROR_SIMUL_EX: pszName = "KVM_INTERNAL_ERROR_SIMUL_EX"; break;
2101	case KVM_INTERNAL_ERROR_DELIVERY_EV: pszName = "KVM_INTERNAL_ERROR_DELIVERY_EV"; break;
2102	case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON: pszName = "KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON"; break;
2103	default: pszName = "unknown"; break;
2104	}
2105	LogRel(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%s) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror, pszName,
2106	pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
2107	return VERR_NEM_IPE_0;
2108	}
2109	}
2110
2111	/*
2112	* Execute instruction in IEM and try get on with it.
2113	*/
2114	Log2(("nemR3LnxHandleInternalError: Executing instruction at %04x:%08RX64 in IEM\n",
2115	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip));
2116	VBOXSTRICTRC rcStrict = nemHCLnxImportState(pVCpu,
2117	IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_INHIBIT_INT
2118	\| CPUMCTX_EXTRN_INHIBIT_NMI,
2119	&pVCpu->cpum.GstCtx, pRun);
2120	if (RT_SUCCESS(rcStrict))
2121	rcStrict = IEMExecOne(pVCpu);
2122	return rcStrict;
2123	}
2124
2125
2126	/**
2127	* Handles KVM_EXIT_IO.
2128	*/
2129	static VBOXSTRICTRC nemHCLnxHandleExitIo(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
2130	{
2131	/*
2132	* Input validation.
2133	*/
2134	Assert(pRun->io.count > 0);
2135	Assert(pRun->io.size == 1 \|\| pRun->io.size == 2 \|\| pRun->io.size == 4);
2136	Assert(pRun->io.direction == KVM_EXIT_IO_IN \|\| pRun->io.direction == KVM_EXIT_IO_OUT);
2137	Assert(pRun->io.data_offset < pVM->nem.s.cbVCpuMmap);
2138	Assert(pRun->io.data_offset + pRun->io.size * pRun->io.count <= pVM->nem.s.cbVCpuMmap);
2139
2140	/*
2141	* We cannot actually act on the exit history here, because the I/O port
2142	* exit is stateful and the instruction will be completed in the next
2143	* KVM_RUN call. There seems no way to avoid this.
2144	*/
2145	EMHistoryAddExit(pVCpu,
2146	pRun->io.count == 1
2147	? ( pRun->io.direction == KVM_EXIT_IO_IN
2148	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ)
2149	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE))
2150	: ( pRun->io.direction == KVM_EXIT_IO_IN
2151	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)
2152	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)),
2153	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2154
2155	/*
2156	* Do the requested job.
2157	*/
2158	VBOXSTRICTRC rcStrict;
2159	RTPTRUNION uPtrData;
2160	uPtrData.pu8 = (uint8_t *)pRun + pRun->io.data_offset;
2161	if (pRun->io.count == 1)
2162	{
2163	if (pRun->io.direction == KVM_EXIT_IO_IN)
2164	{
2165	uint32_t uValue = 0;
2166	rcStrict = IOMIOPortRead(pVM, pVCpu, pRun->io.port, &uValue, pRun->io.size);
2167	Log4(("IOExit/%u: %04x:%08RX64: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2168	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2169	pRun->io.port, pRun->io.size, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2170	if (IOM_SUCCESS(rcStrict))
2171	{
2172	if (pRun->io.size == 4)
2173	*uPtrData.pu32 = uValue;
2174	else if (pRun->io.size == 2)
2175	*uPtrData.pu16 = (uint16_t)uValue;
2176	else
2177	*uPtrData.pu8 = (uint8_t)uValue;
2178	}
2179	}
2180	else
2181	{
2182	uint32_t const uValue = pRun->io.size == 4 ? *uPtrData.pu32
2183	: pRun->io.size == 2 ? *uPtrData.pu16
2184	: *uPtrData.pu8;
2185	rcStrict = IOMIOPortWrite(pVM, pVCpu, pRun->io.port, uValue, pRun->io.size);
2186	Log4(("IOExit/%u: %04x:%08RX64: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2187	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2188	pRun->io.port, uValue, pRun->io.size, VBOXSTRICTRC_VAL(rcStrict) ));
2189	}
2190	}
2191	else
2192	{
2193	uint32_t cTransfers = pRun->io.count;
2194	if (pRun->io.direction == KVM_EXIT_IO_IN)
2195	{
2196	rcStrict = IOMIOPortReadString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
2197	Log4(("IOExit/%u: %04x:%08RX64: REP INS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
2198	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2199	pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
2200	}
2201	else
2202	{
2203	rcStrict = IOMIOPortWriteString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
2204	Log4(("IOExit/%u: %04x:%08RX64: REP OUTS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
2205	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2206	pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
2207	}
2208	Assert(cTransfers == 0);
2209	}
2210	return rcStrict;
2211	}
2212
2213
2214	/**
2215	* Handles KVM_EXIT_MMIO.
2216	*/
2217	static VBOXSTRICTRC nemHCLnxHandleExitMmio(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
2218	{
2219	/*
2220	* Input validation.
2221	*/
2222	Assert(pRun->mmio.len <= sizeof(pRun->mmio.data));
2223	Assert(pRun->mmio.is_write <= 1);
2224
2225	/*
2226	* We cannot actually act on the exit history here, because the MMIO port
2227	* exit is stateful and the instruction will be completed in the next
2228	* KVM_RUN call. There seems no way to circumvent this.
2229	*/
2230	EMHistoryAddExit(pVCpu,
2231	pRun->mmio.is_write
2232	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2233	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2234	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2235
2236	/*
2237	* Do the requested job.
2238	*/
2239	VBOXSTRICTRC rcStrict;
2240	if (pRun->mmio.is_write)
2241	{
2242	rcStrict = PGMPhysWrite(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
2243	Log4(("MmioExit/%u: %04x:%08RX64: WRITE %#x LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
2244	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2245	pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
2246	}
2247	else
2248	{
2249	rcStrict = PGMPhysRead(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
2250	Log4(("MmioExit/%u: %04x:%08RX64: READ %#x LB %u -> %.*Rhxs rcStrict=%Rrc\n",
2251	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2252	pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
2253	}
2254	return rcStrict;
2255	}
2256
2257
2258	static VBOXSTRICTRC nemHCLnxHandleExit(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run pRun, bool pfStatefulExit)
2259	{
2260	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTotal);
2261	switch (pRun->exit_reason)
2262	{
2263	case KVM_EXIT_EXCEPTION:
2264	AssertFailed();
2265	break;
2266
2267	case KVM_EXIT_IO:
2268	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIo);
2269	*pfStatefulExit = true;
2270	return nemHCLnxHandleExitIo(pVM, pVCpu, pRun);
2271
2272	case KVM_EXIT_MMIO:
2273	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMmio);
2274	*pfStatefulExit = true;
2275	return nemHCLnxHandleExitMmio(pVM, pVCpu, pRun);
2276
2277	case KVM_EXIT_IRQ_WINDOW_OPEN:
2278	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2279	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2280	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIrqWindowOpen);
2281	Log5(("IrqWinOpen/%u: %d\n", pVCpu->idCpu, pRun->request_interrupt_window));
2282	pRun->request_interrupt_window = 0;
2283	return VINF_SUCCESS;
2284
2285	case KVM_EXIT_SET_TPR:
2286	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitSetTpr);
2287	AssertFailed();
2288	break;
2289
2290	case KVM_EXIT_TPR_ACCESS:
2291	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTprAccess);
2292	AssertFailed();
2293	break;
2294
2295	case KVM_EXIT_X86_RDMSR:
2296	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitRdMsr);
2297	AssertFailed();
2298	break;
2299
2300	case KVM_EXIT_X86_WRMSR:
2301	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitWrMsr);
2302	AssertFailed();
2303	break;
2304
2305	case KVM_EXIT_HLT:
2306	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
2307	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2308	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
2309	Log5(("Halt/%u\n", pVCpu->idCpu));
2310	return VINF_EM_HALT;
2311
2312	case KVM_EXIT_INTR: /* EINTR */
2313	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERRUPTED),
2314	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2315	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIntr);
2316	Log5(("Intr/%u\n", pVCpu->idCpu));
2317	return VINF_SUCCESS;
2318
2319	case KVM_EXIT_HYPERCALL:
2320	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHypercall);
2321	AssertFailed();
2322	break;
2323
2324	case KVM_EXIT_DEBUG:
2325	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitDebug);
2326	AssertFailed();
2327	break;
2328
2329	case KVM_EXIT_SYSTEM_EVENT:
2330	AssertFailed();
2331	break;
2332	case KVM_EXIT_IOAPIC_EOI:
2333	AssertFailed();
2334	break;
2335	case KVM_EXIT_HYPERV:
2336	AssertFailed();
2337	break;
2338
2339	case KVM_EXIT_DIRTY_RING_FULL:
2340	AssertFailed();
2341	break;
2342	case KVM_EXIT_AP_RESET_HOLD:
2343	AssertFailed();
2344	break;
2345	case KVM_EXIT_X86_BUS_LOCK:
2346	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitBusLock);
2347	AssertFailed();
2348	break;
2349
2350
2351	case KVM_EXIT_SHUTDOWN:
2352	AssertFailed();
2353	break;
2354
2355	case KVM_EXIT_FAIL_ENTRY:
2356	LogRel(("NEM: KVM_EXIT_FAIL_ENTRY! hardware_entry_failure_reason=%#x cpu=%#x\n",
2357	pRun->fail_entry.hardware_entry_failure_reason, pRun->fail_entry.cpu));
2358	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_FAILED_ENTRY),
2359	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2360	return VERR_NEM_IPE_1;
2361
2362	case KVM_EXIT_INTERNAL_ERROR:
2363	/* we're counting sub-reasons inside the function. */
2364	return nemR3LnxHandleInternalError(pVCpu, pRun);
2365
2366	/*
2367	* Foreign and unknowns.
2368	*/
2369	case KVM_EXIT_NMI:
2370	AssertLogRelMsgFailedReturn(("KVM_EXIT_NMI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2371	case KVM_EXIT_EPR:
2372	AssertLogRelMsgFailedReturn(("KVM_EXIT_EPR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2373	case KVM_EXIT_WATCHDOG:
2374	AssertLogRelMsgFailedReturn(("KVM_EXIT_WATCHDOG on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2375	case KVM_EXIT_ARM_NISV:
2376	AssertLogRelMsgFailedReturn(("KVM_EXIT_ARM_NISV on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2377	case KVM_EXIT_S390_STSI:
2378	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_STSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2379	case KVM_EXIT_S390_TSCH:
2380	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_TSCH on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2381	case KVM_EXIT_OSI:
2382	AssertLogRelMsgFailedReturn(("KVM_EXIT_OSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2383	case KVM_EXIT_PAPR_HCALL:
2384	AssertLogRelMsgFailedReturn(("KVM_EXIT_PAPR_HCALL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2385	case KVM_EXIT_S390_UCONTROL:
2386	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_UCONTROL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2387	case KVM_EXIT_DCR:
2388	AssertLogRelMsgFailedReturn(("KVM_EXIT_DCR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2389	case KVM_EXIT_S390_SIEIC:
2390	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_SIEIC on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2391	case KVM_EXIT_S390_RESET:
2392	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_RESET on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2393	case KVM_EXIT_UNKNOWN:
2394	AssertLogRelMsgFailedReturn(("KVM_EXIT_UNKNOWN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2395	case KVM_EXIT_XEN:
2396	AssertLogRelMsgFailedReturn(("KVM_EXIT_XEN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2397	default:
2398	AssertLogRelMsgFailedReturn(("Unknown exit reason %u on VCpu #%u at %04x:%RX64!\n", pRun->exit_reason, pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2399	}
2400
2401	RT_NOREF(pVM, pVCpu, pRun);
2402	return VERR_NOT_IMPLEMENTED;
2403	}
2404
2405
2406	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2407	{
2408	/*
2409	* Try switch to NEM runloop state.
2410	*/
2411	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2412	{ /* likely */ }
2413	else
2414	{
2415	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2416	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
2417	return VINF_SUCCESS;
2418	}
2419
2420	/*
2421	* The run loop.
2422	*/
2423	struct kvm_run * const pRun = pVCpu->nem.s.pRun;
2424	const bool fSingleStepping = DBGFIsStepping(pVCpu);
2425	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2426	bool fStatefulExit = false; /* For MMIO and IO exits. */
2427	for (unsigned iLoop = 0;; iLoop++)
2428	{
2429	/*
2430	* Pending interrupts or such? Need to check and deal with this prior
2431	* to the state syncing.
2432	*/
2433	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_UPDATE_APIC \| VMCPU_FF_INTERRUPT_PIC
2434	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2435	{
2436	/* Try inject interrupt. */
2437	rcStrict = nemHCLnxHandleInterruptFF(pVM, pVCpu, pRun);
2438	if (rcStrict == VINF_SUCCESS)
2439	{ /* likely */ }
2440	else
2441	{
2442	LogFlow(("NEM/%u: breaking: nemHCLnxHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2443	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2444	break;
2445	}
2446	}
2447
2448	/*
2449	* Do not execute in KVM if the A20 isn't enabled.
2450	*/
2451	if (PGMPhysIsA20Enabled(pVCpu))
2452	{ /* likely */ }
2453	else
2454	{
2455	rcStrict = VINF_EM_RESCHEDULE_REM;
2456	LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
2457	break;
2458	}
2459
2460	/*
2461	* Ensure KVM has the whole state.
2462	*/
2463	if ((pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL)
2464	{
2465	int rc2 = nemHCLnxExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx, pRun);
2466	AssertRCReturn(rc2, rc2);
2467	}
2468
2469	/*
2470	* Poll timers and run for a bit.
2471	*
2472	* With the VID approach (ring-0 or ring-3) we can specify a timeout here,
2473	* so we take the time of the next timer event and uses that as a deadline.
2474	* The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
2475	*/
2476	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
2477	* the whole polling job when timers have changed... */
2478	uint64_t offDeltaIgnored;
2479	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
2480	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
2481	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2482	{
2483	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
2484	{
2485	LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
2486	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2487	!!(pRun->s.regs.regs.rflags & X86_EFL_IF), pRun->s.regs.regs.rflags,
2488	pRun->s.regs.sregs.ss.selector, pRun->s.regs.regs.rsp, pRun->s.regs.sregs.cr0));
2489	TMNotifyStartOfExecution(pVM, pVCpu);
2490
2491	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
2492
2493	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
2494	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
2495
2496	#ifdef LOG_ENABLED
2497	if (LogIsFlowEnabled())
2498	{
2499	struct kvm_mp_state MpState = {UINT32_MAX};
2500	ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MP_STATE, &MpState);
2501	LogFlow(("NEM/%u: Exit @ %04x:%08RX64 IF=%d EFL=%#RX64 CR8=%#x Reason=%#x IrqReady=%d Flags=%#x %#lx\n", pVCpu->idCpu,
2502	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->if_flag,
2503	pRun->s.regs.regs.rflags, pRun->s.regs.sregs.cr8, pRun->exit_reason,
2504	pRun->ready_for_interrupt_injection, pRun->flags, MpState.mp_state));
2505	}
2506	#endif
2507	fStatefulExit = false;
2508	if (RT_LIKELY(rcLnx == 0 \|\| errno == EINTR))
2509	{
2510	/*
2511	* Deal with the exit.
2512	*/
2513	rcStrict = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
2514	if (rcStrict == VINF_SUCCESS)
2515	{ /* hopefully likely */ }
2516	else
2517	{
2518	LogFlow(("NEM/%u: breaking: nemHCLnxHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2519	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2520	break;
2521	}
2522	}
2523	else
2524	{
2525	int rc2 = RTErrConvertFromErrno(errno);
2526	AssertLogRelMsgFailedReturn(("KVM_RUN failed: rcLnx=%d errno=%u rc=%Rrc\n", rcLnx, errno, rc2), rc2);
2527	}
2528
2529	/*
2530	* If no relevant FFs are pending, loop.
2531	*/
2532	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2533	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2534	{ /* likely */ }
2535	else
2536	{
2537
2538	/** @todo Try handle pending flags, not just return to EM loops. Take care
2539	* not to set important RCs here unless we've handled an exit. */
2540	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
2541	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
2542	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
2543	break;
2544	}
2545	}
2546	else
2547	{
2548	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
2549	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
2550	break;
2551	}
2552	}
2553	else
2554	{
2555	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
2556	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
2557	break;
2558	}
2559	} /* the run loop */
2560
2561
2562	/*
2563	* If the last exit was stateful, commit the state we provided before
2564	* returning to the EM loop so we have a consistent state and can safely
2565	* be rescheduled and whatnot. This may require us to make multiple runs
2566	* for larger MMIO and I/O operations. Sigh^3.
2567	*
2568	* Note! There is no 'ing way to reset the kernel side completion callback
2569	* for these stateful i/o exits. Very annoying interface.
2570	*/
2571	/** @todo check how this works with string I/O and string MMIO. */
2572	if (fStatefulExit && RT_SUCCESS(rcStrict))
2573	{
2574	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn);
2575	uint32_t const uOrgExit = pRun->exit_reason;
2576	for (uint32_t i = 0; ; i++)
2577	{
2578	pRun->immediate_exit = 1;
2579	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
2580	Log(("NEM/%u: Flushed stateful exit -> %d/%d exit_reason=%d\n", pVCpu->idCpu, rcLnx, errno, pRun->exit_reason));
2581	if (rcLnx == -1 && errno == EINTR)
2582	{
2583	switch (i)
2584	{
2585	case 0: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn1Loop); break;
2586	case 1: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn2Loops); break;
2587	case 2: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn3Loops); break;
2588	default: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn4PlusLoops); break;
2589	}
2590	break;
2591	}
2592	AssertLogRelMsgBreakStmt(rcLnx == 0 && pRun->exit_reason == uOrgExit,
2593	("rcLnx=%d errno=%d exit_reason=%d uOrgExit=%d\n", rcLnx, errno, pRun->exit_reason, uOrgExit),
2594	rcStrict = VERR_NEM_IPE_6);
2595	VBOXSTRICTRC rcStrict2 = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
2596	if (rcStrict2 == VINF_SUCCESS \|\| rcStrict2 == rcStrict)
2597	{ /* likely */ }
2598	else if (RT_FAILURE(rcStrict2))
2599	{
2600	rcStrict = rcStrict2;
2601	break;
2602	}
2603	else
2604	{
2605	AssertLogRelMsgBreakStmt(rcStrict == VINF_SUCCESS,
2606	("rcStrict=%Rrc rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2)),
2607	rcStrict = VERR_NEM_IPE_7);
2608	rcStrict = rcStrict2;
2609	}
2610	}
2611	pRun->immediate_exit = 0;
2612	}
2613
2614	/*
2615	* If the CPU is running, make sure to stop it before we try sync back the
2616	* state and return to EM. We don't sync back the whole state if we can help it.
2617	*/
2618	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
2619	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2620
2621	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
2622	{
2623	/* Try anticipate what we might need. */
2624	uint64_t fImport = CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI /* Required for processing APIC,PIC,NMI & SMI FFs. */
2625	\| IEM_CPUMCTX_EXTRN_MUST_MASK /?/;
2626	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
2627	\|\| RT_FAILURE(rcStrict))
2628	fImport = CPUMCTX_EXTRN_ALL;
2629	# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
2630	else if ( rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
2631	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
2632	fImport = CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS;
2633	else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
2634	fImport = CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS;
2635	# endif
2636	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
2637	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2638	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
2639
2640	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
2641	{
2642	int rc2 = nemHCLnxImportState(pVCpu, fImport, &pVCpu->cpum.GstCtx, pRun);
2643	if (RT_SUCCESS(rc2))
2644	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
2645	else if (RT_SUCCESS(rcStrict))
2646	rcStrict = rc2;
2647	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
2648	pVCpu->cpum.GstCtx.fExtrn = 0;
2649	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
2650	}
2651	else
2652	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2653	}
2654	else
2655	{
2656	pVCpu->cpum.GstCtx.fExtrn = 0;
2657	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2658	}
2659
2660	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
2661	pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
2662	return rcStrict;
2663	}
2664
2665
2666	/** @page pg_nem_linux NEM/linux - Native Execution Manager, Linux.
2667	*
2668	* This is using KVM.
2669	*
2670	*/
2671

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-linux.cpp@ 92556

Download in other formats: