NEMR3Native-linux.cpp@ 96789

Last change on this file since 96789 was 96407, checked in by vboxsync, 2 years ago
scm copyright and license note update
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 117.1 KB

Line
1	/* $Id: NEMR3Native-linux.cpp 96407 2022-08-22 17:43:14Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 Linux backend.
4	*/
5
6	/*
7	* Copyright (C) 2021-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_NEM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <VBox/vmm/nem.h>
35	#include <VBox/vmm/iem.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/apic.h>
38	#include <VBox/vmm/pdm.h>
39	#include <VBox/vmm/trpm.h>
40	#include "NEMInternal.h"
41	#include <VBox/vmm/vmcc.h>
42
43	#include <iprt/alloca.h>
44	#include <iprt/string.h>
45	#include <iprt/system.h>
46	#include <iprt/x86.h>
47
48	#include <errno.h>
49	#include <unistd.h>
50	#include <sys/ioctl.h>
51	#include <sys/fcntl.h>
52	#include <sys/mman.h>
53	#include <linux/kvm.h>
54
55	/*
56	* Supply stuff missing from the kvm.h on the build box.
57	*/
58	#ifndef KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON /* since 5.4 */
59	# define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4
60	#endif
61
62
63
64	/**
65	* Worker for nemR3NativeInit that gets the hypervisor capabilities.
66	*
67	* @returns VBox status code.
68	* @param pVM The cross context VM structure.
69	* @param pErrInfo Where to always return error info.
70	*/
71	static int nemR3LnxInitCheckCapabilities(PVM pVM, PRTERRINFO pErrInfo)
72	{
73	AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
74
75	/*
76	* Capabilities.
77	*/
78	static const struct
79	{
80	const char *pszName;
81	int iCap;
82	uint32_t offNem : 24;
83	uint32_t cbNem : 3;
84	uint32_t fReqNonZero : 1;
85	uint32_t uReserved : 4;
86	} s_aCaps[] =
87	{
88	#define CAP_ENTRY__L(a_Define) { #a_Define, a_Define, UINT32_C(0x00ffffff), 0, 0, 0 }
89	#define CAP_ENTRY__S(a_Define, a_Member) { #a_Define, a_Define, RT_UOFFSETOF(NEM, a_Member), RT_SIZEOFMEMB(NEM, a_Member), 0, 0 }
90	#define CAP_ENTRY_MS(a_Define, a_Member) { #a_Define, a_Define, RT_UOFFSETOF(NEM, a_Member), RT_SIZEOFMEMB(NEM, a_Member), 1, 0 }
91	#define CAP_ENTRY__U(a_Number) { "KVM_CAP_" #a_Number, a_Number, UINT32_C(0x00ffffff), 0, 0, 0 }
92	#define CAP_ENTRY_ML(a_Number) { "KVM_CAP_" #a_Number, a_Number, UINT32_C(0x00ffffff), 0, 1, 0 }
93
94	CAP_ENTRY__L(KVM_CAP_IRQCHIP), /* 0 */
95	CAP_ENTRY_ML(KVM_CAP_HLT),
96	CAP_ENTRY__L(KVM_CAP_MMU_SHADOW_CACHE_CONTROL),
97	CAP_ENTRY_ML(KVM_CAP_USER_MEMORY),
98	CAP_ENTRY__L(KVM_CAP_SET_TSS_ADDR),
99	CAP_ENTRY__U(5),
100	CAP_ENTRY__L(KVM_CAP_VAPIC),
101	CAP_ENTRY__L(KVM_CAP_EXT_CPUID),
102	CAP_ENTRY__L(KVM_CAP_CLOCKSOURCE),
103	CAP_ENTRY__L(KVM_CAP_NR_VCPUS),
104	CAP_ENTRY_MS(KVM_CAP_NR_MEMSLOTS, cMaxMemSlots), /* 10 */
105	CAP_ENTRY__L(KVM_CAP_PIT),
106	CAP_ENTRY__L(KVM_CAP_NOP_IO_DELAY),
107	CAP_ENTRY__L(KVM_CAP_PV_MMU),
108	CAP_ENTRY__L(KVM_CAP_MP_STATE),
109	CAP_ENTRY__L(KVM_CAP_COALESCED_MMIO),
110	CAP_ENTRY__L(KVM_CAP_SYNC_MMU),
111	CAP_ENTRY__U(17),
112	CAP_ENTRY__L(KVM_CAP_IOMMU),
113	CAP_ENTRY__U(19), /* Buggy KVM_CAP_JOIN_MEMORY_REGIONS? */
114	CAP_ENTRY__U(20), /* Mon-working KVM_CAP_DESTROY_MEMORY_REGION? */
115	CAP_ENTRY__L(KVM_CAP_DESTROY_MEMORY_REGION_WORKS), /* 21 */
116	CAP_ENTRY__L(KVM_CAP_USER_NMI),
117	#ifdef __KVM_HAVE_GUEST_DEBUG
118	CAP_ENTRY__L(KVM_CAP_SET_GUEST_DEBUG),
119	#endif
120	#ifdef __KVM_HAVE_PIT
121	CAP_ENTRY__L(KVM_CAP_REINJECT_CONTROL),
122	#endif
123	CAP_ENTRY__L(KVM_CAP_IRQ_ROUTING),
124	CAP_ENTRY__L(KVM_CAP_IRQ_INJECT_STATUS),
125	CAP_ENTRY__U(27),
126	CAP_ENTRY__U(28),
127	CAP_ENTRY__L(KVM_CAP_ASSIGN_DEV_IRQ),
128	CAP_ENTRY__L(KVM_CAP_JOIN_MEMORY_REGIONS_WORKS), /* 30 */
129	#ifdef __KVM_HAVE_MCE
130	CAP_ENTRY__L(KVM_CAP_MCE),
131	#endif
132	CAP_ENTRY__L(KVM_CAP_IRQFD),
133	#ifdef __KVM_HAVE_PIT
134	CAP_ENTRY__L(KVM_CAP_PIT2),
135	#endif
136	CAP_ENTRY__L(KVM_CAP_SET_BOOT_CPU_ID),
137	#ifdef __KVM_HAVE_PIT_STATE2
138	CAP_ENTRY__L(KVM_CAP_PIT_STATE2),
139	#endif
140	CAP_ENTRY__L(KVM_CAP_IOEVENTFD),
141	CAP_ENTRY__L(KVM_CAP_SET_IDENTITY_MAP_ADDR),
142	#ifdef __KVM_HAVE_XEN_HVM
143	CAP_ENTRY__L(KVM_CAP_XEN_HVM),
144	#endif
145	CAP_ENTRY_ML(KVM_CAP_ADJUST_CLOCK),
146	CAP_ENTRY__L(KVM_CAP_INTERNAL_ERROR_DATA), /* 40 */
147	#ifdef __KVM_HAVE_VCPU_EVENTS
148	CAP_ENTRY_ML(KVM_CAP_VCPU_EVENTS),
149	#else
150	CAP_ENTRY_MU(41),
151	#endif
152	CAP_ENTRY__L(KVM_CAP_S390_PSW),
153	CAP_ENTRY__L(KVM_CAP_PPC_SEGSTATE),
154	CAP_ENTRY__L(KVM_CAP_HYPERV),
155	CAP_ENTRY__L(KVM_CAP_HYPERV_VAPIC),
156	CAP_ENTRY__L(KVM_CAP_HYPERV_SPIN),
157	CAP_ENTRY__L(KVM_CAP_PCI_SEGMENT),
158	CAP_ENTRY__L(KVM_CAP_PPC_PAIRED_SINGLES),
159	CAP_ENTRY__L(KVM_CAP_INTR_SHADOW),
160	#ifdef __KVM_HAVE_DEBUGREGS
161	CAP_ENTRY__L(KVM_CAP_DEBUGREGS), /* 50 */
162	#endif
163	CAP_ENTRY__S(KVM_CAP_X86_ROBUST_SINGLESTEP, fRobustSingleStep),
164	CAP_ENTRY__L(KVM_CAP_PPC_OSI),
165	CAP_ENTRY__L(KVM_CAP_PPC_UNSET_IRQ),
166	CAP_ENTRY__L(KVM_CAP_ENABLE_CAP),
167	#ifdef __KVM_HAVE_XSAVE
168	CAP_ENTRY_ML(KVM_CAP_XSAVE),
169	#else
170	CAP_ENTRY_MU(55),
171	#endif
172	#ifdef __KVM_HAVE_XCRS
173	CAP_ENTRY_ML(KVM_CAP_XCRS),
174	#else
175	CAP_ENTRY_MU(56),
176	#endif
177	CAP_ENTRY__L(KVM_CAP_PPC_GET_PVINFO),
178	CAP_ENTRY__L(KVM_CAP_PPC_IRQ_LEVEL),
179	CAP_ENTRY__L(KVM_CAP_ASYNC_PF),
180	CAP_ENTRY__L(KVM_CAP_TSC_CONTROL), /* 60 */
181	CAP_ENTRY__L(KVM_CAP_GET_TSC_KHZ),
182	CAP_ENTRY__L(KVM_CAP_PPC_BOOKE_SREGS),
183	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE),
184	CAP_ENTRY__L(KVM_CAP_PPC_SMT),
185	CAP_ENTRY__L(KVM_CAP_PPC_RMA),
186	CAP_ENTRY__L(KVM_CAP_MAX_VCPUS),
187	CAP_ENTRY__L(KVM_CAP_PPC_HIOR),
188	CAP_ENTRY__L(KVM_CAP_PPC_PAPR),
189	CAP_ENTRY__L(KVM_CAP_SW_TLB),
190	CAP_ENTRY__L(KVM_CAP_ONE_REG), /* 70 */
191	CAP_ENTRY__L(KVM_CAP_S390_GMAP),
192	CAP_ENTRY__L(KVM_CAP_TSC_DEADLINE_TIMER),
193	CAP_ENTRY__L(KVM_CAP_S390_UCONTROL),
194	CAP_ENTRY__L(KVM_CAP_SYNC_REGS),
195	CAP_ENTRY__L(KVM_CAP_PCI_2_3),
196	CAP_ENTRY__L(KVM_CAP_KVMCLOCK_CTRL),
197	CAP_ENTRY__L(KVM_CAP_SIGNAL_MSI),
198	CAP_ENTRY__L(KVM_CAP_PPC_GET_SMMU_INFO),
199	CAP_ENTRY__L(KVM_CAP_S390_COW),
200	CAP_ENTRY__L(KVM_CAP_PPC_ALLOC_HTAB), /* 80 */
201	CAP_ENTRY__L(KVM_CAP_READONLY_MEM),
202	CAP_ENTRY__L(KVM_CAP_IRQFD_RESAMPLE),
203	CAP_ENTRY__L(KVM_CAP_PPC_BOOKE_WATCHDOG),
204	CAP_ENTRY__L(KVM_CAP_PPC_HTAB_FD),
205	CAP_ENTRY__L(KVM_CAP_S390_CSS_SUPPORT),
206	CAP_ENTRY__L(KVM_CAP_PPC_EPR),
207	CAP_ENTRY__L(KVM_CAP_ARM_PSCI),
208	CAP_ENTRY__L(KVM_CAP_ARM_SET_DEVICE_ADDR),
209	CAP_ENTRY__L(KVM_CAP_DEVICE_CTRL),
210	CAP_ENTRY__L(KVM_CAP_IRQ_MPIC), /* 90 */
211	CAP_ENTRY__L(KVM_CAP_PPC_RTAS),
212	CAP_ENTRY__L(KVM_CAP_IRQ_XICS),
213	CAP_ENTRY__L(KVM_CAP_ARM_EL1_32BIT),
214	CAP_ENTRY__L(KVM_CAP_SPAPR_MULTITCE),
215	CAP_ENTRY__L(KVM_CAP_EXT_EMUL_CPUID),
216	CAP_ENTRY__L(KVM_CAP_HYPERV_TIME),
217	CAP_ENTRY__L(KVM_CAP_IOAPIC_POLARITY_IGNORED),
218	CAP_ENTRY__L(KVM_CAP_ENABLE_CAP_VM),
219	CAP_ENTRY__L(KVM_CAP_S390_IRQCHIP),
220	CAP_ENTRY__L(KVM_CAP_IOEVENTFD_NO_LENGTH), /* 100 */
221	CAP_ENTRY__L(KVM_CAP_VM_ATTRIBUTES),
222	CAP_ENTRY__L(KVM_CAP_ARM_PSCI_0_2),
223	CAP_ENTRY__L(KVM_CAP_PPC_FIXUP_HCALL),
224	CAP_ENTRY__L(KVM_CAP_PPC_ENABLE_HCALL),
225	CAP_ENTRY__L(KVM_CAP_CHECK_EXTENSION_VM),
226	CAP_ENTRY__L(KVM_CAP_S390_USER_SIGP),
227	CAP_ENTRY__L(KVM_CAP_S390_VECTOR_REGISTERS),
228	CAP_ENTRY__L(KVM_CAP_S390_MEM_OP),
229	CAP_ENTRY__L(KVM_CAP_S390_USER_STSI),
230	CAP_ENTRY__L(KVM_CAP_S390_SKEYS), /* 110 */
231	CAP_ENTRY__L(KVM_CAP_MIPS_FPU),
232	CAP_ENTRY__L(KVM_CAP_MIPS_MSA),
233	CAP_ENTRY__L(KVM_CAP_S390_INJECT_IRQ),
234	CAP_ENTRY__L(KVM_CAP_S390_IRQ_STATE),
235	CAP_ENTRY__L(KVM_CAP_PPC_HWRNG),
236	CAP_ENTRY__L(KVM_CAP_DISABLE_QUIRKS),
237	CAP_ENTRY__L(KVM_CAP_X86_SMM),
238	CAP_ENTRY__L(KVM_CAP_MULTI_ADDRESS_SPACE),
239	CAP_ENTRY__L(KVM_CAP_GUEST_DEBUG_HW_BPS),
240	CAP_ENTRY__L(KVM_CAP_GUEST_DEBUG_HW_WPS), /* 120 */
241	CAP_ENTRY__L(KVM_CAP_SPLIT_IRQCHIP),
242	CAP_ENTRY__L(KVM_CAP_IOEVENTFD_ANY_LENGTH),
243	CAP_ENTRY__L(KVM_CAP_HYPERV_SYNIC),
244	CAP_ENTRY__L(KVM_CAP_S390_RI),
245	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE_64),
246	CAP_ENTRY__L(KVM_CAP_ARM_PMU_V3),
247	CAP_ENTRY__L(KVM_CAP_VCPU_ATTRIBUTES),
248	CAP_ENTRY__L(KVM_CAP_MAX_VCPU_ID),
249	CAP_ENTRY__L(KVM_CAP_X2APIC_API),
250	CAP_ENTRY__L(KVM_CAP_S390_USER_INSTR0), /* 130 */
251	CAP_ENTRY__L(KVM_CAP_MSI_DEVID),
252	CAP_ENTRY__L(KVM_CAP_PPC_HTM),
253	CAP_ENTRY__L(KVM_CAP_SPAPR_RESIZE_HPT),
254	CAP_ENTRY__L(KVM_CAP_PPC_MMU_RADIX),
255	CAP_ENTRY__L(KVM_CAP_PPC_MMU_HASH_V3),
256	CAP_ENTRY__L(KVM_CAP_IMMEDIATE_EXIT),
257	CAP_ENTRY__L(KVM_CAP_MIPS_VZ),
258	CAP_ENTRY__L(KVM_CAP_MIPS_TE),
259	CAP_ENTRY__L(KVM_CAP_MIPS_64BIT),
260	CAP_ENTRY__L(KVM_CAP_S390_GS), /* 140 */
261	CAP_ENTRY__L(KVM_CAP_S390_AIS),
262	CAP_ENTRY__L(KVM_CAP_SPAPR_TCE_VFIO),
263	CAP_ENTRY__L(KVM_CAP_X86_DISABLE_EXITS),
264	CAP_ENTRY__L(KVM_CAP_ARM_USER_IRQ),
265	CAP_ENTRY__L(KVM_CAP_S390_CMMA_MIGRATION),
266	CAP_ENTRY__L(KVM_CAP_PPC_FWNMI),
267	CAP_ENTRY__L(KVM_CAP_PPC_SMT_POSSIBLE),
268	CAP_ENTRY__L(KVM_CAP_HYPERV_SYNIC2),
269	CAP_ENTRY__L(KVM_CAP_HYPERV_VP_INDEX),
270	CAP_ENTRY__L(KVM_CAP_S390_AIS_MIGRATION), /* 150 */
271	CAP_ENTRY__L(KVM_CAP_PPC_GET_CPU_CHAR),
272	CAP_ENTRY__L(KVM_CAP_S390_BPB),
273	CAP_ENTRY__L(KVM_CAP_GET_MSR_FEATURES),
274	CAP_ENTRY__L(KVM_CAP_HYPERV_EVENTFD),
275	CAP_ENTRY__L(KVM_CAP_HYPERV_TLBFLUSH),
276	CAP_ENTRY__L(KVM_CAP_S390_HPAGE_1M),
277	CAP_ENTRY__L(KVM_CAP_NESTED_STATE),
278	CAP_ENTRY__L(KVM_CAP_ARM_INJECT_SERROR_ESR),
279	CAP_ENTRY__L(KVM_CAP_MSR_PLATFORM_INFO),
280	CAP_ENTRY__L(KVM_CAP_PPC_NESTED_HV), /* 160 */
281	CAP_ENTRY__L(KVM_CAP_HYPERV_SEND_IPI),
282	CAP_ENTRY__L(KVM_CAP_COALESCED_PIO),
283	CAP_ENTRY__L(KVM_CAP_HYPERV_ENLIGHTENED_VMCS),
284	CAP_ENTRY__L(KVM_CAP_EXCEPTION_PAYLOAD),
285	CAP_ENTRY__L(KVM_CAP_ARM_VM_IPA_SIZE),
286	CAP_ENTRY__L(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT),
287	CAP_ENTRY__L(KVM_CAP_HYPERV_CPUID),
288	CAP_ENTRY__L(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2),
289	CAP_ENTRY__L(KVM_CAP_PPC_IRQ_XIVE),
290	CAP_ENTRY__L(KVM_CAP_ARM_SVE), /* 170 */
291	CAP_ENTRY__L(KVM_CAP_ARM_PTRAUTH_ADDRESS),
292	CAP_ENTRY__L(KVM_CAP_ARM_PTRAUTH_GENERIC),
293	CAP_ENTRY__L(KVM_CAP_PMU_EVENT_FILTER),
294	CAP_ENTRY__L(KVM_CAP_ARM_IRQ_LINE_LAYOUT_2),
295	CAP_ENTRY__L(KVM_CAP_HYPERV_DIRECT_TLBFLUSH),
296	CAP_ENTRY__L(KVM_CAP_PPC_GUEST_DEBUG_SSTEP),
297	CAP_ENTRY__L(KVM_CAP_ARM_NISV_TO_USER),
298	CAP_ENTRY__L(KVM_CAP_ARM_INJECT_EXT_DABT),
299	CAP_ENTRY__L(KVM_CAP_S390_VCPU_RESETS),
300	CAP_ENTRY__L(KVM_CAP_S390_PROTECTED), /* 180 */
301	CAP_ENTRY__L(KVM_CAP_PPC_SECURE_GUEST),
302	CAP_ENTRY__L(KVM_CAP_HALT_POLL),
303	CAP_ENTRY__L(KVM_CAP_ASYNC_PF_INT),
304	CAP_ENTRY__L(KVM_CAP_LAST_CPU),
305	CAP_ENTRY__L(KVM_CAP_SMALLER_MAXPHYADDR),
306	CAP_ENTRY__L(KVM_CAP_S390_DIAG318),
307	CAP_ENTRY__L(KVM_CAP_STEAL_TIME),
308	CAP_ENTRY_ML(KVM_CAP_X86_USER_SPACE_MSR), /* (since 5.10) */
309	CAP_ENTRY_ML(KVM_CAP_X86_MSR_FILTER),
310	CAP_ENTRY__L(KVM_CAP_ENFORCE_PV_FEATURE_CPUID), /* 190 */
311	CAP_ENTRY__L(KVM_CAP_SYS_HYPERV_CPUID),
312	CAP_ENTRY__L(KVM_CAP_DIRTY_LOG_RING),
313	CAP_ENTRY__L(KVM_CAP_X86_BUS_LOCK_EXIT),
314	CAP_ENTRY__L(KVM_CAP_PPC_DAWR1),
315	CAP_ENTRY__L(KVM_CAP_SET_GUEST_DEBUG2),
316	CAP_ENTRY__L(KVM_CAP_SGX_ATTRIBUTE),
317	CAP_ENTRY__L(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM),
318	CAP_ENTRY__L(KVM_CAP_PTP_KVM),
319	CAP_ENTRY__U(199),
320	CAP_ENTRY__U(200),
321	CAP_ENTRY__U(201),
322	CAP_ENTRY__U(202),
323	CAP_ENTRY__U(203),
324	CAP_ENTRY__U(204),
325	CAP_ENTRY__U(205),
326	CAP_ENTRY__U(206),
327	CAP_ENTRY__U(207),
328	CAP_ENTRY__U(208),
329	CAP_ENTRY__U(209),
330	CAP_ENTRY__U(210),
331	CAP_ENTRY__U(211),
332	CAP_ENTRY__U(212),
333	CAP_ENTRY__U(213),
334	CAP_ENTRY__U(214),
335	CAP_ENTRY__U(215),
336	CAP_ENTRY__U(216),
337	};
338
339	LogRel(("NEM: KVM capabilities (system):\n"));
340	int rcRet = VINF_SUCCESS;
341	for (unsigned i = 0; i < RT_ELEMENTS(s_aCaps); i++)
342	{
343	int rc = ioctl(pVM->nem.s.fdVm, KVM_CHECK_EXTENSION, s_aCaps[i].iCap);
344	if (rc >= 10)
345	LogRel(("NEM: %36s: %#x (%d)\n", s_aCaps[i].pszName, rc, rc));
346	else if (rc >= 0)
347	LogRel(("NEM: %36s: %d\n", s_aCaps[i].pszName, rc));
348	else
349	LogRel(("NEM: %s failed: %d/%d\n", s_aCaps[i].pszName, rc, errno));
350	switch (s_aCaps[i].cbNem)
351	{
352	case 0:
353	break;
354	case 1:
355	{
356	uint8_t puValue = (uint8_t )&pVM->nem.padding[s_aCaps[i].offNem];
357	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
358	*puValue = (uint8_t)rc;
359	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
360	break;
361	}
362	case 2:
363	{
364	uint16_t puValue = (uint16_t )&pVM->nem.padding[s_aCaps[i].offNem];
365	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
366	*puValue = (uint16_t)rc;
367	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
368	break;
369	}
370	case 4:
371	{
372	uint32_t puValue = (uint32_t )&pVM->nem.padding[s_aCaps[i].offNem];
373	AssertReturn(s_aCaps[i].offNem <= sizeof(NEM) - sizeof(*puValue), VERR_NEM_IPE_0);
374	*puValue = (uint32_t)rc;
375	AssertLogRelMsg((int)*puValue == rc, ("%s: %#x\n", s_aCaps[i].pszName, rc));
376	break;
377	}
378	default:
379	rcRet = RTErrInfoSetF(pErrInfo, VERR_NEM_IPE_0, "s_aCaps[%u] is bad: cbNem=%#x - %s",
380	i, s_aCaps[i].pszName, s_aCaps[i].cbNem);
381	AssertFailedReturn(rcRet);
382	}
383
384	/*
385	* Is a require non-zero entry zero or failing?
386	*/
387	if (s_aCaps[i].fReqNonZero && rc <= 0)
388	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_MISSING_FEATURE,
389	"Required capability '%s' is missing!", s_aCaps[i].pszName);
390	}
391
392	/*
393	* Get per VCpu KVM_RUN MMAP area size.
394	*/
395	int rc = ioctl(pVM->nem.s.fdKvm, KVM_GET_VCPU_MMAP_SIZE, 0UL);
396	if ((unsigned)rc < _64M)
397	{
398	pVM->nem.s.cbVCpuMmap = (uint32_t)rc;
399	LogRel(("NEM: %36s: %#x (%d)\n", "KVM_GET_VCPU_MMAP_SIZE", rc, rc));
400	}
401	else if (rc < 0)
402	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_MISSING_FEATURE, "KVM_GET_VCPU_MMAP_SIZE failed: %d", errno);
403	else
404	rcRet = RTERRINFO_LOG_REL_ADD_F(pErrInfo, VERR_NEM_INIT_FAILED, "Odd KVM_GET_VCPU_MMAP_SIZE value: %#x (%d)", rc, rc);
405
406	/*
407	* Init the slot ID bitmap.
408	*/
409	ASMBitSet(&pVM->nem.s.bmSlotIds[0], 0); /* don't use slot 0 */
410	if (pVM->nem.s.cMaxMemSlots < _32K)
411	ASMBitSetRange(&pVM->nem.s.bmSlotIds[0], pVM->nem.s.cMaxMemSlots, _32K);
412	ASMBitSet(&pVM->nem.s.bmSlotIds[0], _32K - 1); /* don't use the last slot */
413
414	return rcRet;
415	}
416
417
418	/**
419	* Does the early setup of a KVM VM.
420	*
421	* @returns VBox status code.
422	* @param pVM The cross context VM structure.
423	* @param pErrInfo Where to always return error info.
424	*/
425	static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo)
426	{
427	AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
428
429	/*
430	* Enable user space MSRs and let us check everything KVM cannot handle.
431	* We will set up filtering later when ring-3 init has completed.
432	*/
433	struct kvm_enable_cap CapEn =
434	{
435	KVM_CAP_X86_USER_SPACE_MSR, 0,
436	{ KVM_MSR_EXIT_REASON_FILTER \| KVM_MSR_EXIT_REASON_UNKNOWN \| KVM_MSR_EXIT_REASON_INVAL, 0, 0, 0}
437	};
438	int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_ENABLE_CAP, &CapEn);
439	if (rcLnx == -1)
440	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "Failed to enable KVM_CAP_X86_USER_SPACE_MSR failed: %u", errno);
441
442	/*
443	* Create the VCpus.
444	*/
445	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
446	{
447	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
448
449	/* Create it. */
450	pVCpu->nem.s.fdVCpu = ioctl(pVM->nem.s.fdVm, KVM_CREATE_VCPU, (unsigned long)idCpu);
451	if (pVCpu->nem.s.fdVCpu < 0)
452	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VCPU failed for VCpu #%u: %d", idCpu, errno);
453
454	/* Map the KVM_RUN area. */
455	pVCpu->nem.s.pRun = (struct kvm_run *)mmap(NULL, pVM->nem.s.cbVCpuMmap, PROT_READ \| PROT_WRITE, MAP_SHARED,
456	pVCpu->nem.s.fdVCpu, 0 /offset/);
457	if ((void *)pVCpu->nem.s.pRun == MAP_FAILED)
458	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "mmap failed for VCpu #%u: %d", idCpu, errno);
459
460	/* We want all x86 registers and events on each exit. */
461	pVCpu->nem.s.pRun->kvm_valid_regs = KVM_SYNC_X86_REGS \| KVM_SYNC_X86_SREGS \| KVM_SYNC_X86_EVENTS;
462	}
463	return VINF_SUCCESS;
464	}
465
466
467	/** @callback_method_impl{FNVMMEMTRENDEZVOUS} */
468	static DECLCALLBACK(VBOXSTRICTRC) nemR3LnxFixThreadPoke(PVM pVM, PVMCPU pVCpu, void *pvUser)
469	{
470	RT_NOREF(pVM, pvUser);
471	int rc = RTThreadControlPokeSignal(pVCpu->hThread, true /fEnable/);
472	AssertLogRelRC(rc);
473	return VINF_SUCCESS;
474	}
475
476
477	/**
478	* Try initialize the native API.
479	*
480	* This may only do part of the job, more can be done in
481	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
482	*
483	* @returns VBox status code.
484	* @param pVM The cross context VM structure.
485	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
486	* the latter we'll fail if we cannot initialize.
487	* @param fForced Whether the HMForced flag is set and we should
488	* fail if we cannot initialize.
489	*/
490	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
491	{
492	RT_NOREF(pVM, fFallback, fForced);
493	/*
494	* Some state init.
495	*/
496	pVM->nem.s.fdKvm = -1;
497	pVM->nem.s.fdVm = -1;
498	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
499	{
500	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
501	pNemCpu->fdVCpu = -1;
502	}
503
504	/*
505	* Error state.
506	* The error message will be non-empty on failure and 'rc' will be set too.
507	*/
508	RTERRINFOSTATIC ErrInfo;
509	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
510
511	/*
512	* Open kvm subsystem so we can issue system ioctls.
513	*/
514	int rc;
515	int fdKvm = open("/dev/kvm", O_RDWR \| O_CLOEXEC);
516	if (fdKvm >= 0)
517	{
518	pVM->nem.s.fdKvm = fdKvm;
519
520	/*
521	* Create an empty VM since it is recommended we check capabilities on
522	* the VM rather than the system descriptor.
523	*/
524	int fdVm = ioctl(fdKvm, KVM_CREATE_VM, 0UL /* Type must be zero on x86 */);
525	if (fdVm >= 0)
526	{
527	pVM->nem.s.fdVm = fdVm;
528
529	/*
530	* Check capabilities.
531	*/
532	rc = nemR3LnxInitCheckCapabilities(pVM, pErrInfo);
533	if (RT_SUCCESS(rc))
534	{
535	/*
536	* Set up the VM (more on this later).
537	*/
538	rc = nemR3LnxInitSetupVm(pVM, pErrInfo);
539	if (RT_SUCCESS(rc))
540	{
541	/*
542	* Set ourselves as the execution engine and make config adjustments.
543	*/
544	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
545	Log(("NEM: Marked active!\n"));
546	PGMR3EnableNemMode(pVM);
547
548	/*
549	* Register release statistics
550	*/
551	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
552	{
553	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
554	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnDemand, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of on-demand state imports", "/NEM/CPU%u/ImportOnDemand", idCpu);
555	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of state imports on loop return", "/NEM/CPU%u/ImportOnReturn", idCpu);
556	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturnSkipped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of skipped state imports on loop return", "/NEM/CPU%u/ImportOnReturnSkipped", idCpu);
557	STAMR3RegisterF(pVM, &pNemCpu->StatImportPendingInterrupt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times an interrupt was pending when importing from KVM", "/NEM/CPU%u/ImportPendingInterrupt", idCpu);
558	STAMR3RegisterF(pVM, &pNemCpu->StatExportPendingInterrupt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times an interrupt was pending when exporting to KVM", "/NEM/CPU%u/ExportPendingInterrupt", idCpu);
559	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn", idCpu);
560	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn1Loop, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-01-loop", idCpu);
561	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn2Loops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-02-loops", idCpu);
562	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn3Loops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-03-loops", idCpu);
563	STAMR3RegisterF(pVM, &pNemCpu->StatFlushExitOnReturn4PlusLoops, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of times a KVM_EXIT_IO or KVM_EXIT_MMIO was flushed before returning to EM", "/NEM/CPU%u/FlushExitOnReturn-04-to-7-loops", idCpu);
564	STAMR3RegisterF(pVM, &pNemCpu->StatQueryCpuTick, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of TSC queries", "/NEM/CPU%u/QueryCpuTick", idCpu);
565	STAMR3RegisterF(pVM, &pNemCpu->StatExitTotal, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "All exits", "/NEM/CPU%u/Exit", idCpu);
566	STAMR3RegisterF(pVM, &pNemCpu->StatExitIo, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_IO", "/NEM/CPU%u/Exit/Io", idCpu);
567	STAMR3RegisterF(pVM, &pNemCpu->StatExitMmio, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_MMIO", "/NEM/CPU%u/Exit/Mmio", idCpu);
568	STAMR3RegisterF(pVM, &pNemCpu->StatExitSetTpr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_SET_TRP", "/NEM/CPU%u/Exit/SetTpr", idCpu);
569	STAMR3RegisterF(pVM, &pNemCpu->StatExitTprAccess, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_TPR_ACCESS", "/NEM/CPU%u/Exit/TprAccess", idCpu);
570	STAMR3RegisterF(pVM, &pNemCpu->StatExitRdMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_RDMSR", "/NEM/CPU%u/Exit/RdMsr", idCpu);
571	STAMR3RegisterF(pVM, &pNemCpu->StatExitWrMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_WRMSR", "/NEM/CPU%u/Exit/WrMsr", idCpu);
572	STAMR3RegisterF(pVM, &pNemCpu->StatExitIrqWindowOpen, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_IRQ_WINDOWS_OPEN", "/NEM/CPU%u/Exit/IrqWindowOpen", idCpu);
573	STAMR3RegisterF(pVM, &pNemCpu->StatExitHalt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_HLT", "/NEM/CPU%u/Exit/Hlt", idCpu);
574	STAMR3RegisterF(pVM, &pNemCpu->StatExitIntr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTR", "/NEM/CPU%u/Exit/Intr", idCpu);
575	STAMR3RegisterF(pVM, &pNemCpu->StatExitHypercall, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_HYPERCALL", "/NEM/CPU%u/Exit/Hypercall", idCpu);
576	STAMR3RegisterF(pVM, &pNemCpu->StatExitDebug, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_DEBUG", "/NEM/CPU%u/Exit/Debug", idCpu);
577	STAMR3RegisterF(pVM, &pNemCpu->StatExitBusLock, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_BUS_LOCK", "/NEM/CPU%u/Exit/BusLock", idCpu);
578	STAMR3RegisterF(pVM, &pNemCpu->StatExitInternalErrorEmulation, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTERNAL_ERROR/EMULATION", "/NEM/CPU%u/Exit/InternalErrorEmulation", idCpu);
579	STAMR3RegisterF(pVM, &pNemCpu->StatExitInternalErrorFatal, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "KVM_EXIT_INTERNAL_ERROR/*", "/NEM/CPU%u/Exit/InternalErrorFatal", idCpu);
580	}
581
582	/*
583	* Success.
584	*/
585	return VINF_SUCCESS;
586	}
587
588	/*
589	* Bail out.
590	*/
591	}
592	close(fdVm);
593	pVM->nem.s.fdVm = -1;
594	}
595	else
596	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VM failed: %u", errno);
597	close(fdKvm);
598	pVM->nem.s.fdKvm = -1;
599	}
600	else if (errno == EACCES)
601	rc = RTErrInfoSet(pErrInfo, VERR_ACCESS_DENIED, "Do not have access to open /dev/kvm for reading & writing.");
602	else if (errno == ENOENT)
603	rc = RTErrInfoSet(pErrInfo, VERR_NOT_SUPPORTED, "KVM is not availble (/dev/kvm does not exist)");
604	else
605	rc = RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno), "Failed to open '/dev/kvm': %u", errno);
606
607	/*
608	* We only fail if in forced mode, otherwise just log the complaint and return.
609	*/
610	Assert(RTErrInfoIsSet(pErrInfo));
611	if ( (fForced \|\| !fFallback)
612	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
613	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
614	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
615	return VINF_SUCCESS;
616	}
617
618
619	/**
620	* This is called after CPUMR3Init is done.
621	*
622	* @returns VBox status code.
623	* @param pVM The VM handle..
624	*/
625	int nemR3NativeInitAfterCPUM(PVM pVM)
626	{
627	/*
628	* Validate sanity.
629	*/
630	AssertReturn(pVM->nem.s.fdKvm >= 0, VERR_WRONG_ORDER);
631	AssertReturn(pVM->nem.s.fdVm >= 0, VERR_WRONG_ORDER);
632	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
633
634	/** @todo */
635
636	return VINF_SUCCESS;
637	}
638
639
640	/**
641	* Update the CPUID leaves for a VCPU.
642	*
643	* The KVM_SET_CPUID2 call replaces any previous leaves, so we have to redo
644	* everything when there really just are single bit changes. That said, it
645	* looks like KVM update the XCR/XSAVE related stuff as well as the APIC enabled
646	* bit(s), so it should suffice if we do this at startup, I hope.
647	*/
648	static int nemR3LnxUpdateCpuIdsLeaves(PVM pVM, PVMCPU pVCpu)
649	{
650	uint32_t cLeaves = 0;
651	PCCPUMCPUIDLEAF const paLeaves = CPUMR3CpuIdGetPtr(pVM, &cLeaves);
652	struct kvm_cpuid2 pReq = (struct kvm_cpuid2 )alloca(RT_UOFFSETOF_DYN(struct kvm_cpuid2, entries[cLeaves + 2]));
653
654	pReq->nent = cLeaves;
655	pReq->padding = 0;
656
657	for (uint32_t i = 0; i < cLeaves; i++)
658	{
659	CPUMGetGuestCpuId(pVCpu, paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, -1 /f64BitMode/,
660	&pReq->entries[i].eax,
661	&pReq->entries[i].ebx,
662	&pReq->entries[i].ecx,
663	&pReq->entries[i].edx);
664	pReq->entries[i].function = paLeaves[i].uLeaf;
665	pReq->entries[i].index = paLeaves[i].uSubLeaf;
666	pReq->entries[i].flags = !paLeaves[i].fSubLeafMask ? 0 : KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
667	pReq->entries[i].padding[0] = 0;
668	pReq->entries[i].padding[1] = 0;
669	pReq->entries[i].padding[2] = 0;
670	}
671
672	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_CPUID2, pReq);
673	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d cLeaves=%#x\n", rcLnx, errno, cLeaves), RTErrConvertFromErrno(errno));
674
675	return VINF_SUCCESS;
676	}
677
678
679	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
680	{
681	/*
682	* Make RTThreadPoke work again (disabled for avoiding unnecessary
683	* critical section issues in ring-0).
684	*/
685	if (enmWhat == VMINITCOMPLETED_RING3)
686	VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, nemR3LnxFixThreadPoke, NULL);
687
688	/*
689	* Configure CPUIDs after ring-3 init has been done.
690	*/
691	if (enmWhat == VMINITCOMPLETED_RING3)
692	{
693	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
694	{
695	int rc = nemR3LnxUpdateCpuIdsLeaves(pVM, pVM->apCpusR3[idCpu]);
696	AssertRCReturn(rc, rc);
697	}
698	}
699
700	/*
701	* Configure MSRs after ring-3 init is done.
702	*
703	* We only need to tell KVM which MSRs it can handle, as we already
704	* requested KVM_MSR_EXIT_REASON_FILTER, KVM_MSR_EXIT_REASON_UNKNOWN
705	* and KVM_MSR_EXIT_REASON_INVAL in nemR3LnxInitSetupVm, and here we
706	* will use KVM_MSR_FILTER_DEFAULT_DENY. So, all MSRs w/o a 1 in the
707	* bitmaps should be deferred to ring-3.
708	*/
709	if (enmWhat == VMINITCOMPLETED_RING3)
710	{
711	struct kvm_msr_filter MsrFilters = {0}; /* Structure with a couple of implicit paddings on 64-bit systems. */
712	MsrFilters.flags = KVM_MSR_FILTER_DEFAULT_DENY;
713
714	unsigned iRange = 0;
715	#define MSR_RANGE_BEGIN(a_uBase, a_uEnd, a_fFlags) \
716	AssertCompile(0x3000 <= KVM_MSR_FILTER_MAX_BITMAP_SIZE * 8); \
717	uint64_t RT_CONCAT(bm, a_uBase)[0x3000 / 64] = {0}; \
718	do { \
719	uint64_t * const pbm = RT_CONCAT(bm, a_uBase); \
720	uint32_t const uBase = UINT32_C(a_uBase); \
721	uint32_t const cMsrs = UINT32_C(a_uEnd) - UINT32_C(a_uBase); \
722	MsrFilters.ranges[iRange].base = UINT32_C(a_uBase); \
723	MsrFilters.ranges[iRange].nmsrs = cMsrs; \
724	MsrFilters.ranges[iRange].flags = (a_fFlags); \
725	MsrFilters.ranges[iRange].bitmap = (uint8_t *)&RT_CONCAT(bm, a_uBase)[0]
726	#define MSR_RANGE_ADD(a_Msr) \
727	do { Assert((uint32_t)(a_Msr) - uBase < cMsrs); ASMBitSet(pbm, (uint32_t)(a_Msr) - uBase); } while (0)
728	#define MSR_RANGE_END(a_cMinMsrs) \
729	/* optimize the range size before closing: */ \
730	uint32_t cBitmap = cMsrs / 64; \
731	while (cBitmap > ((a_cMinMsrs) + 63 / 64) && pbm[cBitmap - 1] == 0) \
732	cBitmap -= 1; \
733	MsrFilters.ranges[iRange].nmsrs = cBitmap * 64; \
734	iRange++; \
735	} while (0)
736
737	/* 1st Intel range: 0000_0000 to 0000_3000. */
738	MSR_RANGE_BEGIN(0x00000000, 0x00003000, KVM_MSR_FILTER_READ \| KVM_MSR_FILTER_WRITE);
739	MSR_RANGE_ADD(MSR_IA32_TSC);
740	MSR_RANGE_ADD(MSR_IA32_SYSENTER_CS);
741	MSR_RANGE_ADD(MSR_IA32_SYSENTER_ESP);
742	MSR_RANGE_ADD(MSR_IA32_SYSENTER_EIP);
743	MSR_RANGE_ADD(MSR_IA32_CR_PAT);
744	/** @todo more? */
745	MSR_RANGE_END(64);
746
747	/* 1st AMD range: c000_0000 to c000_3000 */
748	MSR_RANGE_BEGIN(0xc0000000, 0xc0003000, KVM_MSR_FILTER_READ \| KVM_MSR_FILTER_WRITE);
749	MSR_RANGE_ADD(MSR_K6_EFER);
750	MSR_RANGE_ADD(MSR_K6_STAR);
751	MSR_RANGE_ADD(MSR_K8_GS_BASE);
752	MSR_RANGE_ADD(MSR_K8_KERNEL_GS_BASE);
753	MSR_RANGE_ADD(MSR_K8_LSTAR);
754	MSR_RANGE_ADD(MSR_K8_CSTAR);
755	MSR_RANGE_ADD(MSR_K8_SF_MASK);
756	MSR_RANGE_ADD(MSR_K8_TSC_AUX);
757	/** @todo add more? */
758	MSR_RANGE_END(64);
759
760	/** @todo Specify other ranges too? Like hyper-V and KVM to make sure we get
761	* the MSR requests instead of KVM. */
762
763	int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_X86_SET_MSR_FILTER, &MsrFilters);
764	if (rcLnx == -1)
765	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
766	"Failed to enable KVM_X86_SET_MSR_FILTER failed: %u", errno);
767	}
768
769	return VINF_SUCCESS;
770	}
771
772
773	int nemR3NativeTerm(PVM pVM)
774	{
775	/*
776	* Per-cpu data
777	*/
778	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
779	{
780	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
781
782	if (pVCpu->nem.s.fdVCpu != -1)
783	{
784	close(pVCpu->nem.s.fdVCpu);
785	pVCpu->nem.s.fdVCpu = -1;
786	}
787	if (pVCpu->nem.s.pRun)
788	{
789	munmap(pVCpu->nem.s.pRun, pVM->nem.s.cbVCpuMmap);
790	pVCpu->nem.s.pRun = NULL;
791	}
792	}
793
794	/*
795	* Global data.
796	*/
797	if (pVM->nem.s.fdVm != -1)
798	{
799	close(pVM->nem.s.fdVm);
800	pVM->nem.s.fdVm = -1;
801	}
802
803	if (pVM->nem.s.fdKvm != -1)
804	{
805	close(pVM->nem.s.fdKvm);
806	pVM->nem.s.fdKvm = -1;
807	}
808	return VINF_SUCCESS;
809	}
810
811
812	/**
813	* VM reset notification.
814	*
815	* @param pVM The cross context VM structure.
816	*/
817	void nemR3NativeReset(PVM pVM)
818	{
819	RT_NOREF(pVM);
820	}
821
822
823	/**
824	* Reset CPU due to INIT IPI or hot (un)plugging.
825	*
826	* @param pVCpu The cross context virtual CPU structure of the CPU being
827	* reset.
828	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
829	*/
830	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
831	{
832	RT_NOREF(pVCpu, fInitIpi);
833	}
834
835
836	/*********************************************************************************************************************************
837	* Memory management *
838	*********************************************************************************************************************************/
839
840
841	/**
842	* Allocates a memory slot ID.
843	*
844	* @returns Slot ID on success, UINT16_MAX on failure.
845	*/
846	static uint16_t nemR3LnxMemSlotIdAlloc(PVM pVM)
847	{
848	/* Use the hint first. */
849	uint16_t idHint = pVM->nem.s.idPrevSlot;
850	if (idHint < _32K - 1)
851	{
852	int32_t idx = ASMBitNextClear(&pVM->nem.s.bmSlotIds, _32K, idHint);
853	Assert(idx < _32K);
854	if (idx > 0 && !ASMAtomicBitTestAndSet(&pVM->nem.s.bmSlotIds, idx))
855	return pVM->nem.s.idPrevSlot = (uint16_t)idx;
856	}
857
858	/*
859	* Search the whole map from the start.
860	*/
861	int32_t idx = ASMBitFirstClear(&pVM->nem.s.bmSlotIds, _32K);
862	Assert(idx < _32K);
863	if (idx > 0 && !ASMAtomicBitTestAndSet(&pVM->nem.s.bmSlotIds, idx))
864	return pVM->nem.s.idPrevSlot = (uint16_t)idx;
865
866	Assert(idx < 0 /shouldn't trigger unless there is a race /);
867	return UINT16_MAX; /* caller is expected to assert. */
868	}
869
870
871	/**
872	* Frees a memory slot ID
873	*/
874	static void nemR3LnxMemSlotIdFree(PVM pVM, uint16_t idSlot)
875	{
876	if (RT_LIKELY(idSlot < _32K && ASMAtomicBitTestAndClear(&pVM->nem.s.bmSlotIds, idSlot)))
877	{ /likely/ }
878	else
879	AssertMsgFailed(("idSlot=%u (%#x)\n", idSlot, idSlot));
880	}
881
882
883
884	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
885	uint8_t pu2State, uint32_t puNemRange)
886	{
887	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
888	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
889
890	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p pu2State=%p (%d) puNemRange=%p (%d) - idSlot=%#x\n",
891	GCPhys, cb, pvR3, pu2State, pu2State, puNemRange, *puNemRange, idSlot));
892
893	struct kvm_userspace_memory_region Region;
894	Region.slot = idSlot;
895	Region.flags = 0;
896	Region.guest_phys_addr = GCPhys;
897	Region.memory_size = cb;
898	Region.userspace_addr = (uintptr_t)pvR3;
899
900	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
901	if (rc == 0)
902	{
903	*pu2State = 0;
904	*puNemRange = idSlot;
905	return VINF_SUCCESS;
906	}
907
908	LogRel(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p, idSlot=%#x failed: %u/%u\n", GCPhys, cb, pvR3, idSlot, rc, errno));
909	nemR3LnxMemSlotIdFree(pVM, idSlot);
910	return VERR_NEM_MAP_PAGES_FAILED;
911	}
912
913
914	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
915	{
916	RT_NOREF(pVM);
917	return true;
918	}
919
920
921	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
922	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
923	{
924	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d) puNemRange=%p (%#x)\n",
925	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, pu2State, puNemRange, puNemRange ? puNemRange : UINT32_MAX));
926	RT_NOREF(pvRam);
927
928	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
929	{
930	/** @todo implement splitting and whatnot of ranges if we want to be 100%
931	* conforming (just modify RAM registrations in MM.cpp to test). */
932	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p\n", GCPhys, cb, fFlags, pvRam, pvMmio2),
933	VERR_NEM_MAP_PAGES_FAILED);
934	}
935
936	/*
937	* Register MMIO2.
938	*/
939	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
940	{
941	AssertReturn(pvMmio2, VERR_NEM_MAP_PAGES_FAILED);
942	AssertReturn(puNemRange, VERR_NEM_MAP_PAGES_FAILED);
943
944	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
945	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
946
947	struct kvm_userspace_memory_region Region;
948	Region.slot = idSlot;
949	Region.flags = fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES ? KVM_MEM_LOG_DIRTY_PAGES : 0;
950	Region.guest_phys_addr = GCPhys;
951	Region.memory_size = cb;
952	Region.userspace_addr = (uintptr_t)pvMmio2;
953
954	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
955	if (rc == 0)
956	{
957	*pu2State = 0;
958	*puNemRange = idSlot;
959	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvMmio2=%p - idSlot=%#x\n",
960	GCPhys, cb, fFlags, pvMmio2, idSlot));
961	return VINF_SUCCESS;
962	}
963
964	nemR3LnxMemSlotIdFree(pVM, idSlot);
965	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvMmio2=%p, idSlot=%#x failed: %u/%u\n",
966	GCPhys, cb, fFlags, pvMmio2, idSlot, errno, rc),
967	VERR_NEM_MAP_PAGES_FAILED);
968	}
969
970	/* MMIO, don't care. */
971	*pu2State = 0;
972	*puNemRange = UINT32_MAX;
973	return VINF_SUCCESS;
974	}
975
976
977	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
978	void pvRam, void pvMmio2, uint32_t *puNemRange)
979	{
980	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
981	return VINF_SUCCESS;
982	}
983
984
985	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
986	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
987	{
988	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p puNemRange=%p (%#x)\n",
989	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
990	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
991
992	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
993	{
994	/** @todo implement splitting and whatnot of ranges if we want to be 100%
995	* conforming (just modify RAM registrations in MM.cpp to test). */
996	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p\n", GCPhys, cb, fFlags, pvRam, pvMmio2),
997	VERR_NEM_UNMAP_PAGES_FAILED);
998	}
999
1000	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
1001	{
1002	uint32_t const idSlot = *puNemRange;
1003	AssertReturn(idSlot > 0 && idSlot < _32K, VERR_NEM_IPE_4);
1004	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, idSlot), VERR_NEM_IPE_4);
1005
1006	struct kvm_userspace_memory_region Region;
1007	Region.slot = idSlot;
1008	Region.flags = 0;
1009	Region.guest_phys_addr = GCPhys;
1010	Region.memory_size = 0; /* this deregisters it. */
1011	Region.userspace_addr = (uintptr_t)pvMmio2;
1012
1013	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
1014	if (rc == 0)
1015	{
1016	if (pu2State)
1017	*pu2State = 0;
1018	*puNemRange = UINT32_MAX;
1019	nemR3LnxMemSlotIdFree(pVM, idSlot);
1020	return VINF_SUCCESS;
1021	}
1022
1023	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvMmio2=%p, idSlot=%#x failed: %u/%u\n",
1024	GCPhys, cb, fFlags, pvMmio2, idSlot, errno, rc),
1025	VERR_NEM_UNMAP_PAGES_FAILED);
1026	}
1027
1028	if (pu2State)
1029	*pu2State = UINT8_MAX;
1030	return VINF_SUCCESS;
1031	}
1032
1033
1034	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
1035	void *pvBitmap, size_t cbBitmap)
1036	{
1037	AssertReturn(uNemRange > 0 && uNemRange < _32K, VERR_NEM_IPE_4);
1038	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, uNemRange), VERR_NEM_IPE_4);
1039
1040	RT_NOREF(GCPhys, cbBitmap);
1041
1042	struct kvm_dirty_log DirtyLog;
1043	DirtyLog.slot = uNemRange;
1044	DirtyLog.padding1 = 0;
1045	DirtyLog.dirty_bitmap = pvBitmap;
1046
1047	int rc = ioctl(pVM->nem.s.fdVm, KVM_GET_DIRTY_LOG, &DirtyLog);
1048	AssertLogRelMsgReturn(rc == 0, ("%RGp LB %RGp idSlot=%#x failed: %u/%u\n", GCPhys, cb, uNemRange, errno, rc),
1049	VERR_NEM_QUERY_DIRTY_BITMAP_FAILED);
1050
1051	return VINF_SUCCESS;
1052	}
1053
1054
1055	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
1056	uint8_t pu2State, uint32_t puNemRange)
1057	{
1058	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
1059	*pu2State = UINT8_MAX;
1060
1061	/* Don't support puttint ROM where there is already RAM. For
1062	now just shuffle the registrations till it works... */
1063	AssertLogRelMsgReturn(!(fFlags & NEM_NOTIFY_PHYS_ROM_F_REPLACE), ("%RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags),
1064	VERR_NEM_MAP_PAGES_FAILED);
1065
1066	/** @todo figure out how to do shadow ROMs. */
1067
1068	/*
1069	* We only allocate a slot number here in case we need to use it to
1070	* fend of physical handler fun.
1071	*/
1072	uint16_t idSlot = nemR3LnxMemSlotIdAlloc(pVM);
1073	AssertLogRelReturn(idSlot < _32K, VERR_NEM_MAP_PAGES_FAILED);
1074
1075	*pu2State = 0;
1076	*puNemRange = idSlot;
1077	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x pvPages=%p - idSlot=%#x\n",
1078	GCPhys, cb, fFlags, pvPages, idSlot));
1079	RT_NOREF(GCPhys, cb, fFlags, pvPages);
1080	return VINF_SUCCESS;
1081	}
1082
1083
1084	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
1085	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
1086	{
1087	Log5(("NEMR3NotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
1088	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
1089
1090	AssertPtrReturn(pvPages, VERR_NEM_IPE_5);
1091
1092	uint32_t const idSlot = *puNemRange;
1093	AssertReturn(idSlot > 0 && idSlot < _32K, VERR_NEM_IPE_4);
1094	AssertReturn(ASMBitTest(pVM->nem.s.bmSlotIds, idSlot), VERR_NEM_IPE_4);
1095
1096	*pu2State = UINT8_MAX;
1097
1098	/*
1099	* Do the actual setting of the user pages here now that we've
1100	* got a valid pvPages (typically isn't available during the early
1101	* notification, unless we're replacing RAM).
1102	*/
1103	struct kvm_userspace_memory_region Region;
1104	Region.slot = idSlot;
1105	Region.flags = 0;
1106	Region.guest_phys_addr = GCPhys;
1107	Region.memory_size = cb;
1108	Region.userspace_addr = (uintptr_t)pvPages;
1109
1110	int rc = ioctl(pVM->nem.s.fdVm, KVM_SET_USER_MEMORY_REGION, &Region);
1111	if (rc == 0)
1112	{
1113	*pu2State = 0;
1114	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x pvPages=%p - idSlot=%#x\n",
1115	GCPhys, cb, fFlags, pvPages, idSlot));
1116	return VINF_SUCCESS;
1117	}
1118	AssertLogRelMsgFailedReturn(("%RGp LB %RGp fFlags=%#x, pvPages=%p, idSlot=%#x failed: %u/%u\n",
1119	GCPhys, cb, fFlags, pvPages, idSlot, errno, rc),
1120	VERR_NEM_MAP_PAGES_FAILED);
1121	}
1122
1123
1124	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
1125	{
1126	Log(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
1127	Assert(VM_IS_NEM_ENABLED(pVCpu->CTX_SUFF(pVM)));
1128	RT_NOREF(pVCpu, fEnabled);
1129	}
1130
1131
1132	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
1133	RTR3PTR pvMemR3, uint8_t *pu2State)
1134	{
1135	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
1136	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
1137
1138	*pu2State = UINT8_MAX;
1139	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
1140	}
1141
1142
1143	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
1144	{
1145	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
1146	RT_NOREF(pVM, enmKind, GCPhys, cb);
1147	}
1148
1149
1150	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
1151	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
1152	{
1153	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
1154	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
1155	RT_NOREF(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM);
1156	}
1157
1158
1159	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
1160	PGMPAGETYPE enmType, uint8_t *pu2State)
1161	{
1162	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
1163	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
1164	RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
1165	return VINF_SUCCESS;
1166	}
1167
1168
1169	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
1170	PGMPAGETYPE enmType, uint8_t *pu2State)
1171	{
1172	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
1173	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
1174	Assert(VM_IS_NEM_ENABLED(pVM));
1175	RT_NOREF(pVM, GCPhys, HCPhys, pvR3, fPageProt, enmType, pu2State);
1176
1177	}
1178
1179
1180	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
1181	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
1182	{
1183	Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp pvNewR3=%p fPageProt=%#x enmType=%d *pu2State=%d\n",
1184	GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, *pu2State));
1185	Assert(VM_IS_NEM_ENABLED(pVM));
1186	RT_NOREF(pVM, GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, pu2State);
1187	}
1188
1189
1190	/*********************************************************************************************************************************
1191	* CPU State *
1192	*********************************************************************************************************************************/
1193
1194	/**
1195	* Worker that imports selected state from KVM.
1196	*/
1197	static int nemHCLnxImportState(PVMCPUCC pVCpu, uint64_t fWhat, PCPUMCTX pCtx, struct kvm_run *pRun)
1198	{
1199	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
1200	if (!fWhat)
1201	return VINF_SUCCESS;
1202
1203	/*
1204	* Stuff that goes into kvm_run::s.regs.regs:
1205	*/
1206	if (fWhat & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK))
1207	{
1208	if (fWhat & CPUMCTX_EXTRN_RIP)
1209	pCtx->rip = pRun->s.regs.regs.rip;
1210	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
1211	pCtx->rflags.u = pRun->s.regs.regs.rflags;
1212
1213	if (fWhat & CPUMCTX_EXTRN_RAX)
1214	pCtx->rax = pRun->s.regs.regs.rax;
1215	if (fWhat & CPUMCTX_EXTRN_RCX)
1216	pCtx->rcx = pRun->s.regs.regs.rcx;
1217	if (fWhat & CPUMCTX_EXTRN_RDX)
1218	pCtx->rdx = pRun->s.regs.regs.rdx;
1219	if (fWhat & CPUMCTX_EXTRN_RBX)
1220	pCtx->rbx = pRun->s.regs.regs.rbx;
1221	if (fWhat & CPUMCTX_EXTRN_RSP)
1222	pCtx->rsp = pRun->s.regs.regs.rsp;
1223	if (fWhat & CPUMCTX_EXTRN_RBP)
1224	pCtx->rbp = pRun->s.regs.regs.rbp;
1225	if (fWhat & CPUMCTX_EXTRN_RSI)
1226	pCtx->rsi = pRun->s.regs.regs.rsi;
1227	if (fWhat & CPUMCTX_EXTRN_RDI)
1228	pCtx->rdi = pRun->s.regs.regs.rdi;
1229	if (fWhat & CPUMCTX_EXTRN_R8_R15)
1230	{
1231	pCtx->r8 = pRun->s.regs.regs.r8;
1232	pCtx->r9 = pRun->s.regs.regs.r9;
1233	pCtx->r10 = pRun->s.regs.regs.r10;
1234	pCtx->r11 = pRun->s.regs.regs.r11;
1235	pCtx->r12 = pRun->s.regs.regs.r12;
1236	pCtx->r13 = pRun->s.regs.regs.r13;
1237	pCtx->r14 = pRun->s.regs.regs.r14;
1238	pCtx->r15 = pRun->s.regs.regs.r15;
1239	}
1240	}
1241
1242	/*
1243	* Stuff that goes into kvm_run::s.regs.sregs.
1244	*
1245	* Note! The apic_base can be ignored because we gets all MSR writes to it
1246	* and VBox always keeps the correct value.
1247	*/
1248	bool fMaybeChangedMode = false;
1249	bool fUpdateCr3 = false;
1250	if (fWhat & ( CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_CR_MASK
1251	\| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_APIC_TPR))
1252	{
1253	/** @todo what about Attr.n.u4LimitHigh? */
1254	#define NEM_LNX_IMPORT_SEG(a_CtxSeg, a_KvmSeg) do { \
1255	(a_CtxSeg).u64Base = (a_KvmSeg).base; \
1256	(a_CtxSeg).u32Limit = (a_KvmSeg).limit; \
1257	(a_CtxSeg).ValidSel = (a_CtxSeg).Sel = (a_KvmSeg).selector; \
1258	(a_CtxSeg).Attr.n.u4Type = (a_KvmSeg).type; \
1259	(a_CtxSeg).Attr.n.u1DescType = (a_KvmSeg).s; \
1260	(a_CtxSeg).Attr.n.u2Dpl = (a_KvmSeg).dpl; \
1261	(a_CtxSeg).Attr.n.u1Present = (a_KvmSeg).present; \
1262	(a_CtxSeg).Attr.n.u1Available = (a_KvmSeg).avl; \
1263	(a_CtxSeg).Attr.n.u1Long = (a_KvmSeg).l; \
1264	(a_CtxSeg).Attr.n.u1DefBig = (a_KvmSeg).db; \
1265	(a_CtxSeg).Attr.n.u1Granularity = (a_KvmSeg).g; \
1266	(a_CtxSeg).Attr.n.u1Unusable = (a_KvmSeg).unusable; \
1267	(a_CtxSeg).fFlags = CPUMSELREG_FLAGS_VALID; \
1268	CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &(a_CtxSeg)); \
1269	} while (0)
1270
1271	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
1272	{
1273	if (fWhat & CPUMCTX_EXTRN_ES)
1274	NEM_LNX_IMPORT_SEG(pCtx->es, pRun->s.regs.sregs.es);
1275	if (fWhat & CPUMCTX_EXTRN_CS)
1276	NEM_LNX_IMPORT_SEG(pCtx->cs, pRun->s.regs.sregs.cs);
1277	if (fWhat & CPUMCTX_EXTRN_SS)
1278	NEM_LNX_IMPORT_SEG(pCtx->ss, pRun->s.regs.sregs.ss);
1279	if (fWhat & CPUMCTX_EXTRN_DS)
1280	NEM_LNX_IMPORT_SEG(pCtx->ds, pRun->s.regs.sregs.ds);
1281	if (fWhat & CPUMCTX_EXTRN_FS)
1282	NEM_LNX_IMPORT_SEG(pCtx->fs, pRun->s.regs.sregs.fs);
1283	if (fWhat & CPUMCTX_EXTRN_GS)
1284	NEM_LNX_IMPORT_SEG(pCtx->gs, pRun->s.regs.sregs.gs);
1285	}
1286	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
1287	{
1288	if (fWhat & CPUMCTX_EXTRN_GDTR)
1289	{
1290	pCtx->gdtr.pGdt = pRun->s.regs.sregs.gdt.base;
1291	pCtx->gdtr.cbGdt = pRun->s.regs.sregs.gdt.limit;
1292	}
1293	if (fWhat & CPUMCTX_EXTRN_IDTR)
1294	{
1295	pCtx->idtr.pIdt = pRun->s.regs.sregs.idt.base;
1296	pCtx->idtr.cbIdt = pRun->s.regs.sregs.idt.limit;
1297	}
1298	if (fWhat & CPUMCTX_EXTRN_LDTR)
1299	NEM_LNX_IMPORT_SEG(pCtx->ldtr, pRun->s.regs.sregs.ldt);
1300	if (fWhat & CPUMCTX_EXTRN_TR)
1301	NEM_LNX_IMPORT_SEG(pCtx->tr, pRun->s.regs.sregs.tr);
1302	}
1303	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
1304	{
1305	if (fWhat & CPUMCTX_EXTRN_CR0)
1306	{
1307	if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
1308	{
1309	CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
1310	fMaybeChangedMode = true;
1311	}
1312	}
1313	if (fWhat & CPUMCTX_EXTRN_CR2)
1314	pCtx->cr2 = pRun->s.regs.sregs.cr2;
1315	if (fWhat & CPUMCTX_EXTRN_CR3)
1316	{
1317	if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
1318	{
1319	CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
1320	fUpdateCr3 = true;
1321	}
1322	}
1323	if (fWhat & CPUMCTX_EXTRN_CR4)
1324	{
1325	if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
1326	{
1327	CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
1328	fMaybeChangedMode = true;
1329	}
1330	}
1331	}
1332	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1333	APICSetTpr(pVCpu, (uint8_t)pRun->s.regs.sregs.cr8 << 4);
1334	if (fWhat & CPUMCTX_EXTRN_EFER)
1335	{
1336	if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
1337	{
1338	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
1339	if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1340	PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
1341	pCtx->msrEFER = pRun->s.regs.sregs.efer;
1342	fMaybeChangedMode = true;
1343	}
1344	}
1345	#undef NEM_LNX_IMPORT_SEG
1346	}
1347
1348	/*
1349	* Debug registers.
1350	*/
1351	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
1352	{
1353	struct kvm_debugregs DbgRegs = {{0}};
1354	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
1355	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1356
1357	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1358	{
1359	pCtx->dr[0] = DbgRegs.db[0];
1360	pCtx->dr[1] = DbgRegs.db[1];
1361	pCtx->dr[2] = DbgRegs.db[2];
1362	pCtx->dr[3] = DbgRegs.db[3];
1363	}
1364	if (fWhat & CPUMCTX_EXTRN_DR6)
1365	pCtx->dr[6] = DbgRegs.dr6;
1366	if (fWhat & CPUMCTX_EXTRN_DR7)
1367	pCtx->dr[7] = DbgRegs.dr7;
1368	}
1369
1370	/*
1371	* FPU, SSE, AVX, ++.
1372	*/
1373	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx))
1374	{
1375	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1376	{
1377	fWhat \|= CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE; /* we do all or nothing at all */
1378
1379	AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
1380	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XSAVE, &pCtx->XState);
1381	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1382	}
1383
1384	if (fWhat & CPUMCTX_EXTRN_XCRx)
1385	{
1386	struct kvm_xcrs Xcrs =
1387	{ /.nr_xcrs = / 2,
1388	/.flags = / 0,
1389	/.xcrs= / {
1390	{ /.xcr =/ 0, /.reserved=/ 0, /.value=/ pCtx->aXcr[0] },
1391	{ /.xcr =/ 1, /.reserved=/ 0, /.value=/ pCtx->aXcr[1] },
1392	}
1393	};
1394
1395	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XCRS, &Xcrs);
1396	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1397
1398	pCtx->aXcr[0] = Xcrs.xcrs[0].value;
1399	pCtx->aXcr[1] = Xcrs.xcrs[1].value;
1400	}
1401	}
1402
1403	/*
1404	* MSRs.
1405	*/
1406	if (fWhat & ( CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_SYSENTER_MSRS
1407	\| CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
1408	{
1409	union
1410	{
1411	struct kvm_msrs Core;
1412	uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
1413	} uBuf;
1414	uint64_t *pauDsts[32];
1415	uint32_t iMsr = 0;
1416	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1417
1418	#define ADD_MSR(a_Msr, a_uValue) do { \
1419	Assert(iMsr < 32); \
1420	uBuf.Core.entries[iMsr].index = (a_Msr); \
1421	uBuf.Core.entries[iMsr].reserved = 0; \
1422	uBuf.Core.entries[iMsr].data = UINT64_MAX; \
1423	pauDsts[iMsr] = &(a_uValue); \
1424	iMsr += 1; \
1425	} while (0)
1426
1427	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1428	ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1429	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1430	{
1431	ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
1432	ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
1433	ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
1434	ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1435	}
1436	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1437	{
1438	ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
1439	ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
1440	ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
1441	}
1442	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1443	ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1444	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1445	{
1446	ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
1447	/** @todo What do we _have_ to add here?
1448	* We also have: Mttr, MiscEnable, FeatureControl. /
1449	}
1450
1451	uBuf.Core.pad = 0;
1452	uBuf.Core.nmsrs = iMsr;
1453	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MSRS, &uBuf);
1454	AssertMsgReturn(rc == (int)iMsr,
1455	("rc=%d iMsr=%d (->%#x) errno=%d\n",
1456	rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
1457	VERR_NEM_IPE_3);
1458
1459	while (iMsr-- > 0)
1460	*pauDsts[iMsr] = uBuf.Core.entries[iMsr].data;
1461	#undef ADD_MSR
1462	}
1463
1464	/*
1465	* Interruptibility state and pending interrupts.
1466	*/
1467	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1468	{
1469	fWhat \|= CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI; /* always do both, see export and interrupt FF handling */
1470
1471	struct kvm_vcpu_events KvmEvents = {0};
1472	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
1473	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
1474
1475	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
1476	pVCpu->cpum.GstCtx.rip = pRun->s.regs.regs.rip;
1477
1478	if (KvmEvents.interrupt.shadow)
1479	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
1480	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1481	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1482
1483	if (KvmEvents.nmi.masked)
1484	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
1485	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
1486	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
1487
1488	if (KvmEvents.interrupt.injected)
1489	{
1490	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportPendingInterrupt);
1491	TRPMAssertTrap(pVCpu, KvmEvents.interrupt.nr, !KvmEvents.interrupt.soft ? TRPM_HARDWARE_INT : TRPM_SOFTWARE_INT);
1492	}
1493
1494	Assert(KvmEvents.nmi.injected == 0);
1495	Assert(KvmEvents.nmi.pending == 0);
1496	}
1497
1498	/*
1499	* Update the external mask.
1500	*/
1501	pCtx->fExtrn &= ~fWhat;
1502	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1503	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1504	pVCpu->cpum.GstCtx.fExtrn = 0;
1505
1506	/*
1507	* We sometimes need to update PGM on the guest status.
1508	*/
1509	if (!fMaybeChangedMode && !fUpdateCr3)
1510	{ /* likely */ }
1511	else
1512	{
1513	/*
1514	* Make sure we got all the state PGM might need.
1515	*/
1516	Log7(("nemHCLnxImportState: fMaybeChangedMode=%d fUpdateCr3=%d fExtrnNeeded=%#RX64\n", fMaybeChangedMode, fUpdateCr3,
1517	pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER) ));
1518	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER))
1519	{
1520	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR0)
1521	{
1522	if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
1523	{
1524	CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
1525	fMaybeChangedMode = true;
1526	}
1527	}
1528	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR3)
1529	{
1530	if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
1531	{
1532	CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
1533	fUpdateCr3 = true;
1534	}
1535	}
1536	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR4)
1537	{
1538	if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
1539	{
1540	CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
1541	fMaybeChangedMode = true;
1542	}
1543	}
1544	if (fWhat & CPUMCTX_EXTRN_EFER)
1545	{
1546	if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
1547	{
1548	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
1549	if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1550	PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
1551	pCtx->msrEFER = pRun->s.regs.sregs.efer;
1552	fMaybeChangedMode = true;
1553	}
1554	}
1555
1556	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_EFER);
1557	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1558	pVCpu->cpum.GstCtx.fExtrn = 0;
1559	}
1560
1561	/*
1562	* Notify PGM about the changes.
1563	*/
1564	if (fMaybeChangedMode)
1565	{
1566	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4,
1567	pVCpu->cpum.GstCtx.msrEFER, false /fForce/);
1568	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1569	}
1570
1571	if (fUpdateCr3)
1572	{
1573	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
1574	if (rc == VINF_SUCCESS)
1575	{ /* likely */ }
1576	else
1577	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1578	}
1579	}
1580
1581	return VINF_SUCCESS;
1582	}
1583
1584
1585	/**
1586	* Interface for importing state on demand (used by IEM).
1587	*
1588	* @returns VBox status code.
1589	* @param pVCpu The cross context CPU structure.
1590	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1591	*/
1592	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1593	{
1594	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
1595	return nemHCLnxImportState(pVCpu, fWhat, &pVCpu->cpum.GstCtx, pVCpu->nem.s.pRun);
1596	}
1597
1598
1599	/**
1600	* Exports state to KVM.
1601	*/
1602	static int nemHCLnxExportState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, struct kvm_run *pRun)
1603	{
1604	uint64_t const fExtrn = ~pCtx->fExtrn & CPUMCTX_EXTRN_ALL;
1605	Assert((~fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL);
1606
1607	/*
1608	* Stuff that goes into kvm_run::s.regs.regs:
1609	*/
1610	if (fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK))
1611	{
1612	if (fExtrn & CPUMCTX_EXTRN_RIP)
1613	pRun->s.regs.regs.rip = pCtx->rip;
1614	if (fExtrn & CPUMCTX_EXTRN_RFLAGS)
1615	pRun->s.regs.regs.rflags = pCtx->rflags.u;
1616
1617	if (fExtrn & CPUMCTX_EXTRN_RAX)
1618	pRun->s.regs.regs.rax = pCtx->rax;
1619	if (fExtrn & CPUMCTX_EXTRN_RCX)
1620	pRun->s.regs.regs.rcx = pCtx->rcx;
1621	if (fExtrn & CPUMCTX_EXTRN_RDX)
1622	pRun->s.regs.regs.rdx = pCtx->rdx;
1623	if (fExtrn & CPUMCTX_EXTRN_RBX)
1624	pRun->s.regs.regs.rbx = pCtx->rbx;
1625	if (fExtrn & CPUMCTX_EXTRN_RSP)
1626	pRun->s.regs.regs.rsp = pCtx->rsp;
1627	if (fExtrn & CPUMCTX_EXTRN_RBP)
1628	pRun->s.regs.regs.rbp = pCtx->rbp;
1629	if (fExtrn & CPUMCTX_EXTRN_RSI)
1630	pRun->s.regs.regs.rsi = pCtx->rsi;
1631	if (fExtrn & CPUMCTX_EXTRN_RDI)
1632	pRun->s.regs.regs.rdi = pCtx->rdi;
1633	if (fExtrn & CPUMCTX_EXTRN_R8_R15)
1634	{
1635	pRun->s.regs.regs.r8 = pCtx->r8;
1636	pRun->s.regs.regs.r9 = pCtx->r9;
1637	pRun->s.regs.regs.r10 = pCtx->r10;
1638	pRun->s.regs.regs.r11 = pCtx->r11;
1639	pRun->s.regs.regs.r12 = pCtx->r12;
1640	pRun->s.regs.regs.r13 = pCtx->r13;
1641	pRun->s.regs.regs.r14 = pCtx->r14;
1642	pRun->s.regs.regs.r15 = pCtx->r15;
1643	}
1644	pRun->kvm_dirty_regs \|= KVM_SYNC_X86_REGS;
1645	}
1646
1647	/*
1648	* Stuff that goes into kvm_run::s.regs.sregs:
1649	*
1650	* The APIC base register updating is a little suboptimal... But at least
1651	* VBox always has the right base register value, so it's one directional.
1652	*/
1653	uint64_t const uApicBase = APICGetBaseMsrNoCheck(pVCpu);
1654	if ( (fExtrn & ( CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK \| CPUMCTX_EXTRN_CR_MASK
1655	\| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_APIC_TPR))
1656	\|\| uApicBase != pVCpu->nem.s.uKvmApicBase)
1657	{
1658	if ((pVCpu->nem.s.uKvmApicBase ^ uApicBase) & MSR_IA32_APICBASE_EN)
1659	Log(("NEM/%u: APICBASE_EN changed %#010RX64 -> %#010RX64\n", pVCpu->idCpu, pVCpu->nem.s.uKvmApicBase, uApicBase));
1660	pRun->s.regs.sregs.apic_base = uApicBase;
1661	pVCpu->nem.s.uKvmApicBase = uApicBase;
1662
1663	if (fExtrn & CPUMCTX_EXTRN_APIC_TPR)
1664	pRun->s.regs.sregs.cr8 = CPUMGetGuestCR8(pVCpu);
1665
1666	#define NEM_LNX_EXPORT_SEG(a_KvmSeg, a_CtxSeg) do { \
1667	(a_KvmSeg).base = (a_CtxSeg).u64Base; \
1668	(a_KvmSeg).limit = (a_CtxSeg).u32Limit; \
1669	(a_KvmSeg).selector = (a_CtxSeg).Sel; \
1670	(a_KvmSeg).type = (a_CtxSeg).Attr.n.u4Type; \
1671	(a_KvmSeg).s = (a_CtxSeg).Attr.n.u1DescType; \
1672	(a_KvmSeg).dpl = (a_CtxSeg).Attr.n.u2Dpl; \
1673	(a_KvmSeg).present = (a_CtxSeg).Attr.n.u1Present; \
1674	(a_KvmSeg).avl = (a_CtxSeg).Attr.n.u1Available; \
1675	(a_KvmSeg).l = (a_CtxSeg).Attr.n.u1Long; \
1676	(a_KvmSeg).db = (a_CtxSeg).Attr.n.u1DefBig; \
1677	(a_KvmSeg).g = (a_CtxSeg).Attr.n.u1Granularity; \
1678	(a_KvmSeg).unusable = (a_CtxSeg).Attr.n.u1Unusable; \
1679	(a_KvmSeg).padding = 0; \
1680	} while (0)
1681
1682	if (fExtrn & CPUMCTX_EXTRN_SREG_MASK)
1683	{
1684	if (fExtrn & CPUMCTX_EXTRN_ES)
1685	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.es, pCtx->es);
1686	if (fExtrn & CPUMCTX_EXTRN_CS)
1687	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.cs, pCtx->cs);
1688	if (fExtrn & CPUMCTX_EXTRN_SS)
1689	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ss, pCtx->ss);
1690	if (fExtrn & CPUMCTX_EXTRN_DS)
1691	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ds, pCtx->ds);
1692	if (fExtrn & CPUMCTX_EXTRN_FS)
1693	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.fs, pCtx->fs);
1694	if (fExtrn & CPUMCTX_EXTRN_GS)
1695	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.gs, pCtx->gs);
1696	}
1697	if (fExtrn & CPUMCTX_EXTRN_TABLE_MASK)
1698	{
1699	if (fExtrn & CPUMCTX_EXTRN_GDTR)
1700	{
1701	pRun->s.regs.sregs.gdt.base = pCtx->gdtr.pGdt;
1702	pRun->s.regs.sregs.gdt.limit = pCtx->gdtr.cbGdt;
1703	pRun->s.regs.sregs.gdt.padding[0] = 0;
1704	pRun->s.regs.sregs.gdt.padding[1] = 0;
1705	pRun->s.regs.sregs.gdt.padding[2] = 0;
1706	}
1707	if (fExtrn & CPUMCTX_EXTRN_IDTR)
1708	{
1709	pRun->s.regs.sregs.idt.base = pCtx->idtr.pIdt;
1710	pRun->s.regs.sregs.idt.limit = pCtx->idtr.cbIdt;
1711	pRun->s.regs.sregs.idt.padding[0] = 0;
1712	pRun->s.regs.sregs.idt.padding[1] = 0;
1713	pRun->s.regs.sregs.idt.padding[2] = 0;
1714	}
1715	if (fExtrn & CPUMCTX_EXTRN_LDTR)
1716	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ldt, pCtx->ldtr);
1717	if (fExtrn & CPUMCTX_EXTRN_TR)
1718	NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.tr, pCtx->tr);
1719	}
1720	if (fExtrn & CPUMCTX_EXTRN_CR_MASK)
1721	{
1722	if (fExtrn & CPUMCTX_EXTRN_CR0)
1723	pRun->s.regs.sregs.cr0 = pCtx->cr0;
1724	if (fExtrn & CPUMCTX_EXTRN_CR2)
1725	pRun->s.regs.sregs.cr2 = pCtx->cr2;
1726	if (fExtrn & CPUMCTX_EXTRN_CR3)
1727	pRun->s.regs.sregs.cr3 = pCtx->cr3;
1728	if (fExtrn & CPUMCTX_EXTRN_CR4)
1729	pRun->s.regs.sregs.cr4 = pCtx->cr4;
1730	}
1731	if (fExtrn & CPUMCTX_EXTRN_EFER)
1732	pRun->s.regs.sregs.efer = pCtx->msrEFER;
1733
1734	RT_ZERO(pRun->s.regs.sregs.interrupt_bitmap); /* this is an alternative interrupt injection interface */
1735
1736	pRun->kvm_dirty_regs \|= KVM_SYNC_X86_SREGS;
1737	}
1738
1739	/*
1740	* Debug registers.
1741	*/
1742	if (fExtrn & CPUMCTX_EXTRN_DR_MASK)
1743	{
1744	struct kvm_debugregs DbgRegs = {{0}};
1745
1746	if ((fExtrn & CPUMCTX_EXTRN_DR_MASK) != CPUMCTX_EXTRN_DR_MASK)
1747	{
1748	/* Partial debug state, we must get DbgRegs first so we can merge: */
1749	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
1750	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1751	}
1752
1753	if (fExtrn & CPUMCTX_EXTRN_DR0_DR3)
1754	{
1755	DbgRegs.db[0] = pCtx->dr[0];
1756	DbgRegs.db[1] = pCtx->dr[1];
1757	DbgRegs.db[2] = pCtx->dr[2];
1758	DbgRegs.db[3] = pCtx->dr[3];
1759	}
1760	if (fExtrn & CPUMCTX_EXTRN_DR6)
1761	DbgRegs.dr6 = pCtx->dr[6];
1762	if (fExtrn & CPUMCTX_EXTRN_DR7)
1763	DbgRegs.dr7 = pCtx->dr[7];
1764
1765	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_DEBUGREGS, &DbgRegs);
1766	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1767	}
1768
1769	/*
1770	* FPU, SSE, AVX, ++.
1771	*/
1772	if (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx))
1773	{
1774	if (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1775	{
1776	/** @todo could IEM just grab state partial control in some situations? */
1777	Assert( (fExtrn & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1778	== (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE)); /* no partial states */
1779
1780	AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
1781	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XSAVE, &pCtx->XState);
1782	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1783	}
1784
1785	if (fExtrn & CPUMCTX_EXTRN_XCRx)
1786	{
1787	struct kvm_xcrs Xcrs =
1788	{ /.nr_xcrs = / 2,
1789	/.flags = / 0,
1790	/.xcrs= / {
1791	{ /.xcr =/ 0, /.reserved=/ 0, /.value=/ pCtx->aXcr[0] },
1792	{ /.xcr =/ 1, /.reserved=/ 0, /.value=/ pCtx->aXcr[1] },
1793	}
1794	};
1795
1796	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XCRS, &Xcrs);
1797	AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
1798	}
1799	}
1800
1801	/*
1802	* MSRs.
1803	*/
1804	if (fExtrn & ( CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_SYSENTER_MSRS
1805	\| CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
1806	{
1807	union
1808	{
1809	struct kvm_msrs Core;
1810	uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
1811	} uBuf;
1812	uint32_t iMsr = 0;
1813	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1814
1815	#define ADD_MSR(a_Msr, a_uValue) do { \
1816	Assert(iMsr < 32); \
1817	uBuf.Core.entries[iMsr].index = (a_Msr); \
1818	uBuf.Core.entries[iMsr].reserved = 0; \
1819	uBuf.Core.entries[iMsr].data = (a_uValue); \
1820	iMsr += 1; \
1821	} while (0)
1822
1823	if (fExtrn & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1824	ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1825	if (fExtrn & CPUMCTX_EXTRN_SYSCALL_MSRS)
1826	{
1827	ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
1828	ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
1829	ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
1830	ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1831	}
1832	if (fExtrn & CPUMCTX_EXTRN_SYSENTER_MSRS)
1833	{
1834	ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
1835	ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
1836	ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
1837	}
1838	if (fExtrn & CPUMCTX_EXTRN_TSC_AUX)
1839	ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
1840	if (fExtrn & CPUMCTX_EXTRN_OTHER_MSRS)
1841	{
1842	ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
1843	/** @todo What do we _have_ to add here?
1844	* We also have: Mttr, MiscEnable, FeatureControl. /
1845	}
1846
1847	uBuf.Core.pad = 0;
1848	uBuf.Core.nmsrs = iMsr;
1849	int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_MSRS, &uBuf);
1850	AssertMsgReturn(rc == (int)iMsr,
1851	("rc=%d iMsr=%d (->%#x) errno=%d\n",
1852	rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
1853	VERR_NEM_IPE_3);
1854	}
1855
1856	/*
1857	* Interruptibility state.
1858	*
1859	* Note! This I/O control function sets most fields passed in, so when
1860	* raising an interrupt, NMI, SMI or exception, this must be done
1861	* by the code doing the rasing or we'll overwrite it here.
1862	*/
1863	if (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1864	{
1865	Assert( (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1866	== (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI));
1867
1868	struct kvm_vcpu_events KvmEvents = {0};
1869
1870	KvmEvents.flags = KVM_VCPUEVENT_VALID_SHADOW;
1871	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1872	{
1873	if (pRun->s.regs.regs.rip == EMGetInhibitInterruptsPC(pVCpu))
1874	KvmEvents.interrupt.shadow = KVM_X86_SHADOW_INT_MOV_SS \| KVM_X86_SHADOW_INT_STI;
1875	else
1876	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1877	}
1878
1879	/* No flag - this is updated unconditionally. */
1880	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
1881	KvmEvents.nmi.masked = 1;
1882
1883	if (TRPMHasTrap(pVCpu))
1884	{
1885	TRPMEVENT enmType = TRPM_32BIT_HACK;
1886	uint8_t bTrapNo = 0;
1887	TRPMQueryTrap(pVCpu, &bTrapNo, &enmType);
1888	Log(("nemHCLnxExportState: Pending trap: bTrapNo=%#x enmType=%d\n", bTrapNo, enmType));
1889	if ( enmType == TRPM_HARDWARE_INT
1890	\|\| enmType == TRPM_SOFTWARE_INT)
1891	{
1892	KvmEvents.interrupt.soft = enmType == TRPM_SOFTWARE_INT;
1893	KvmEvents.interrupt.nr = bTrapNo;
1894	KvmEvents.interrupt.injected = 1;
1895	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExportPendingInterrupt);
1896	TRPMResetTrap(pVCpu);
1897	}
1898	else
1899	AssertFailed();
1900	}
1901
1902	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
1903	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
1904	}
1905
1906	/*
1907	* KVM now owns all the state.
1908	*/
1909	pCtx->fExtrn = CPUMCTX_EXTRN_KEEPER_NEM \| CPUMCTX_EXTRN_ALL;
1910
1911	RT_NOREF(pVM);
1912	return VINF_SUCCESS;
1913	}
1914
1915
1916	/**
1917	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
1918	*
1919	* @returns VBox status code.
1920	* @param pVCpu The cross context CPU structure.
1921	* @param pcTicks Where to return the CPU tick count.
1922	* @param puAux Where to return the TSC_AUX register value.
1923	*/
1924	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
1925	{
1926	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
1927	// KVM_GET_CLOCK?
1928	RT_NOREF(pVCpu, pcTicks, puAux);
1929	return VINF_SUCCESS;
1930	}
1931
1932
1933	/**
1934	* Resumes CPU clock (TSC) on all virtual CPUs.
1935	*
1936	* This is called by TM when the VM is started, restored, resumed or similar.
1937	*
1938	* @returns VBox status code.
1939	* @param pVM The cross context VM structure.
1940	* @param pVCpu The cross context CPU structure of the calling EMT.
1941	* @param uPausedTscValue The TSC value at the time of pausing.
1942	*/
1943	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
1944	{
1945	// KVM_SET_CLOCK?
1946	RT_NOREF(pVM, pVCpu, uPausedTscValue);
1947	return VINF_SUCCESS;
1948	}
1949
1950
1951	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
1952	{
1953	RT_NOREF(pVM);
1954	return NEM_FEAT_F_NESTED_PAGING
1955	\| NEM_FEAT_F_FULL_GST_EXEC
1956	\| NEM_FEAT_F_XSAVE_XRSTOR;
1957	}
1958
1959
1960
1961	/*********************************************************************************************************************************
1962	* Execution *
1963	*********************************************************************************************************************************/
1964
1965
1966	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
1967	{
1968	/*
1969	* Only execute when the A20 gate is enabled as I cannot immediately
1970	* spot any A20 support in KVM.
1971	*/
1972	RT_NOREF(pVM);
1973	Assert(VM_IS_NEM_ENABLED(pVM));
1974	return PGMPhysIsA20Enabled(pVCpu);
1975	}
1976
1977
1978	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
1979	{
1980	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
1981	return false;
1982	}
1983
1984
1985	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
1986	{
1987	int rc = RTThreadPoke(pVCpu->hThread);
1988	LogFlow(("nemR3NativeNotifyFF: #%u -> %Rrc\n", pVCpu->idCpu, rc));
1989	AssertRC(rc);
1990	RT_NOREF(pVM, fFlags);
1991	}
1992
1993
1994	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
1995	{
1996	RT_NOREF(pVM, fUseDebugLoop);
1997	return false;
1998	}
1999
2000
2001	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
2002	{
2003	RT_NOREF(pVM, pVCpu, fUseDebugLoop);
2004	return false;
2005	}
2006
2007
2008	/**
2009	* Deals with pending interrupt FFs prior to executing guest code.
2010	*/
2011	static VBOXSTRICTRC nemHCLnxHandleInterruptFF(PVM pVM, PVMCPU pVCpu, struct kvm_run *pRun)
2012	{
2013	RT_NOREF_PV(pVM);
2014
2015	/*
2016	* Do not doing anything if TRPM has something pending already as we can
2017	* only inject one event per KVM_RUN call. This can only happend if we
2018	* can directly from the loop in EM, so the inhibit bits must be internal.
2019	*/
2020	if (!TRPMHasTrap(pVCpu))
2021	{ /* semi likely */ }
2022	else
2023	{
2024	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI)));
2025	Log8(("nemHCLnxHandleInterruptFF: TRPM has an pending event already\n"));
2026	return VINF_SUCCESS;
2027	}
2028
2029	/*
2030	* First update APIC. We ASSUME this won't need TPR/CR8.
2031	*/
2032	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
2033	{
2034	APICUpdatePendingInterrupts(pVCpu);
2035	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
2036	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2037	return VINF_SUCCESS;
2038	}
2039
2040	/*
2041	* We don't currently implement SMIs.
2042	*/
2043	AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
2044
2045	/*
2046	* In KVM the CPUMCTX_EXTRN_INHIBIT_INT and CPUMCTX_EXTRN_INHIBIT_NMI states
2047	* are tied together with interrupt and NMI delivery, so we must get and
2048	* synchronize these all in one go and set both CPUMCTX_EXTRN_INHIBIT_XXX flags.
2049	* If we don't we may lose the interrupt/NMI we marked pending here when the
2050	* state is exported again before execution.
2051	*/
2052	struct kvm_vcpu_events KvmEvents = {0};
2053	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
2054	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2055
2056	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
2057	pRun->s.regs.regs.rip = pVCpu->cpum.GstCtx.rip;
2058
2059	KvmEvents.flags \|= KVM_VCPUEVENT_VALID_SHADOW;
2060	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_INT))
2061	{
2062	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2063	KvmEvents.interrupt.shadow = 0;
2064	else if (EMGetInhibitInterruptsPC(pVCpu) == pRun->s.regs.regs.rip)
2065	KvmEvents.interrupt.shadow = KVM_X86_SHADOW_INT_MOV_SS \| KVM_X86_SHADOW_INT_STI;
2066	else
2067	{
2068	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2069	KvmEvents.interrupt.shadow = 0;
2070	}
2071	}
2072	else if (KvmEvents.interrupt.shadow)
2073	EMSetInhibitInterruptsPC(pVCpu, pRun->s.regs.regs.rip);
2074	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2075	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2076
2077	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_NMI))
2078	KvmEvents.nmi.masked = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS) ? 1 : 0;
2079	else if (KvmEvents.nmi.masked)
2080	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
2081	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
2082	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
2083
2084	/* KVM will own the INT + NMI inhibit state soon: */
2085	pVCpu->cpum.GstCtx.fExtrn = (pVCpu->cpum.GstCtx.fExtrn & ~CPUMCTX_EXTRN_KEEPER_MASK)
2086	\| CPUMCTX_EXTRN_KEEPER_NEM \| CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI;
2087
2088	/*
2089	* NMI? Try deliver it first.
2090	*/
2091	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
2092	{
2093	#if 0
2094	int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_NMI, 0UL);
2095	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2096	#else
2097	KvmEvents.flags \|= KVM_VCPUEVENT_VALID_NMI_PENDING;
2098	KvmEvents.nmi.pending = 1;
2099	#endif
2100	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
2101	Log8(("Queuing NMI on %u\n", pVCpu->idCpu));
2102	}
2103
2104	/*
2105	* APIC or PIC interrupt?
2106	*/
2107	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
2108	{
2109	if (pRun->s.regs.regs.rflags & X86_EFL_IF)
2110	{
2111	if (KvmEvents.interrupt.shadow == 0)
2112	{
2113	/*
2114	* If CR8 is in KVM, update the VBox copy so PDMGetInterrupt will
2115	* work correctly.
2116	*/
2117	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_APIC_TPR)
2118	APICSetTpr(pVCpu, (uint8_t)pRun->cr8 << 4);
2119
2120	uint8_t bInterrupt;
2121	int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
2122	if (RT_SUCCESS(rc))
2123	{
2124	Assert(KvmEvents.interrupt.injected == false);
2125	#if 0
2126	int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_INTERRUPT, (unsigned long)bInterrupt);
2127	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2128	#else
2129	KvmEvents.interrupt.nr = bInterrupt;
2130	KvmEvents.interrupt.soft = false;
2131	KvmEvents.interrupt.injected = true;
2132	#endif
2133	Log8(("Queuing interrupt %#x on %u: %04x:%08RX64 efl=%#x\n", bInterrupt, pVCpu->idCpu,
2134	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags));
2135	}
2136	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR) /** @todo this isn't extremely efficient if we get a lot of exits... */
2137	Log8(("VERR_APIC_INTR_MASKED_BY_TPR\n")); /* We'll get a TRP exit - no interrupt window needed. */
2138	else
2139	Log8(("PDMGetInterrupt failed -> %Rrc\n", rc));
2140	}
2141	else
2142	{
2143	pRun->request_interrupt_window = 1;
2144	Log8(("Interrupt window pending on %u (#2)\n", pVCpu->idCpu));
2145	}
2146	}
2147	else
2148	{
2149	pRun->request_interrupt_window = 1;
2150	Log8(("Interrupt window pending on %u (#1)\n", pVCpu->idCpu));
2151	}
2152	}
2153
2154	/*
2155	* Now, update the state.
2156	*/
2157	/** @todo skip when possible... */
2158	rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
2159	AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
2160
2161	return VINF_SUCCESS;
2162	}
2163
2164
2165	/**
2166	* Handles KVM_EXIT_INTERNAL_ERROR.
2167	*/
2168	static VBOXSTRICTRC nemR3LnxHandleInternalError(PVMCPU pVCpu, struct kvm_run *pRun)
2169	{
2170	Log(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%d) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror,
2171	pRun->internal.suberror, pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
2172
2173	/*
2174	* Deal with each suberror, returning if we don't want IEM to handle it.
2175	*/
2176	switch (pRun->internal.suberror)
2177	{
2178	case KVM_INTERNAL_ERROR_EMULATION:
2179	{
2180	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_EMULATION),
2181	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2182	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorEmulation);
2183	break;
2184	}
2185
2186	case KVM_INTERNAL_ERROR_SIMUL_EX:
2187	case KVM_INTERNAL_ERROR_DELIVERY_EV:
2188	case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON:
2189	default:
2190	{
2191	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_FATAL),
2192	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2193	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorFatal);
2194	const char *pszName;
2195	switch (pRun->internal.suberror)
2196	{
2197	case KVM_INTERNAL_ERROR_EMULATION: pszName = "KVM_INTERNAL_ERROR_EMULATION"; break;
2198	case KVM_INTERNAL_ERROR_SIMUL_EX: pszName = "KVM_INTERNAL_ERROR_SIMUL_EX"; break;
2199	case KVM_INTERNAL_ERROR_DELIVERY_EV: pszName = "KVM_INTERNAL_ERROR_DELIVERY_EV"; break;
2200	case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON: pszName = "KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON"; break;
2201	default: pszName = "unknown"; break;
2202	}
2203	LogRel(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%s) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror, pszName,
2204	pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
2205	return VERR_NEM_IPE_0;
2206	}
2207	}
2208
2209	/*
2210	* Execute instruction in IEM and try get on with it.
2211	*/
2212	Log2(("nemR3LnxHandleInternalError: Executing instruction at %04x:%08RX64 in IEM\n",
2213	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip));
2214	VBOXSTRICTRC rcStrict = nemHCLnxImportState(pVCpu,
2215	IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_INHIBIT_INT
2216	\| CPUMCTX_EXTRN_INHIBIT_NMI,
2217	&pVCpu->cpum.GstCtx, pRun);
2218	if (RT_SUCCESS(rcStrict))
2219	rcStrict = IEMExecOne(pVCpu);
2220	return rcStrict;
2221	}
2222
2223
2224	/**
2225	* Handles KVM_EXIT_IO.
2226	*/
2227	static VBOXSTRICTRC nemHCLnxHandleExitIo(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
2228	{
2229	/*
2230	* Input validation.
2231	*/
2232	Assert(pRun->io.count > 0);
2233	Assert(pRun->io.size == 1 \|\| pRun->io.size == 2 \|\| pRun->io.size == 4);
2234	Assert(pRun->io.direction == KVM_EXIT_IO_IN \|\| pRun->io.direction == KVM_EXIT_IO_OUT);
2235	Assert(pRun->io.data_offset < pVM->nem.s.cbVCpuMmap);
2236	Assert(pRun->io.data_offset + pRun->io.size * pRun->io.count <= pVM->nem.s.cbVCpuMmap);
2237
2238	/*
2239	* We cannot easily act on the exit history here, because the I/O port
2240	* exit is stateful and the instruction will be completed in the next
2241	* KVM_RUN call. There seems no way to avoid this.
2242	*/
2243	EMHistoryAddExit(pVCpu,
2244	pRun->io.count == 1
2245	? ( pRun->io.direction == KVM_EXIT_IO_IN
2246	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ)
2247	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE))
2248	: ( pRun->io.direction == KVM_EXIT_IO_IN
2249	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)
2250	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)),
2251	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2252
2253	/*
2254	* Do the requested job.
2255	*/
2256	VBOXSTRICTRC rcStrict;
2257	RTPTRUNION uPtrData;
2258	uPtrData.pu8 = (uint8_t *)pRun + pRun->io.data_offset;
2259	if (pRun->io.count == 1)
2260	{
2261	if (pRun->io.direction == KVM_EXIT_IO_IN)
2262	{
2263	uint32_t uValue = 0;
2264	rcStrict = IOMIOPortRead(pVM, pVCpu, pRun->io.port, &uValue, pRun->io.size);
2265	Log4(("IOExit/%u: %04x:%08RX64: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2266	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2267	pRun->io.port, pRun->io.size, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2268	if (IOM_SUCCESS(rcStrict))
2269	{
2270	if (pRun->io.size == 4)
2271	*uPtrData.pu32 = uValue;
2272	else if (pRun->io.size == 2)
2273	*uPtrData.pu16 = (uint16_t)uValue;
2274	else
2275	*uPtrData.pu8 = (uint8_t)uValue;
2276	}
2277	}
2278	else
2279	{
2280	uint32_t const uValue = pRun->io.size == 4 ? *uPtrData.pu32
2281	: pRun->io.size == 2 ? *uPtrData.pu16
2282	: *uPtrData.pu8;
2283	rcStrict = IOMIOPortWrite(pVM, pVCpu, pRun->io.port, uValue, pRun->io.size);
2284	Log4(("IOExit/%u: %04x:%08RX64: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2285	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2286	pRun->io.port, uValue, pRun->io.size, VBOXSTRICTRC_VAL(rcStrict) ));
2287	}
2288	}
2289	else
2290	{
2291	uint32_t cTransfers = pRun->io.count;
2292	if (pRun->io.direction == KVM_EXIT_IO_IN)
2293	{
2294	rcStrict = IOMIOPortReadString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
2295	Log4(("IOExit/%u: %04x:%08RX64: REP INS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
2296	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2297	pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
2298	}
2299	else
2300	{
2301	rcStrict = IOMIOPortWriteString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
2302	Log4(("IOExit/%u: %04x:%08RX64: REP OUTS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
2303	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2304	pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
2305	}
2306	Assert(cTransfers == 0);
2307	}
2308	return rcStrict;
2309	}
2310
2311
2312	/**
2313	* Handles KVM_EXIT_MMIO.
2314	*/
2315	static VBOXSTRICTRC nemHCLnxHandleExitMmio(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
2316	{
2317	/*
2318	* Input validation.
2319	*/
2320	Assert(pRun->mmio.len <= sizeof(pRun->mmio.data));
2321	Assert(pRun->mmio.is_write <= 1);
2322
2323	/*
2324	* We cannot easily act on the exit history here, because the MMIO port
2325	* exit is stateful and the instruction will be completed in the next
2326	* KVM_RUN call. There seems no way to circumvent this.
2327	*/
2328	EMHistoryAddExit(pVCpu,
2329	pRun->mmio.is_write
2330	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2331	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2332	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2333
2334	/*
2335	* Do the requested job.
2336	*/
2337	VBOXSTRICTRC rcStrict;
2338	if (pRun->mmio.is_write)
2339	{
2340	rcStrict = PGMPhysWrite(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
2341	Log4(("MmioExit/%u: %04x:%08RX64: WRITE %#x LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
2342	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2343	pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
2344	}
2345	else
2346	{
2347	rcStrict = PGMPhysRead(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
2348	Log4(("MmioExit/%u: %04x:%08RX64: READ %#x LB %u -> %.*Rhxs rcStrict=%Rrc\n",
2349	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2350	pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
2351	}
2352	return rcStrict;
2353	}
2354
2355
2356	/**
2357	* Handles KVM_EXIT_RDMSR
2358	*/
2359	static VBOXSTRICTRC nemHCLnxHandleExitRdMsr(PVMCPUCC pVCpu, struct kvm_run *pRun)
2360	{
2361	/*
2362	* Input validation.
2363	*/
2364	Assert( pRun->msr.reason == KVM_MSR_EXIT_REASON_INVAL
2365	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_UNKNOWN
2366	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
2367
2368	/*
2369	* We cannot easily act on the exit history here, because the MSR exit is
2370	* stateful and the instruction will be completed in the next KVM_RUN call.
2371	* There seems no way to circumvent this.
2372	*/
2373	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2374	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2375
2376	/*
2377	* Do the requested job.
2378	*/
2379	uint64_t uValue = 0;
2380	VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pRun->msr.index, &uValue);
2381	pRun->msr.data = uValue;
2382	if (rcStrict != VERR_CPUM_RAISE_GP_0)
2383	{
2384	Log3(("MsrRead/%u: %04x:%08RX64: msr=%#010x (reason=%#x) -> %#RX64 rcStrict=%Rrc\n", pVCpu->idCpu,
2385	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.reason, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2386	pRun->msr.error = 0;
2387	}
2388	else
2389	{
2390	Log3(("MsrRead/%u: %04x:%08RX64: msr=%#010x (reason%#x)-> %#RX64 rcStrict=#GP!\n", pVCpu->idCpu,
2391	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.reason, uValue));
2392	pRun->msr.error = 1;
2393	rcStrict = VINF_SUCCESS;
2394	}
2395	return rcStrict;
2396	}
2397
2398
2399	/**
2400	* Handles KVM_EXIT_WRMSR
2401	*/
2402	static VBOXSTRICTRC nemHCLnxHandleExitWrMsr(PVMCPUCC pVCpu, struct kvm_run *pRun)
2403	{
2404	/*
2405	* Input validation.
2406	*/
2407	Assert( pRun->msr.reason == KVM_MSR_EXIT_REASON_INVAL
2408	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_UNKNOWN
2409	\|\| pRun->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
2410
2411	/*
2412	* We cannot easily act on the exit history here, because the MSR exit is
2413	* stateful and the instruction will be completed in the next KVM_RUN call.
2414	* There seems no way to circumvent this.
2415	*/
2416	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE),
2417	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2418
2419	/*
2420	* Do the requested job.
2421	*/
2422	VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pRun->msr.index, pRun->msr.data);
2423	if (rcStrict != VERR_CPUM_RAISE_GP_0)
2424	{
2425	Log3(("MsrWrite/%u: %04x:%08RX64: msr=%#010x := %#RX64 (reason=%#x) -> rcStrict=%Rrc\n", pVCpu->idCpu,
2426	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.data, pRun->msr.reason, VBOXSTRICTRC_VAL(rcStrict) ));
2427	pRun->msr.error = 0;
2428	}
2429	else
2430	{
2431	Log3(("MsrWrite/%u: %04x:%08RX64: msr=%#010x := %#RX64 (reason%#x)-> rcStrict=#GP!\n", pVCpu->idCpu,
2432	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.data, pRun->msr.reason));
2433	pRun->msr.error = 1;
2434	rcStrict = VINF_SUCCESS;
2435	}
2436	return rcStrict;
2437	}
2438
2439
2440
2441	static VBOXSTRICTRC nemHCLnxHandleExit(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run pRun, bool pfStatefulExit)
2442	{
2443	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTotal);
2444	switch (pRun->exit_reason)
2445	{
2446	case KVM_EXIT_EXCEPTION:
2447	AssertFailed();
2448	break;
2449
2450	case KVM_EXIT_IO:
2451	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIo);
2452	*pfStatefulExit = true;
2453	return nemHCLnxHandleExitIo(pVM, pVCpu, pRun);
2454
2455	case KVM_EXIT_MMIO:
2456	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMmio);
2457	*pfStatefulExit = true;
2458	return nemHCLnxHandleExitMmio(pVM, pVCpu, pRun);
2459
2460	case KVM_EXIT_IRQ_WINDOW_OPEN:
2461	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2462	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2463	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIrqWindowOpen);
2464	Log5(("IrqWinOpen/%u: %d\n", pVCpu->idCpu, pRun->request_interrupt_window));
2465	pRun->request_interrupt_window = 0;
2466	return VINF_SUCCESS;
2467
2468	case KVM_EXIT_SET_TPR:
2469	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitSetTpr);
2470	AssertFailed();
2471	break;
2472
2473	case KVM_EXIT_TPR_ACCESS:
2474	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTprAccess);
2475	AssertFailed();
2476	break;
2477
2478	case KVM_EXIT_X86_RDMSR:
2479	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitRdMsr);
2480	*pfStatefulExit = true;
2481	return nemHCLnxHandleExitRdMsr(pVCpu, pRun);
2482
2483	case KVM_EXIT_X86_WRMSR:
2484	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitWrMsr);
2485	*pfStatefulExit = true;
2486	return nemHCLnxHandleExitWrMsr(pVCpu, pRun);
2487
2488	case KVM_EXIT_HLT:
2489	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
2490	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2491	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
2492	Log5(("Halt/%u\n", pVCpu->idCpu));
2493	return VINF_EM_HALT;
2494
2495	case KVM_EXIT_INTR: /* EINTR */
2496	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERRUPTED),
2497	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2498	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIntr);
2499	Log5(("Intr/%u\n", pVCpu->idCpu));
2500	return VINF_SUCCESS;
2501
2502	case KVM_EXIT_HYPERCALL:
2503	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHypercall);
2504	AssertFailed();
2505	break;
2506
2507	case KVM_EXIT_DEBUG:
2508	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitDebug);
2509	AssertFailed();
2510	break;
2511
2512	case KVM_EXIT_SYSTEM_EVENT:
2513	AssertFailed();
2514	break;
2515	case KVM_EXIT_IOAPIC_EOI:
2516	AssertFailed();
2517	break;
2518	case KVM_EXIT_HYPERV:
2519	AssertFailed();
2520	break;
2521
2522	case KVM_EXIT_DIRTY_RING_FULL:
2523	AssertFailed();
2524	break;
2525	case KVM_EXIT_AP_RESET_HOLD:
2526	AssertFailed();
2527	break;
2528	case KVM_EXIT_X86_BUS_LOCK:
2529	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitBusLock);
2530	AssertFailed();
2531	break;
2532
2533
2534	case KVM_EXIT_SHUTDOWN:
2535	AssertFailed();
2536	break;
2537
2538	case KVM_EXIT_FAIL_ENTRY:
2539	LogRel(("NEM: KVM_EXIT_FAIL_ENTRY! hardware_entry_failure_reason=%#x cpu=%#x\n",
2540	pRun->fail_entry.hardware_entry_failure_reason, pRun->fail_entry.cpu));
2541	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_FAILED_ENTRY),
2542	pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
2543	return VERR_NEM_IPE_1;
2544
2545	case KVM_EXIT_INTERNAL_ERROR:
2546	/* we're counting sub-reasons inside the function. */
2547	return nemR3LnxHandleInternalError(pVCpu, pRun);
2548
2549	/*
2550	* Foreign and unknowns.
2551	*/
2552	case KVM_EXIT_NMI:
2553	AssertLogRelMsgFailedReturn(("KVM_EXIT_NMI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2554	case KVM_EXIT_EPR:
2555	AssertLogRelMsgFailedReturn(("KVM_EXIT_EPR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2556	case KVM_EXIT_WATCHDOG:
2557	AssertLogRelMsgFailedReturn(("KVM_EXIT_WATCHDOG on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2558	case KVM_EXIT_ARM_NISV:
2559	AssertLogRelMsgFailedReturn(("KVM_EXIT_ARM_NISV on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2560	case KVM_EXIT_S390_STSI:
2561	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_STSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2562	case KVM_EXIT_S390_TSCH:
2563	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_TSCH on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2564	case KVM_EXIT_OSI:
2565	AssertLogRelMsgFailedReturn(("KVM_EXIT_OSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2566	case KVM_EXIT_PAPR_HCALL:
2567	AssertLogRelMsgFailedReturn(("KVM_EXIT_PAPR_HCALL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2568	case KVM_EXIT_S390_UCONTROL:
2569	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_UCONTROL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2570	case KVM_EXIT_DCR:
2571	AssertLogRelMsgFailedReturn(("KVM_EXIT_DCR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2572	case KVM_EXIT_S390_SIEIC:
2573	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_SIEIC on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2574	case KVM_EXIT_S390_RESET:
2575	AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_RESET on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2576	case KVM_EXIT_UNKNOWN:
2577	AssertLogRelMsgFailedReturn(("KVM_EXIT_UNKNOWN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2578	case KVM_EXIT_XEN:
2579	AssertLogRelMsgFailedReturn(("KVM_EXIT_XEN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2580	default:
2581	AssertLogRelMsgFailedReturn(("Unknown exit reason %u on VCpu #%u at %04x:%RX64!\n", pRun->exit_reason, pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
2582	}
2583
2584	RT_NOREF(pVM, pVCpu, pRun);
2585	return VERR_NOT_IMPLEMENTED;
2586	}
2587
2588
2589	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2590	{
2591	/*
2592	* Try switch to NEM runloop state.
2593	*/
2594	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2595	{ /* likely */ }
2596	else
2597	{
2598	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2599	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
2600	return VINF_SUCCESS;
2601	}
2602
2603	/*
2604	* The run loop.
2605	*/
2606	struct kvm_run * const pRun = pVCpu->nem.s.pRun;
2607	const bool fSingleStepping = DBGFIsStepping(pVCpu);
2608	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2609	bool fStatefulExit = false; /* For MMIO and IO exits. */
2610	for (unsigned iLoop = 0;; iLoop++)
2611	{
2612	/*
2613	* Pending interrupts or such? Need to check and deal with this prior
2614	* to the state syncing.
2615	*/
2616	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_UPDATE_APIC \| VMCPU_FF_INTERRUPT_PIC
2617	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2618	{
2619	/* Try inject interrupt. */
2620	rcStrict = nemHCLnxHandleInterruptFF(pVM, pVCpu, pRun);
2621	if (rcStrict == VINF_SUCCESS)
2622	{ /* likely */ }
2623	else
2624	{
2625	LogFlow(("NEM/%u: breaking: nemHCLnxHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2626	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2627	break;
2628	}
2629	}
2630
2631	/*
2632	* Do not execute in KVM if the A20 isn't enabled.
2633	*/
2634	if (PGMPhysIsA20Enabled(pVCpu))
2635	{ /* likely */ }
2636	else
2637	{
2638	rcStrict = VINF_EM_RESCHEDULE_REM;
2639	LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
2640	break;
2641	}
2642
2643	/*
2644	* Ensure KVM has the whole state.
2645	*/
2646	if ((pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL)
2647	{
2648	int rc2 = nemHCLnxExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx, pRun);
2649	AssertRCReturn(rc2, rc2);
2650	}
2651
2652	/*
2653	* Poll timers and run for a bit.
2654	*
2655	* With the VID approach (ring-0 or ring-3) we can specify a timeout here,
2656	* so we take the time of the next timer event and uses that as a deadline.
2657	* The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
2658	*/
2659	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
2660	* the whole polling job when timers have changed... */
2661	uint64_t offDeltaIgnored;
2662	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
2663	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
2664	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2665	{
2666	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
2667	{
2668	LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
2669	pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
2670	!!(pRun->s.regs.regs.rflags & X86_EFL_IF), pRun->s.regs.regs.rflags,
2671	pRun->s.regs.sregs.ss.selector, pRun->s.regs.regs.rsp, pRun->s.regs.sregs.cr0));
2672	TMNotifyStartOfExecution(pVM, pVCpu);
2673
2674	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
2675
2676	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
2677	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
2678
2679	#ifdef LOG_ENABLED
2680	if (LogIsFlowEnabled())
2681	{
2682	struct kvm_mp_state MpState = {UINT32_MAX};
2683	ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MP_STATE, &MpState);
2684	LogFlow(("NEM/%u: Exit @ %04x:%08RX64 IF=%d EFL=%#RX64 CR8=%#x Reason=%#x IrqReady=%d Flags=%#x %#lx\n", pVCpu->idCpu,
2685	pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->if_flag,
2686	pRun->s.regs.regs.rflags, pRun->s.regs.sregs.cr8, pRun->exit_reason,
2687	pRun->ready_for_interrupt_injection, pRun->flags, MpState.mp_state));
2688	}
2689	#endif
2690	fStatefulExit = false;
2691	if (RT_LIKELY(rcLnx == 0 \|\| errno == EINTR))
2692	{
2693	/*
2694	* Deal with the exit.
2695	*/
2696	rcStrict = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
2697	if (rcStrict == VINF_SUCCESS)
2698	{ /* hopefully likely */ }
2699	else
2700	{
2701	LogFlow(("NEM/%u: breaking: nemHCLnxHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2702	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2703	break;
2704	}
2705	}
2706	else
2707	{
2708	int rc2 = RTErrConvertFromErrno(errno);
2709	AssertLogRelMsgFailedReturn(("KVM_RUN failed: rcLnx=%d errno=%u rc=%Rrc\n", rcLnx, errno, rc2), rc2);
2710	}
2711
2712	/*
2713	* If no relevant FFs are pending, loop.
2714	*/
2715	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2716	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2717	{ /* likely */ }
2718	else
2719	{
2720
2721	/** @todo Try handle pending flags, not just return to EM loops. Take care
2722	* not to set important RCs here unless we've handled an exit. */
2723	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
2724	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
2725	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
2726	break;
2727	}
2728	}
2729	else
2730	{
2731	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
2732	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
2733	break;
2734	}
2735	}
2736	else
2737	{
2738	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
2739	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
2740	break;
2741	}
2742	} /* the run loop */
2743
2744
2745	/*
2746	* If the last exit was stateful, commit the state we provided before
2747	* returning to the EM loop so we have a consistent state and can safely
2748	* be rescheduled and whatnot. This may require us to make multiple runs
2749	* for larger MMIO and I/O operations. Sigh^3.
2750	*
2751	* Note! There is no 'ing way to reset the kernel side completion callback
2752	* for these stateful i/o exits. Very annoying interface.
2753	*/
2754	/** @todo check how this works with string I/O and string MMIO. */
2755	if (fStatefulExit && RT_SUCCESS(rcStrict))
2756	{
2757	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn);
2758	uint32_t const uOrgExit = pRun->exit_reason;
2759	for (uint32_t i = 0; ; i++)
2760	{
2761	pRun->immediate_exit = 1;
2762	int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
2763	Log(("NEM/%u: Flushed stateful exit -> %d/%d exit_reason=%d\n", pVCpu->idCpu, rcLnx, errno, pRun->exit_reason));
2764	if (rcLnx == -1 && errno == EINTR)
2765	{
2766	switch (i)
2767	{
2768	case 0: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn1Loop); break;
2769	case 1: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn2Loops); break;
2770	case 2: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn3Loops); break;
2771	default: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn4PlusLoops); break;
2772	}
2773	break;
2774	}
2775	AssertLogRelMsgBreakStmt(rcLnx == 0 && pRun->exit_reason == uOrgExit,
2776	("rcLnx=%d errno=%d exit_reason=%d uOrgExit=%d\n", rcLnx, errno, pRun->exit_reason, uOrgExit),
2777	rcStrict = VERR_NEM_IPE_6);
2778	VBOXSTRICTRC rcStrict2 = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
2779	if (rcStrict2 == VINF_SUCCESS \|\| rcStrict2 == rcStrict)
2780	{ /* likely */ }
2781	else if (RT_FAILURE(rcStrict2))
2782	{
2783	rcStrict = rcStrict2;
2784	break;
2785	}
2786	else
2787	{
2788	AssertLogRelMsgBreakStmt(rcStrict == VINF_SUCCESS,
2789	("rcStrict=%Rrc rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2)),
2790	rcStrict = VERR_NEM_IPE_7);
2791	rcStrict = rcStrict2;
2792	}
2793	}
2794	pRun->immediate_exit = 0;
2795	}
2796
2797	/*
2798	* If the CPU is running, make sure to stop it before we try sync back the
2799	* state and return to EM. We don't sync back the whole state if we can help it.
2800	*/
2801	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
2802	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2803
2804	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
2805	{
2806	/* Try anticipate what we might need. */
2807	uint64_t fImport = CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI /* Required for processing APIC,PIC,NMI & SMI FFs. */
2808	\| IEM_CPUMCTX_EXTRN_MUST_MASK /?/;
2809	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
2810	\|\| RT_FAILURE(rcStrict))
2811	fImport = CPUMCTX_EXTRN_ALL;
2812	# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
2813	else if ( rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
2814	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
2815	fImport = CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS;
2816	else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
2817	fImport = CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS;
2818	# endif
2819	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
2820	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2821	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
2822
2823	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
2824	{
2825	int rc2 = nemHCLnxImportState(pVCpu, fImport, &pVCpu->cpum.GstCtx, pRun);
2826	if (RT_SUCCESS(rc2))
2827	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
2828	else if (RT_SUCCESS(rcStrict))
2829	rcStrict = rc2;
2830	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
2831	pVCpu->cpum.GstCtx.fExtrn = 0;
2832	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
2833	}
2834	else
2835	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2836	}
2837	else
2838	{
2839	pVCpu->cpum.GstCtx.fExtrn = 0;
2840	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2841	}
2842
2843	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
2844	pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
2845	return rcStrict;
2846	}
2847
2848
2849	/** @page pg_nem_linux NEM/linux - Native Execution Manager, Linux.
2850	*
2851	* This is using KVM.
2852	*
2853	*/
2854

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-linux.cpp@ 96789

Download in other formats: