NEMAllNativeTemplate-win.cpp.h@ 75249

Last change on this file since 75249 was 74798, checked in by vboxsync, 6 years ago
vm.h,VMM: Prep work for 64-bit VMCPU::fLocalForcedActions. Define VMCPU_WITH_64_BIT_FFS to enable. bugref:9180
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 231.0 KB

Line
1	/* $Id: NEMAllNativeTemplate-win.cpp.h 74798 2018-10-12 12:25:41Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, Windows code template ring-0/3.
4	*/
5
6	/*
7	* Copyright (C) 2018 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Defined Constants And Macros *
21	*********************************************************************************************************************************/
22	/** Copy back a segment from hyper-V. */
23	#define NEM_WIN_COPY_BACK_SEG(a_Dst, a_Src) \
24	do { \
25	(a_Dst).u64Base = (a_Src).Base; \
26	(a_Dst).u32Limit = (a_Src).Limit; \
27	(a_Dst).ValidSel = (a_Dst).Sel = (a_Src).Selector; \
28	(a_Dst).Attr.u = (a_Src).Attributes; \
29	(a_Dst).fFlags = CPUMSELREG_FLAGS_VALID; \
30	} while (0)
31
32	/** @def NEMWIN_ASSERT_MSG_REG_VAL
33	* Asserts the correctness of a register value in a message/context.
34	*/
35	#if 0
36	# define NEMWIN_NEED_GET_REGISTER
37	# if defined(IN_RING0) \|\| defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
38	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_pGVCpu, a_enmReg, a_Expr, a_Msg) \
39	do { \
40	HV_REGISTER_VALUE TmpVal; \
41	nemHCWinGetRegister(a_pVCpu, a_pGVCpu, a_enmReg, &TmpVal); \
42	AssertMsg(a_Expr, a_Msg); \
43	} while (0)
44	# else
45	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_pGVCpu, a_enmReg, a_Expr, a_Msg) \
46	do { \
47	WHV_REGISTER_VALUE TmpVal; \
48	nemR3WinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
49	AssertMsg(a_Expr, a_Msg); \
50	} while (0)
51	# endif
52	#else
53	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_pGVCpu, a_enmReg, a_Expr, a_Msg) do { } while (0)
54	#endif
55
56	/** @def NEMWIN_ASSERT_MSG_REG_VAL
57	* Asserts the correctness of a 64-bit register value in a message/context.
58	*/
59	#define NEMWIN_ASSERT_MSG_REG_VAL64(a_pVCpu, a_pGVCpu, a_enmReg, a_u64Val) \
60	NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_pGVCpu, a_enmReg, (a_u64Val) == TmpVal.Reg64, \
61	(#a_u64Val "=%#RX64, expected %#RX64\n", (a_u64Val), TmpVal.Reg64))
62	/** @def NEMWIN_ASSERT_MSG_REG_VAL
63	* Asserts the correctness of a segment register value in a message/context.
64	*/
65	#define NEMWIN_ASSERT_MSG_REG_SEG(a_pVCpu, a_pGVCpu, a_enmReg, a_SReg) \
66	NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_pGVCpu, a_enmReg, \
67	(a_SReg).Base == TmpVal.Segment.Base \
68	&& (a_SReg).Limit == TmpVal.Segment.Limit \
69	&& (a_SReg).Selector == TmpVal.Segment.Selector \
70	&& (a_SReg).Attributes == TmpVal.Segment.Attributes, \
71	( #a_SReg "=%#RX16 {%#RX64 LB %#RX32,%#RX16} expected %#RX16 {%#RX64 LB %#RX32,%#RX16}\n", \
72	(a_SReg).Selector, (a_SReg).Base, (a_SReg).Limit, (a_SReg).Attributes, \
73	TmpVal.Segment.Selector, TmpVal.Segment.Base, TmpVal.Segment.Limit, TmpVal.Segment.Attributes))
74
75
76	/*********************************************************************************************************************************
77	* Global Variables *
78	*********************************************************************************************************************************/
79	/** NEM_WIN_PAGE_STATE_XXX names. */
80	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
81
82	/** HV_INTERCEPT_ACCESS_TYPE names. */
83	static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" };
84
85
86	/*********************************************************************************************************************************
87	* Internal Functions *
88	*********************************************************************************************************************************/
89	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
90	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged);
91
92
93
94	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
95
96	/**
97	* Wrapper around VMMR0_DO_NEM_MAP_PAGES for a single page.
98	*
99	* @returns VBox status code.
100	* @param pVM The cross context VM structure.
101	* @param pVCpu The cross context virtual CPU structure of the caller.
102	* @param GCPhysSrc The source page. Does not need to be page aligned.
103	* @param GCPhysDst The destination page. Same as @a GCPhysSrc except for
104	* when A20 is disabled.
105	* @param fFlags HV_MAP_GPA_XXX.
106	*/
107	DECLINLINE(int) nemHCWinHypercallMapPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fFlags)
108	{
109	#ifdef IN_RING0
110	/** @todo optimize further, caller generally has the physical address. */
111	PGVM pGVM = GVMMR0FastGetGVMByVM(pVM);
112	AssertReturn(pGVM, VERR_INVALID_VM_HANDLE);
113	return nemR0WinMapPages(pGVM, pVM, &pGVM->aCpus[pVCpu->idCpu],
114	GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
115	GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
116	1, fFlags);
117	#else
118	pVCpu->nem.s.Hypercall.MapPages.GCPhysSrc = GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
119	pVCpu->nem.s.Hypercall.MapPages.GCPhysDst = GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
120	pVCpu->nem.s.Hypercall.MapPages.cPages = 1;
121	pVCpu->nem.s.Hypercall.MapPages.fFlags = fFlags;
122	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_MAP_PAGES, 0, NULL);
123	#endif
124	}
125
126
127	/**
128	* Wrapper around VMMR0_DO_NEM_UNMAP_PAGES for a single page.
129	*
130	* @returns VBox status code.
131	* @param pVM The cross context VM structure.
132	* @param pVCpu The cross context virtual CPU structure of the caller.
133	* @param GCPhys The page to unmap. Does not need to be page aligned.
134	*/
135	DECLINLINE(int) nemHCWinHypercallUnmapPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
136	{
137	# ifdef IN_RING0
138	PGVM pGVM = GVMMR0FastGetGVMByVM(pVM);
139	AssertReturn(pGVM, VERR_INVALID_VM_HANDLE);
140	return nemR0WinUnmapPages(pGVM, &pGVM->aCpus[pVCpu->idCpu], GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, 1);
141	# else
142	pVCpu->nem.s.Hypercall.UnmapPages.GCPhys = GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
143	pVCpu->nem.s.Hypercall.UnmapPages.cPages = 1;
144	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_UNMAP_PAGES, 0, NULL);
145	# endif
146	}
147
148	#endif /* NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
149	#ifndef IN_RING0
150
151	NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVM pVM, PVMCPU pVCpu)
152	{
153	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
154	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
155	if (pVM->nem.s.fUseRing0Runloop)
156	# endif
157	{
158	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPORT_STATE, 0, NULL);
159	AssertLogRelRCReturn(rc, rc);
160	return rc;
161	}
162	# endif
163	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
164
165	/*
166	* The following is very similar to what nemR0WinExportState() does.
167	*/
168	WHV_REGISTER_NAME aenmNames[128];
169	WHV_REGISTER_VALUE aValues[128];
170
171	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK);
172	if ( !fWhat
173	&& pVCpu->nem.s.fCurrentInterruptWindows == pVCpu->nem.s.fDesiredInterruptWindows)
174	return VINF_SUCCESS;
175	uintptr_t iReg = 0;
176
177	# define ADD_REG64(a_enmName, a_uValue) do { \
178	aenmNames[iReg] = (a_enmName); \
179	aValues[iReg].Reg128.High64 = 0; \
180	aValues[iReg].Reg64 = (a_uValue); \
181	iReg++; \
182	} while (0)
183	# define ADD_REG128(a_enmName, a_uValueLo, a_uValueHi) do { \
184	aenmNames[iReg] = (a_enmName); \
185	aValues[iReg].Reg128.Low64 = (a_uValueLo); \
186	aValues[iReg].Reg128.High64 = (a_uValueHi); \
187	iReg++; \
188	} while (0)
189
190	/* GPRs */
191	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
192	{
193	if (fWhat & CPUMCTX_EXTRN_RAX)
194	ADD_REG64(WHvX64RegisterRax, pVCpu->cpum.GstCtx.rax);
195	if (fWhat & CPUMCTX_EXTRN_RCX)
196	ADD_REG64(WHvX64RegisterRcx, pVCpu->cpum.GstCtx.rcx);
197	if (fWhat & CPUMCTX_EXTRN_RDX)
198	ADD_REG64(WHvX64RegisterRdx, pVCpu->cpum.GstCtx.rdx);
199	if (fWhat & CPUMCTX_EXTRN_RBX)
200	ADD_REG64(WHvX64RegisterRbx, pVCpu->cpum.GstCtx.rbx);
201	if (fWhat & CPUMCTX_EXTRN_RSP)
202	ADD_REG64(WHvX64RegisterRsp, pVCpu->cpum.GstCtx.rsp);
203	if (fWhat & CPUMCTX_EXTRN_RBP)
204	ADD_REG64(WHvX64RegisterRbp, pVCpu->cpum.GstCtx.rbp);
205	if (fWhat & CPUMCTX_EXTRN_RSI)
206	ADD_REG64(WHvX64RegisterRsi, pVCpu->cpum.GstCtx.rsi);
207	if (fWhat & CPUMCTX_EXTRN_RDI)
208	ADD_REG64(WHvX64RegisterRdi, pVCpu->cpum.GstCtx.rdi);
209	if (fWhat & CPUMCTX_EXTRN_R8_R15)
210	{
211	ADD_REG64(WHvX64RegisterR8, pVCpu->cpum.GstCtx.r8);
212	ADD_REG64(WHvX64RegisterR9, pVCpu->cpum.GstCtx.r9);
213	ADD_REG64(WHvX64RegisterR10, pVCpu->cpum.GstCtx.r10);
214	ADD_REG64(WHvX64RegisterR11, pVCpu->cpum.GstCtx.r11);
215	ADD_REG64(WHvX64RegisterR12, pVCpu->cpum.GstCtx.r12);
216	ADD_REG64(WHvX64RegisterR13, pVCpu->cpum.GstCtx.r13);
217	ADD_REG64(WHvX64RegisterR14, pVCpu->cpum.GstCtx.r14);
218	ADD_REG64(WHvX64RegisterR15, pVCpu->cpum.GstCtx.r15);
219	}
220	}
221
222	/* RIP & Flags */
223	if (fWhat & CPUMCTX_EXTRN_RIP)
224	ADD_REG64(WHvX64RegisterRip, pVCpu->cpum.GstCtx.rip);
225	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
226	ADD_REG64(WHvX64RegisterRflags, pVCpu->cpum.GstCtx.rflags.u);
227
228	/* Segments */
229	# define ADD_SEG(a_enmName, a_SReg) \
230	do { \
231	aenmNames[iReg] = a_enmName; \
232	aValues[iReg].Segment.Base = (a_SReg).u64Base; \
233	aValues[iReg].Segment.Limit = (a_SReg).u32Limit; \
234	aValues[iReg].Segment.Selector = (a_SReg).Sel; \
235	aValues[iReg].Segment.Attributes = (a_SReg).Attr.u; \
236	iReg++; \
237	} while (0)
238	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
239	{
240	if (fWhat & CPUMCTX_EXTRN_ES)
241	ADD_SEG(WHvX64RegisterEs, pVCpu->cpum.GstCtx.es);
242	if (fWhat & CPUMCTX_EXTRN_CS)
243	ADD_SEG(WHvX64RegisterCs, pVCpu->cpum.GstCtx.cs);
244	if (fWhat & CPUMCTX_EXTRN_SS)
245	ADD_SEG(WHvX64RegisterSs, pVCpu->cpum.GstCtx.ss);
246	if (fWhat & CPUMCTX_EXTRN_DS)
247	ADD_SEG(WHvX64RegisterDs, pVCpu->cpum.GstCtx.ds);
248	if (fWhat & CPUMCTX_EXTRN_FS)
249	ADD_SEG(WHvX64RegisterFs, pVCpu->cpum.GstCtx.fs);
250	if (fWhat & CPUMCTX_EXTRN_GS)
251	ADD_SEG(WHvX64RegisterGs, pVCpu->cpum.GstCtx.gs);
252	}
253
254	/* Descriptor tables & task segment. */
255	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
256	{
257	if (fWhat & CPUMCTX_EXTRN_LDTR)
258	ADD_SEG(WHvX64RegisterLdtr, pVCpu->cpum.GstCtx.ldtr);
259	if (fWhat & CPUMCTX_EXTRN_TR)
260	ADD_SEG(WHvX64RegisterTr, pVCpu->cpum.GstCtx.tr);
261	if (fWhat & CPUMCTX_EXTRN_IDTR)
262	{
263	aenmNames[iReg] = WHvX64RegisterIdtr;
264	aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.idtr.cbIdt;
265	aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.idtr.pIdt;
266	iReg++;
267	}
268	if (fWhat & CPUMCTX_EXTRN_GDTR)
269	{
270	aenmNames[iReg] = WHvX64RegisterGdtr;
271	aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.gdtr.cbGdt;
272	aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.gdtr.pGdt;
273	iReg++;
274	}
275	}
276
277	/* Control registers. */
278	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
279	{
280	if (fWhat & CPUMCTX_EXTRN_CR0)
281	ADD_REG64(WHvX64RegisterCr0, pVCpu->cpum.GstCtx.cr0);
282	if (fWhat & CPUMCTX_EXTRN_CR2)
283	ADD_REG64(WHvX64RegisterCr2, pVCpu->cpum.GstCtx.cr2);
284	if (fWhat & CPUMCTX_EXTRN_CR3)
285	ADD_REG64(WHvX64RegisterCr3, pVCpu->cpum.GstCtx.cr3);
286	if (fWhat & CPUMCTX_EXTRN_CR4)
287	ADD_REG64(WHvX64RegisterCr4, pVCpu->cpum.GstCtx.cr4);
288	}
289	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
290	ADD_REG64(WHvX64RegisterCr8, CPUMGetGuestCR8(pVCpu));
291
292	/* Debug registers. */
293	/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
294	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
295	{
296	ADD_REG64(WHvX64RegisterDr0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
297	ADD_REG64(WHvX64RegisterDr1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
298	ADD_REG64(WHvX64RegisterDr2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
299	ADD_REG64(WHvX64RegisterDr3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
300	}
301	if (fWhat & CPUMCTX_EXTRN_DR6)
302	ADD_REG64(WHvX64RegisterDr6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
303	if (fWhat & CPUMCTX_EXTRN_DR7)
304	ADD_REG64(WHvX64RegisterDr7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
305
306	/* Floating point state. */
307	if (fWhat & CPUMCTX_EXTRN_X87)
308	{
309	ADD_REG128(WHvX64RegisterFpMmx0, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[1]);
310	ADD_REG128(WHvX64RegisterFpMmx1, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[1]);
311	ADD_REG128(WHvX64RegisterFpMmx2, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[1]);
312	ADD_REG128(WHvX64RegisterFpMmx3, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[1]);
313	ADD_REG128(WHvX64RegisterFpMmx4, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[1]);
314	ADD_REG128(WHvX64RegisterFpMmx5, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[1]);
315	ADD_REG128(WHvX64RegisterFpMmx6, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[1]);
316	ADD_REG128(WHvX64RegisterFpMmx7, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[1]);
317
318	aenmNames[iReg] = WHvX64RegisterFpControlStatus;
319	aValues[iReg].FpControlStatus.FpControl = pVCpu->cpum.GstCtx.pXStateR3->x87.FCW;
320	aValues[iReg].FpControlStatus.FpStatus = pVCpu->cpum.GstCtx.pXStateR3->x87.FSW;
321	aValues[iReg].FpControlStatus.FpTag = pVCpu->cpum.GstCtx.pXStateR3->x87.FTW;
322	aValues[iReg].FpControlStatus.Reserved = pVCpu->cpum.GstCtx.pXStateR3->x87.FTW >> 8;
323	aValues[iReg].FpControlStatus.LastFpOp = pVCpu->cpum.GstCtx.pXStateR3->x87.FOP;
324	aValues[iReg].FpControlStatus.LastFpRip = (pVCpu->cpum.GstCtx.pXStateR3->x87.FPUIP)
325	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.CS << 32)
326	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd1 << 48);
327	iReg++;
328
329	aenmNames[iReg] = WHvX64RegisterXmmControlStatus;
330	aValues[iReg].XmmControlStatus.LastFpRdp = (pVCpu->cpum.GstCtx.pXStateR3->x87.FPUDP)
331	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.DS << 32)
332	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd2 << 48);
333	aValues[iReg].XmmControlStatus.XmmStatusControl = pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR;
334	aValues[iReg].XmmControlStatus.XmmStatusControlMask = pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
335	iReg++;
336	}
337
338	/* Vector state. */
339	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
340	{
341	ADD_REG128(WHvX64RegisterXmm0, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Hi);
342	ADD_REG128(WHvX64RegisterXmm1, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Hi);
343	ADD_REG128(WHvX64RegisterXmm2, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Hi);
344	ADD_REG128(WHvX64RegisterXmm3, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Hi);
345	ADD_REG128(WHvX64RegisterXmm4, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Hi);
346	ADD_REG128(WHvX64RegisterXmm5, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Hi);
347	ADD_REG128(WHvX64RegisterXmm6, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Hi);
348	ADD_REG128(WHvX64RegisterXmm7, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Hi);
349	ADD_REG128(WHvX64RegisterXmm8, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Hi);
350	ADD_REG128(WHvX64RegisterXmm9, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Hi);
351	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Hi);
352	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Hi);
353	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Hi);
354	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Hi);
355	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Hi);
356	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Hi);
357	}
358
359	/* MSRs */
360	// WHvX64RegisterTsc - don't touch
361	if (fWhat & CPUMCTX_EXTRN_EFER)
362	ADD_REG64(WHvX64RegisterEfer, pVCpu->cpum.GstCtx.msrEFER);
363	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
364	ADD_REG64(WHvX64RegisterKernelGsBase, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
365	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
366	{
367	ADD_REG64(WHvX64RegisterSysenterCs, pVCpu->cpum.GstCtx.SysEnter.cs);
368	ADD_REG64(WHvX64RegisterSysenterEip, pVCpu->cpum.GstCtx.SysEnter.eip);
369	ADD_REG64(WHvX64RegisterSysenterEsp, pVCpu->cpum.GstCtx.SysEnter.esp);
370	}
371	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
372	{
373	ADD_REG64(WHvX64RegisterStar, pVCpu->cpum.GstCtx.msrSTAR);
374	ADD_REG64(WHvX64RegisterLstar, pVCpu->cpum.GstCtx.msrLSTAR);
375	ADD_REG64(WHvX64RegisterCstar, pVCpu->cpum.GstCtx.msrCSTAR);
376	ADD_REG64(WHvX64RegisterSfmask, pVCpu->cpum.GstCtx.msrSFMASK);
377	}
378	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
379	{
380	ADD_REG64(WHvX64RegisterApicBase, APICGetBaseMsrNoCheck(pVCpu));
381	ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
382	#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
383	ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
384	#endif
385	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
386	ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
387	ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
388	ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
389	ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
390	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000, pCtxMsrs->msr.MtrrFix4K_C0000);
391	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000, pCtxMsrs->msr.MtrrFix4K_C8000);
392	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000, pCtxMsrs->msr.MtrrFix4K_D0000);
393	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000, pCtxMsrs->msr.MtrrFix4K_D8000);
394	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000, pCtxMsrs->msr.MtrrFix4K_E0000);
395	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000, pCtxMsrs->msr.MtrrFix4K_E8000);
396	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000, pCtxMsrs->msr.MtrrFix4K_F0000);
397	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000, pCtxMsrs->msr.MtrrFix4K_F8000);
398	ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
399	#if 0 /** @todo these registers aren't available? Might explain something.. .*/
400	const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pGVM->pVM);
401	if (enmCpuVendor != CPUMCPUVENDOR_AMD)
402	{
403	ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
404	ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
405	}
406	#endif
407	}
408
409	/* event injection (clear it). */
410	if (fWhat & CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)
411	ADD_REG64(WHvRegisterPendingInterruption, 0);
412
413	/* Interruptibility state. This can get a little complicated since we get
414	half of the state via HV_X64_VP_EXECUTION_STATE. */
415	if ( (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
416	== (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI) )
417	{
418	ADD_REG64(WHvRegisterInterruptState, 0);
419	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
420	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
421	aValues[iReg - 1].InterruptState.InterruptShadow = 1;
422	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
423	aValues[iReg - 1].InterruptState.NmiMasked = 1;
424	}
425	else if (fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT)
426	{
427	if ( pVCpu->nem.s.fLastInterruptShadow
428	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
429	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip))
430	{
431	ADD_REG64(WHvRegisterInterruptState, 0);
432	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
433	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
434	aValues[iReg - 1].InterruptState.InterruptShadow = 1;
435	/** @todo Retrieve NMI state, currently assuming it's zero. (yes this may happen on I/O) */
436	//if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
437	// aValues[iReg - 1].InterruptState.NmiMasked = 1;
438	}
439	}
440	else
441	Assert(!(fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI));
442
443	/* Interrupt windows. Always set if active as Hyper-V seems to be forgetful. */
444	uint8_t const fDesiredIntWin = pVCpu->nem.s.fDesiredInterruptWindows;
445	if ( fDesiredIntWin
446	\|\| pVCpu->nem.s.fCurrentInterruptWindows != fDesiredIntWin)
447	{
448	pVCpu->nem.s.fCurrentInterruptWindows = pVCpu->nem.s.fDesiredInterruptWindows;
449	ADD_REG64(WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin);
450	Assert(aValues[iReg - 1].DeliverabilityNotifications.NmiNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_NMI));
451	Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_REGULAR));
452	Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptPriority == (fDesiredIntWin & NEM_WIN_INTW_F_PRIO_MASK) >> NEM_WIN_INTW_F_PRIO_SHIFT);
453	}
454
455	/// @todo WHvRegisterPendingEvent
456
457	/*
458	* Set the registers.
459	*/
460	Assert(iReg < RT_ELEMENTS(aValues));
461	Assert(iReg < RT_ELEMENTS(aenmNames));
462	# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
463	Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
464	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
465	# endif
466	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
467	if (SUCCEEDED(hrc))
468	{
469	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK \| CPUMCTX_EXTRN_KEEPER_NEM;
470	return VINF_SUCCESS;
471	}
472	AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
473	pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
474	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
475	return VERR_INTERNAL_ERROR;
476
477	# undef ADD_REG64
478	# undef ADD_REG128
479	# undef ADD_SEG
480
481	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
482	}
483
484
485	NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVM pVM, PVMCPU pVCpu, uint64_t fWhat)
486	{
487	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
488	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
489	if (pVM->nem.s.fUseRing0Runloop)
490	# endif
491	{
492	/* See NEMR0ImportState */
493	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_IMPORT_STATE, fWhat, NULL);
494	if (RT_SUCCESS(rc))
495	return rc;
496	if (rc == VERR_NEM_FLUSH_TLB)
497	return PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /fGlobal/);
498	AssertLogRelRCReturn(rc, rc);
499	return rc;
500	}
501	# endif
502	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
503	WHV_REGISTER_NAME aenmNames[128];
504
505	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
506	uintptr_t iReg = 0;
507
508	/* GPRs */
509	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
510	{
511	if (fWhat & CPUMCTX_EXTRN_RAX)
512	aenmNames[iReg++] = WHvX64RegisterRax;
513	if (fWhat & CPUMCTX_EXTRN_RCX)
514	aenmNames[iReg++] = WHvX64RegisterRcx;
515	if (fWhat & CPUMCTX_EXTRN_RDX)
516	aenmNames[iReg++] = WHvX64RegisterRdx;
517	if (fWhat & CPUMCTX_EXTRN_RBX)
518	aenmNames[iReg++] = WHvX64RegisterRbx;
519	if (fWhat & CPUMCTX_EXTRN_RSP)
520	aenmNames[iReg++] = WHvX64RegisterRsp;
521	if (fWhat & CPUMCTX_EXTRN_RBP)
522	aenmNames[iReg++] = WHvX64RegisterRbp;
523	if (fWhat & CPUMCTX_EXTRN_RSI)
524	aenmNames[iReg++] = WHvX64RegisterRsi;
525	if (fWhat & CPUMCTX_EXTRN_RDI)
526	aenmNames[iReg++] = WHvX64RegisterRdi;
527	if (fWhat & CPUMCTX_EXTRN_R8_R15)
528	{
529	aenmNames[iReg++] = WHvX64RegisterR8;
530	aenmNames[iReg++] = WHvX64RegisterR9;
531	aenmNames[iReg++] = WHvX64RegisterR10;
532	aenmNames[iReg++] = WHvX64RegisterR11;
533	aenmNames[iReg++] = WHvX64RegisterR12;
534	aenmNames[iReg++] = WHvX64RegisterR13;
535	aenmNames[iReg++] = WHvX64RegisterR14;
536	aenmNames[iReg++] = WHvX64RegisterR15;
537	}
538	}
539
540	/* RIP & Flags */
541	if (fWhat & CPUMCTX_EXTRN_RIP)
542	aenmNames[iReg++] = WHvX64RegisterRip;
543	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
544	aenmNames[iReg++] = WHvX64RegisterRflags;
545
546	/* Segments */
547	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
548	{
549	if (fWhat & CPUMCTX_EXTRN_ES)
550	aenmNames[iReg++] = WHvX64RegisterEs;
551	if (fWhat & CPUMCTX_EXTRN_CS)
552	aenmNames[iReg++] = WHvX64RegisterCs;
553	if (fWhat & CPUMCTX_EXTRN_SS)
554	aenmNames[iReg++] = WHvX64RegisterSs;
555	if (fWhat & CPUMCTX_EXTRN_DS)
556	aenmNames[iReg++] = WHvX64RegisterDs;
557	if (fWhat & CPUMCTX_EXTRN_FS)
558	aenmNames[iReg++] = WHvX64RegisterFs;
559	if (fWhat & CPUMCTX_EXTRN_GS)
560	aenmNames[iReg++] = WHvX64RegisterGs;
561	}
562
563	/* Descriptor tables. */
564	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
565	{
566	if (fWhat & CPUMCTX_EXTRN_LDTR)
567	aenmNames[iReg++] = WHvX64RegisterLdtr;
568	if (fWhat & CPUMCTX_EXTRN_TR)
569	aenmNames[iReg++] = WHvX64RegisterTr;
570	if (fWhat & CPUMCTX_EXTRN_IDTR)
571	aenmNames[iReg++] = WHvX64RegisterIdtr;
572	if (fWhat & CPUMCTX_EXTRN_GDTR)
573	aenmNames[iReg++] = WHvX64RegisterGdtr;
574	}
575
576	/* Control registers. */
577	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
578	{
579	if (fWhat & CPUMCTX_EXTRN_CR0)
580	aenmNames[iReg++] = WHvX64RegisterCr0;
581	if (fWhat & CPUMCTX_EXTRN_CR2)
582	aenmNames[iReg++] = WHvX64RegisterCr2;
583	if (fWhat & CPUMCTX_EXTRN_CR3)
584	aenmNames[iReg++] = WHvX64RegisterCr3;
585	if (fWhat & CPUMCTX_EXTRN_CR4)
586	aenmNames[iReg++] = WHvX64RegisterCr4;
587	}
588	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
589	aenmNames[iReg++] = WHvX64RegisterCr8;
590
591	/* Debug registers. */
592	if (fWhat & CPUMCTX_EXTRN_DR7)
593	aenmNames[iReg++] = WHvX64RegisterDr7;
594	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
595	{
596	if (!(fWhat & CPUMCTX_EXTRN_DR7) && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_DR7))
597	{
598	fWhat \|= CPUMCTX_EXTRN_DR7;
599	aenmNames[iReg++] = WHvX64RegisterDr7;
600	}
601	aenmNames[iReg++] = WHvX64RegisterDr0;
602	aenmNames[iReg++] = WHvX64RegisterDr1;
603	aenmNames[iReg++] = WHvX64RegisterDr2;
604	aenmNames[iReg++] = WHvX64RegisterDr3;
605	}
606	if (fWhat & CPUMCTX_EXTRN_DR6)
607	aenmNames[iReg++] = WHvX64RegisterDr6;
608
609	/* Floating point state. */
610	if (fWhat & CPUMCTX_EXTRN_X87)
611	{
612	aenmNames[iReg++] = WHvX64RegisterFpMmx0;
613	aenmNames[iReg++] = WHvX64RegisterFpMmx1;
614	aenmNames[iReg++] = WHvX64RegisterFpMmx2;
615	aenmNames[iReg++] = WHvX64RegisterFpMmx3;
616	aenmNames[iReg++] = WHvX64RegisterFpMmx4;
617	aenmNames[iReg++] = WHvX64RegisterFpMmx5;
618	aenmNames[iReg++] = WHvX64RegisterFpMmx6;
619	aenmNames[iReg++] = WHvX64RegisterFpMmx7;
620	aenmNames[iReg++] = WHvX64RegisterFpControlStatus;
621	}
622	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
623	aenmNames[iReg++] = WHvX64RegisterXmmControlStatus;
624
625	/* Vector state. */
626	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
627	{
628	aenmNames[iReg++] = WHvX64RegisterXmm0;
629	aenmNames[iReg++] = WHvX64RegisterXmm1;
630	aenmNames[iReg++] = WHvX64RegisterXmm2;
631	aenmNames[iReg++] = WHvX64RegisterXmm3;
632	aenmNames[iReg++] = WHvX64RegisterXmm4;
633	aenmNames[iReg++] = WHvX64RegisterXmm5;
634	aenmNames[iReg++] = WHvX64RegisterXmm6;
635	aenmNames[iReg++] = WHvX64RegisterXmm7;
636	aenmNames[iReg++] = WHvX64RegisterXmm8;
637	aenmNames[iReg++] = WHvX64RegisterXmm9;
638	aenmNames[iReg++] = WHvX64RegisterXmm10;
639	aenmNames[iReg++] = WHvX64RegisterXmm11;
640	aenmNames[iReg++] = WHvX64RegisterXmm12;
641	aenmNames[iReg++] = WHvX64RegisterXmm13;
642	aenmNames[iReg++] = WHvX64RegisterXmm14;
643	aenmNames[iReg++] = WHvX64RegisterXmm15;
644	}
645
646	/* MSRs */
647	// WHvX64RegisterTsc - don't touch
648	if (fWhat & CPUMCTX_EXTRN_EFER)
649	aenmNames[iReg++] = WHvX64RegisterEfer;
650	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
651	aenmNames[iReg++] = WHvX64RegisterKernelGsBase;
652	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
653	{
654	aenmNames[iReg++] = WHvX64RegisterSysenterCs;
655	aenmNames[iReg++] = WHvX64RegisterSysenterEip;
656	aenmNames[iReg++] = WHvX64RegisterSysenterEsp;
657	}
658	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
659	{
660	aenmNames[iReg++] = WHvX64RegisterStar;
661	aenmNames[iReg++] = WHvX64RegisterLstar;
662	aenmNames[iReg++] = WHvX64RegisterCstar;
663	aenmNames[iReg++] = WHvX64RegisterSfmask;
664	}
665
666	//#ifdef LOG_ENABLED
667	// const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pGVM->pVM);
668	//#endif
669	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
670	{
671	aenmNames[iReg++] = WHvX64RegisterApicBase; /// @todo APIC BASE
672	aenmNames[iReg++] = WHvX64RegisterPat;
673	#if 0 /def LOG_ENABLED/ /** @todo Check if WHvX64RegisterMsrMtrrCap works... */
674	aenmNames[iReg++] = WHvX64RegisterMsrMtrrCap;
675	#endif
676	aenmNames[iReg++] = WHvX64RegisterMsrMtrrDefType;
677	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix64k00000;
678	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16k80000;
679	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16kA0000;
680	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC0000;
681	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC8000;
682	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD0000;
683	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD8000;
684	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE0000;
685	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE8000;
686	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF0000;
687	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF8000;
688	aenmNames[iReg++] = WHvX64RegisterTscAux;
689	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
690	//#ifdef LOG_ENABLED
691	// if (enmCpuVendor != CPUMCPUVENDOR_AMD)
692	// aenmNames[iReg++] = HvX64RegisterIa32FeatureControl;
693	//#endif
694	}
695
696	/* Interruptibility. */
697	if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
698	{
699	aenmNames[iReg++] = WHvRegisterInterruptState;
700	aenmNames[iReg++] = WHvX64RegisterRip;
701	}
702
703	/* event injection */
704	aenmNames[iReg++] = WHvRegisterPendingInterruption;
705	aenmNames[iReg++] = WHvRegisterPendingEvent0; /** @todo renamed to WHvRegisterPendingEvent */
706
707	size_t const cRegs = iReg;
708	Assert(cRegs < RT_ELEMENTS(aenmNames));
709
710	/*
711	* Get the registers.
712	*/
713	WHV_REGISTER_VALUE aValues[128];
714	RT_ZERO(aValues);
715	Assert(RT_ELEMENTS(aValues) >= cRegs);
716	Assert(RT_ELEMENTS(aenmNames) >= cRegs);
717	# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
718	Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
719	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
720	# endif
721	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, (uint32_t)cRegs, aValues);
722	AssertLogRelMsgReturn(SUCCEEDED(hrc),
723	("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
724	pVM->nem.s.hPartition, pVCpu->idCpu, cRegs, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
725	, VERR_NEM_GET_REGISTERS_FAILED);
726
727	iReg = 0;
728	# define GET_REG64(a_DstVar, a_enmName) do { \
729	Assert(aenmNames[iReg] == (a_enmName)); \
730	(a_DstVar) = aValues[iReg].Reg64; \
731	iReg++; \
732	} while (0)
733	# define GET_REG64_LOG7(a_DstVar, a_enmName, a_szLogName) do { \
734	Assert(aenmNames[iReg] == (a_enmName)); \
735	if ((a_DstVar) != aValues[iReg].Reg64) \
736	Log7(("NEM/%u: " a_szLogName " changed %RX64 -> %RX64\n", pVCpu->idCpu, (a_DstVar), aValues[iReg].Reg64)); \
737	(a_DstVar) = aValues[iReg].Reg64; \
738	iReg++; \
739	} while (0)
740	# define GET_REG128(a_DstVarLo, a_DstVarHi, a_enmName) do { \
741	Assert(aenmNames[iReg] == a_enmName); \
742	(a_DstVarLo) = aValues[iReg].Reg128.Low64; \
743	(a_DstVarHi) = aValues[iReg].Reg128.High64; \
744	iReg++; \
745	} while (0)
746	# define GET_SEG(a_SReg, a_enmName) do { \
747	Assert(aenmNames[iReg] == (a_enmName)); \
748	NEM_WIN_COPY_BACK_SEG(a_SReg, aValues[iReg].Segment); \
749	iReg++; \
750	} while (0)
751
752	/* GPRs */
753	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
754	{
755	if (fWhat & CPUMCTX_EXTRN_RAX)
756	GET_REG64(pVCpu->cpum.GstCtx.rax, WHvX64RegisterRax);
757	if (fWhat & CPUMCTX_EXTRN_RCX)
758	GET_REG64(pVCpu->cpum.GstCtx.rcx, WHvX64RegisterRcx);
759	if (fWhat & CPUMCTX_EXTRN_RDX)
760	GET_REG64(pVCpu->cpum.GstCtx.rdx, WHvX64RegisterRdx);
761	if (fWhat & CPUMCTX_EXTRN_RBX)
762	GET_REG64(pVCpu->cpum.GstCtx.rbx, WHvX64RegisterRbx);
763	if (fWhat & CPUMCTX_EXTRN_RSP)
764	GET_REG64(pVCpu->cpum.GstCtx.rsp, WHvX64RegisterRsp);
765	if (fWhat & CPUMCTX_EXTRN_RBP)
766	GET_REG64(pVCpu->cpum.GstCtx.rbp, WHvX64RegisterRbp);
767	if (fWhat & CPUMCTX_EXTRN_RSI)
768	GET_REG64(pVCpu->cpum.GstCtx.rsi, WHvX64RegisterRsi);
769	if (fWhat & CPUMCTX_EXTRN_RDI)
770	GET_REG64(pVCpu->cpum.GstCtx.rdi, WHvX64RegisterRdi);
771	if (fWhat & CPUMCTX_EXTRN_R8_R15)
772	{
773	GET_REG64(pVCpu->cpum.GstCtx.r8, WHvX64RegisterR8);
774	GET_REG64(pVCpu->cpum.GstCtx.r9, WHvX64RegisterR9);
775	GET_REG64(pVCpu->cpum.GstCtx.r10, WHvX64RegisterR10);
776	GET_REG64(pVCpu->cpum.GstCtx.r11, WHvX64RegisterR11);
777	GET_REG64(pVCpu->cpum.GstCtx.r12, WHvX64RegisterR12);
778	GET_REG64(pVCpu->cpum.GstCtx.r13, WHvX64RegisterR13);
779	GET_REG64(pVCpu->cpum.GstCtx.r14, WHvX64RegisterR14);
780	GET_REG64(pVCpu->cpum.GstCtx.r15, WHvX64RegisterR15);
781	}
782	}
783
784	/* RIP & Flags */
785	if (fWhat & CPUMCTX_EXTRN_RIP)
786	GET_REG64(pVCpu->cpum.GstCtx.rip, WHvX64RegisterRip);
787	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
788	GET_REG64(pVCpu->cpum.GstCtx.rflags.u, WHvX64RegisterRflags);
789
790	/* Segments */
791	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
792	{
793	if (fWhat & CPUMCTX_EXTRN_ES)
794	GET_SEG(pVCpu->cpum.GstCtx.es, WHvX64RegisterEs);
795	if (fWhat & CPUMCTX_EXTRN_CS)
796	GET_SEG(pVCpu->cpum.GstCtx.cs, WHvX64RegisterCs);
797	if (fWhat & CPUMCTX_EXTRN_SS)
798	GET_SEG(pVCpu->cpum.GstCtx.ss, WHvX64RegisterSs);
799	if (fWhat & CPUMCTX_EXTRN_DS)
800	GET_SEG(pVCpu->cpum.GstCtx.ds, WHvX64RegisterDs);
801	if (fWhat & CPUMCTX_EXTRN_FS)
802	GET_SEG(pVCpu->cpum.GstCtx.fs, WHvX64RegisterFs);
803	if (fWhat & CPUMCTX_EXTRN_GS)
804	GET_SEG(pVCpu->cpum.GstCtx.gs, WHvX64RegisterGs);
805	}
806
807	/* Descriptor tables and the task segment. */
808	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
809	{
810	if (fWhat & CPUMCTX_EXTRN_LDTR)
811	GET_SEG(pVCpu->cpum.GstCtx.ldtr, WHvX64RegisterLdtr);
812
813	if (fWhat & CPUMCTX_EXTRN_TR)
814	{
815	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
816	avoid to trigger sanity assertions around the code, always fix this. */
817	GET_SEG(pVCpu->cpum.GstCtx.tr, WHvX64RegisterTr);
818	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
819	{
820	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
821	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
822	break;
823	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
824	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
825	break;
826	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
827	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
828	break;
829	}
830	}
831	if (fWhat & CPUMCTX_EXTRN_IDTR)
832	{
833	Assert(aenmNames[iReg] == WHvX64RegisterIdtr);
834	pVCpu->cpum.GstCtx.idtr.cbIdt = aValues[iReg].Table.Limit;
835	pVCpu->cpum.GstCtx.idtr.pIdt = aValues[iReg].Table.Base;
836	iReg++;
837	}
838	if (fWhat & CPUMCTX_EXTRN_GDTR)
839	{
840	Assert(aenmNames[iReg] == WHvX64RegisterGdtr);
841	pVCpu->cpum.GstCtx.gdtr.cbGdt = aValues[iReg].Table.Limit;
842	pVCpu->cpum.GstCtx.gdtr.pGdt = aValues[iReg].Table.Base;
843	iReg++;
844	}
845	}
846
847	/* Control registers. */
848	bool fMaybeChangedMode = false;
849	bool fUpdateCr3 = false;
850	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
851	{
852	if (fWhat & CPUMCTX_EXTRN_CR0)
853	{
854	Assert(aenmNames[iReg] == WHvX64RegisterCr0);
855	if (pVCpu->cpum.GstCtx.cr0 != aValues[iReg].Reg64)
856	{
857	CPUMSetGuestCR0(pVCpu, aValues[iReg].Reg64);
858	fMaybeChangedMode = true;
859	}
860	iReg++;
861	}
862	if (fWhat & CPUMCTX_EXTRN_CR2)
863	GET_REG64(pVCpu->cpum.GstCtx.cr2, WHvX64RegisterCr2);
864	if (fWhat & CPUMCTX_EXTRN_CR3)
865	{
866	if (pVCpu->cpum.GstCtx.cr3 != aValues[iReg].Reg64)
867	{
868	CPUMSetGuestCR3(pVCpu, aValues[iReg].Reg64);
869	fUpdateCr3 = true;
870	}
871	iReg++;
872	}
873	if (fWhat & CPUMCTX_EXTRN_CR4)
874	{
875	if (pVCpu->cpum.GstCtx.cr4 != aValues[iReg].Reg64)
876	{
877	CPUMSetGuestCR4(pVCpu, aValues[iReg].Reg64);
878	fMaybeChangedMode = true;
879	}
880	iReg++;
881	}
882	}
883	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
884	{
885	Assert(aenmNames[iReg] == WHvX64RegisterCr8);
886	APICSetTpr(pVCpu, (uint8_t)aValues[iReg].Reg64 << 4);
887	iReg++;
888	}
889
890	/* Debug registers. */
891	if (fWhat & CPUMCTX_EXTRN_DR7)
892	{
893	Assert(aenmNames[iReg] == WHvX64RegisterDr7);
894	if (pVCpu->cpum.GstCtx.dr[7] != aValues[iReg].Reg64)
895	CPUMSetGuestDR7(pVCpu, aValues[iReg].Reg64);
896	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
897	iReg++;
898	}
899	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
900	{
901	Assert(aenmNames[iReg] == WHvX64RegisterDr0);
902	Assert(aenmNames[iReg+3] == WHvX64RegisterDr3);
903	if (pVCpu->cpum.GstCtx.dr[0] != aValues[iReg].Reg64)
904	CPUMSetGuestDR0(pVCpu, aValues[iReg].Reg64);
905	iReg++;
906	if (pVCpu->cpum.GstCtx.dr[1] != aValues[iReg].Reg64)
907	CPUMSetGuestDR1(pVCpu, aValues[iReg].Reg64);
908	iReg++;
909	if (pVCpu->cpum.GstCtx.dr[2] != aValues[iReg].Reg64)
910	CPUMSetGuestDR2(pVCpu, aValues[iReg].Reg64);
911	iReg++;
912	if (pVCpu->cpum.GstCtx.dr[3] != aValues[iReg].Reg64)
913	CPUMSetGuestDR3(pVCpu, aValues[iReg].Reg64);
914	iReg++;
915	}
916	if (fWhat & CPUMCTX_EXTRN_DR6)
917	{
918	Assert(aenmNames[iReg] == WHvX64RegisterDr6);
919	if (pVCpu->cpum.GstCtx.dr[6] != aValues[iReg].Reg64)
920	CPUMSetGuestDR6(pVCpu, aValues[iReg].Reg64);
921	iReg++;
922	}
923
924	/* Floating point state. */
925	if (fWhat & CPUMCTX_EXTRN_X87)
926	{
927	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[1], WHvX64RegisterFpMmx0);
928	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[1], WHvX64RegisterFpMmx1);
929	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[1], WHvX64RegisterFpMmx2);
930	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[1], WHvX64RegisterFpMmx3);
931	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[1], WHvX64RegisterFpMmx4);
932	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[1], WHvX64RegisterFpMmx5);
933	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[1], WHvX64RegisterFpMmx6);
934	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[1], WHvX64RegisterFpMmx7);
935
936	Assert(aenmNames[iReg] == WHvX64RegisterFpControlStatus);
937	pVCpu->cpum.GstCtx.pXStateR3->x87.FCW = aValues[iReg].FpControlStatus.FpControl;
938	pVCpu->cpum.GstCtx.pXStateR3->x87.FSW = aValues[iReg].FpControlStatus.FpStatus;
939	pVCpu->cpum.GstCtx.pXStateR3->x87.FTW = aValues[iReg].FpControlStatus.FpTag
940	/\| (aValues[iReg].FpControlStatus.Reserved << 8)/;
941	pVCpu->cpum.GstCtx.pXStateR3->x87.FOP = aValues[iReg].FpControlStatus.LastFpOp;
942	pVCpu->cpum.GstCtx.pXStateR3->x87.FPUIP = (uint32_t)aValues[iReg].FpControlStatus.LastFpRip;
943	pVCpu->cpum.GstCtx.pXStateR3->x87.CS = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 32);
944	pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd1 = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 48);
945	iReg++;
946	}
947
948	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
949	{
950	Assert(aenmNames[iReg] == WHvX64RegisterXmmControlStatus);
951	if (fWhat & CPUMCTX_EXTRN_X87)
952	{
953	pVCpu->cpum.GstCtx.pXStateR3->x87.FPUDP = (uint32_t)aValues[iReg].XmmControlStatus.LastFpRdp;
954	pVCpu->cpum.GstCtx.pXStateR3->x87.DS = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 32);
955	pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd2 = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 48);
956	}
957	pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR = aValues[iReg].XmmControlStatus.XmmStatusControl;
958	pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR_MASK = aValues[iReg].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
959	iReg++;
960	}
961
962	/* Vector state. */
963	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
964	{
965	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Hi, WHvX64RegisterXmm0);
966	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Hi, WHvX64RegisterXmm1);
967	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Hi, WHvX64RegisterXmm2);
968	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Hi, WHvX64RegisterXmm3);
969	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Hi, WHvX64RegisterXmm4);
970	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Hi, WHvX64RegisterXmm5);
971	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Hi, WHvX64RegisterXmm6);
972	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Hi, WHvX64RegisterXmm7);
973	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Hi, WHvX64RegisterXmm8);
974	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Hi, WHvX64RegisterXmm9);
975	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Hi, WHvX64RegisterXmm10);
976	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Hi, WHvX64RegisterXmm11);
977	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Hi, WHvX64RegisterXmm12);
978	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Hi, WHvX64RegisterXmm13);
979	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Hi, WHvX64RegisterXmm14);
980	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Hi, WHvX64RegisterXmm15);
981	}
982
983	/* MSRs */
984	// WHvX64RegisterTsc - don't touch
985	if (fWhat & CPUMCTX_EXTRN_EFER)
986	{
987	Assert(aenmNames[iReg] == WHvX64RegisterEfer);
988	if (aValues[iReg].Reg64 != pVCpu->cpum.GstCtx.msrEFER)
989	{
990	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, aValues[iReg].Reg64));
991	if ((aValues[iReg].Reg64 ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
992	PGMNotifyNxeChanged(pVCpu, RT_BOOL(aValues[iReg].Reg64 & MSR_K6_EFER_NXE));
993	pVCpu->cpum.GstCtx.msrEFER = aValues[iReg].Reg64;
994	fMaybeChangedMode = true;
995	}
996	iReg++;
997	}
998	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
999	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrKERNELGSBASE, WHvX64RegisterKernelGsBase, "MSR KERNEL_GS_BASE");
1000	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1001	{
1002	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.cs, WHvX64RegisterSysenterCs, "MSR SYSENTER.CS");
1003	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.eip, WHvX64RegisterSysenterEip, "MSR SYSENTER.EIP");
1004	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.esp, WHvX64RegisterSysenterEsp, "MSR SYSENTER.ESP");
1005	}
1006	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1007	{
1008	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSTAR, WHvX64RegisterStar, "MSR STAR");
1009	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrLSTAR, WHvX64RegisterLstar, "MSR LSTAR");
1010	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrCSTAR, WHvX64RegisterCstar, "MSR CSTAR");
1011	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSFMASK, WHvX64RegisterSfmask, "MSR SFMASK");
1012	}
1013	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1014	{
1015	Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
1016	const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
1017	if (aValues[iReg].Reg64 != uOldBase)
1018	{
1019	Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
1020	pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
1021	int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
1022	AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
1023	}
1024	iReg++;
1025
1026	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
1027	#if 0 /def LOG_ENABLED/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
1028	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
1029	#endif
1030	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1031	GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType, WHvX64RegisterMsrMtrrDefType, "MSR MTRR_DEF_TYPE");
1032	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
1033	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
1034	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
1035	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000, WHvX64RegisterMsrMtrrFix4kC0000, "MSR MTRR_FIX_4K_C0000");
1036	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000, WHvX64RegisterMsrMtrrFix4kC8000, "MSR MTRR_FIX_4K_C8000");
1037	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000, WHvX64RegisterMsrMtrrFix4kD0000, "MSR MTRR_FIX_4K_D0000");
1038	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000, WHvX64RegisterMsrMtrrFix4kD8000, "MSR MTRR_FIX_4K_D8000");
1039	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000, WHvX64RegisterMsrMtrrFix4kE0000, "MSR MTRR_FIX_4K_E0000");
1040	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000, WHvX64RegisterMsrMtrrFix4kE8000, "MSR MTRR_FIX_4K_E8000");
1041	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000, WHvX64RegisterMsrMtrrFix4kF0000, "MSR MTRR_FIX_4K_F0000");
1042	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000, WHvX64RegisterMsrMtrrFix4kF8000, "MSR MTRR_FIX_4K_F8000");
1043	GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX");
1044	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
1045	}
1046
1047	/* Interruptibility. */
1048	if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
1049	{
1050	Assert(aenmNames[iReg] == WHvRegisterInterruptState);
1051	Assert(aenmNames[iReg + 1] == WHvX64RegisterRip);
1052
1053	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
1054	{
1055	pVCpu->nem.s.fLastInterruptShadow = aValues[iReg].InterruptState.InterruptShadow;
1056	if (aValues[iReg].InterruptState.InterruptShadow)
1057	EMSetInhibitInterruptsPC(pVCpu, aValues[iReg + 1].Reg64);
1058	else
1059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1060	}
1061
1062	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
1063	{
1064	if (aValues[iReg].InterruptState.NmiMasked)
1065	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
1066	else
1067	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
1068	}
1069
1070	fWhat \|= CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI;
1071	iReg += 2;
1072	}
1073
1074	/* Event injection. */
1075	/// @todo WHvRegisterPendingInterruption
1076	Assert(aenmNames[iReg] == WHvRegisterPendingInterruption);
1077	if (aValues[iReg].PendingInterruption.InterruptionPending)
1078	{
1079	Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
1080	aValues[iReg].PendingInterruption.InterruptionType, aValues[iReg].PendingInterruption.InterruptionVector,
1081	aValues[iReg].PendingInterruption.DeliverErrorCode, aValues[iReg].PendingInterruption.ErrorCode,
1082	aValues[iReg].PendingInterruption.InstructionLength, aValues[iReg].PendingInterruption.NestedEvent));
1083	AssertMsg((aValues[iReg].PendingInterruption.AsUINT64 & UINT64_C(0xfc00)) == 0,
1084	("%#RX64\n", aValues[iReg].PendingInterruption.AsUINT64));
1085	}
1086
1087	/// @todo WHvRegisterPendingEvent0 (renamed to WHvRegisterPendingEvent).
1088
1089	/* Almost done, just update extrn flags and maybe change PGM mode. */
1090	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1091	if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
1092	pVCpu->cpum.GstCtx.fExtrn = 0;
1093
1094	/* Typical. */
1095	if (!fMaybeChangedMode && !fUpdateCr3)
1096	return VINF_SUCCESS;
1097
1098	/*
1099	* Slow.
1100	*/
1101	if (fMaybeChangedMode)
1102	{
1103	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
1104	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1105	}
1106
1107	if (fUpdateCr3)
1108	{
1109	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
1110	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1111	}
1112
1113	return VINF_SUCCESS;
1114	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1115	}
1116
1117	#endif /* !IN_RING0 */
1118
1119
1120	/**
1121	* Interface for importing state on demand (used by IEM).
1122	*
1123	* @returns VBox status code.
1124	* @param pVCpu The cross context CPU structure.
1125	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1126	*/
1127	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
1128	{
1129	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
1130
1131	#ifdef IN_RING0
1132	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1133	/** @todo improve and secure this translation */
1134	PGVM pGVM = GVMMR0ByHandle(pVCpu->pVMR0->hSelf);
1135	AssertReturn(pGVM, VERR_INVALID_VMCPU_HANDLE);
1136	VMCPUID idCpu = pVCpu->idCpu;
1137	ASMCompilerBarrier();
1138	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_VMCPU_HANDLE);
1139
1140	return nemR0WinImportState(pGVM, &pGVM->aCpus[idCpu], &pVCpu->cpum.GstCtx, fWhat, true /fCanUpdateCr3/);
1141	# else
1142	RT_NOREF(pVCpu, fWhat);
1143	return VERR_NOT_IMPLEMENTED;
1144	# endif
1145	#else
1146	return nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
1147	#endif
1148	}
1149
1150
1151	/**
1152	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
1153	*
1154	* @returns VBox status code.
1155	* @param pVCpu The cross context CPU structure.
1156	* @param pcTicks Where to return the CPU tick count.
1157	* @param puAux Where to return the TSC_AUX register value.
1158	*/
1159	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPU pVCpu, uint64_t pcTicks, uint32_t puAux)
1160	{
1161	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
1162
1163	#ifdef IN_RING3
1164	PVM pVM = pVCpu->CTX_SUFF(pVM);
1165	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1166	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1167
1168	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1169	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
1170	if (pVM->nem.s.fUseRing0Runloop)
1171	# endif
1172	{
1173	/* Call ring-0 and get the values. */
1174	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_QUERY_CPU_TICK, 0, NULL);
1175	AssertLogRelRCReturn(rc, rc);
1176	*pcTicks = pVCpu->nem.s.Hypercall.QueryCpuTick.cTicks;
1177	if (puAux)
1178	*puAux = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX
1179	? pVCpu->nem.s.Hypercall.QueryCpuTick.uAux : CPUMGetGuestTscAux(pVCpu);
1180	return VINF_SUCCESS;
1181	}
1182	# endif
1183	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1184	/* Call the offical API. */
1185	WHV_REGISTER_NAME aenmNames[2] = { WHvX64RegisterTsc, WHvX64RegisterTscAux };
1186	WHV_REGISTER_VALUE aValues[2] = { {0, 0}, {0, 0} };
1187	Assert(RT_ELEMENTS(aenmNames) == RT_ELEMENTS(aValues));
1188	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, 2, aValues);
1189	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1190	("WHvGetVirtualProcessorRegisters(%p, %u,{tsc,tsc_aux},2,) -> %Rhrc (Last=%#x/%u)\n",
1191	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1192	, VERR_NEM_GET_REGISTERS_FAILED);
1193	*pcTicks = aValues[0].Reg64;
1194	if (puAux)
1195	*pcTicks = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX ? aValues[0].Reg64 : CPUMGetGuestTscAux(pVCpu);
1196	return VINF_SUCCESS;
1197	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1198	#else /* IN_RING0 */
1199	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1200	/** @todo improve and secure this translation */
1201	PGVM pGVM = GVMMR0ByHandle(pVCpu->pVMR0->hSelf);
1202	AssertReturn(pGVM, VERR_INVALID_VMCPU_HANDLE);
1203	VMCPUID idCpu = pVCpu->idCpu;
1204	ASMCompilerBarrier();
1205	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_VMCPU_HANDLE);
1206
1207	int rc = nemR0WinQueryCpuTick(pGVM, &pGVM->aCpus[idCpu], pcTicks, puAux);
1208	if (RT_SUCCESS(rc) && puAux && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX))
1209	*puAux = CPUMGetGuestTscAux(pVCpu);
1210	return rc;
1211	# else
1212	RT_NOREF(pVCpu, pcTicks, puAux);
1213	return VERR_NOT_IMPLEMENTED;
1214	# endif
1215	#endif /* IN_RING0 */
1216	}
1217
1218
1219	/**
1220	* Resumes CPU clock (TSC) on all virtual CPUs.
1221	*
1222	* This is called by TM when the VM is started, restored, resumed or similar.
1223	*
1224	* @returns VBox status code.
1225	* @param pVM The cross context VM structure.
1226	* @param pVCpu The cross context CPU structure of the calling EMT.
1227	* @param uPausedTscValue The TSC value at the time of pausing.
1228	*/
1229	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVM pVM, PVMCPU pVCpu, uint64_t uPausedTscValue)
1230	{
1231	#ifdef IN_RING0
1232	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1233	/** @todo improve and secure this translation */
1234	PGVM pGVM = GVMMR0ByHandle(pVM->hSelf);
1235	AssertReturn(pGVM, VERR_INVALID_VMCPU_HANDLE);
1236	VMCPUID idCpu = pVCpu->idCpu;
1237	ASMCompilerBarrier();
1238	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_VMCPU_HANDLE);
1239
1240	return nemR0WinResumeCpuTickOnAll(pGVM, &pGVM->aCpus[idCpu], uPausedTscValue);
1241	# else
1242	RT_NOREF(pVM, pVCpu, uPausedTscValue);
1243	return VERR_NOT_IMPLEMENTED;
1244	# endif
1245	#else /* IN_RING3 */
1246	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1247	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1248
1249	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1250	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
1251	if (pVM->nem.s.fUseRing0Runloop)
1252	# endif
1253	{
1254	/* Call ring-0 and do it all there. */
1255	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL, uPausedTscValue, NULL);
1256	}
1257	# endif
1258	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1259	/*
1260	* Call the offical API to do the job.
1261	*/
1262	if (pVM->cCpus > 1)
1263	RTThreadYield(); /* Try decrease the chance that we get rescheduled in the middle. */
1264
1265	/* Start with the first CPU. */
1266	WHV_REGISTER_NAME enmName = WHvX64RegisterTsc;
1267	WHV_REGISTER_VALUE Value = {0, 0};
1268	Value.Reg64 = uPausedTscValue;
1269	uint64_t const uFirstTsc = ASMReadTSC();
1270	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, 0 /iCpu/, &enmName, 1, &Value);
1271	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1272	("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64) -> %Rhrc (Last=%#x/%u)\n",
1273	pVM->nem.s.hPartition, uPausedTscValue, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1274	, VERR_NEM_SET_TSC);
1275
1276	/* Do the other CPUs, adjusting for elapsed TSC and keeping finger crossed
1277	that we don't introduce too much drift here. */
1278	for (VMCPUID iCpu = 1; iCpu < pVM->cCpus; iCpu++)
1279	{
1280	Assert(enmName == WHvX64RegisterTsc);
1281	const uint64_t offDelta = (ASMReadTSC() - uFirstTsc);
1282	Value.Reg64 = uPausedTscValue + offDelta;
1283	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, iCpu, &enmName, 1, &Value);
1284	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1285	("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64 + %#RX64) -> %Rhrc (Last=%#x/%u)\n",
1286	pVM->nem.s.hPartition, iCpu, uPausedTscValue, offDelta, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1287	, VERR_NEM_SET_TSC);
1288	}
1289
1290	return VINF_SUCCESS;
1291	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1292	#endif /* IN_RING3 */
1293	}
1294
1295	#ifdef NEMWIN_NEED_GET_REGISTER
1296	# if defined(IN_RING0) \|\| defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
1297	/** Worker for assertion macro. */
1298	NEM_TMPL_STATIC int nemHCWinGetRegister(PVMCPU pVCpu, PGVMCPU pGVCpu, uint32_t enmReg, HV_REGISTER_VALUE *pRetValue)
1299	{
1300	RT_ZERO(*pRetValue);
1301	# ifdef IN_RING3
1302	RT_NOREF(pVCpu, pGVCpu, enmReg);
1303	return VERR_NOT_IMPLEMENTED;
1304	# else
1305	NOREF(pVCpu);
1306
1307	/*
1308	* Hypercall parameters.
1309	*/
1310	HV_INPUT_GET_VP_REGISTERS pInput = (HV_INPUT_GET_VP_REGISTERS )pGVCpu->nem.s.HypercallData.pbPage;
1311	AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
1312	AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API);
1313
1314	pInput->PartitionId = pGVCpu->pGVM->nem.s.idHvPartition;
1315	pInput->VpIndex = pGVCpu->idCpu;
1316	pInput->fFlags = 0;
1317	pInput->Names[0] = (HV_REGISTER_NAME)enmReg;
1318
1319	size_t const cbInput = RT_ALIGN_Z(RT_UOFFSETOF(HV_INPUT_GET_VP_REGISTERS, Names[1]), 32);
1320	HV_REGISTER_VALUE paValues = (HV_REGISTER_VALUE )((uint8_t *)pInput + cbInput);
1321	RT_BZERO(paValues, sizeof(paValues[0]) * 1);
1322
1323	/*
1324	* Make the hypercall and copy out the value.
1325	*/
1326	uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallGetVpRegisters, 1),
1327	pGVCpu->nem.s.HypercallData.HCPhysPage,
1328	pGVCpu->nem.s.HypercallData.HCPhysPage + cbInput);
1329	AssertLogRelMsgReturn(uResult == HV_MAKE_CALL_REP_RET(1), ("uResult=%RX64 cRegs=%#x\n", uResult, 1),
1330	VERR_NEM_GET_REGISTERS_FAILED);
1331
1332	*pRetValue = paValues[0];
1333	return VINF_SUCCESS;
1334	# endif
1335	}
1336	# else
1337	/** Worker for assertion macro. */
1338	NEM_TMPL_STATIC int nemR3WinGetRegister(PVMCPU a_pVCpu, uint32_t a_enmReg, WHV_REGISTER_VALUE pValue)
1339	{
1340	RT_ZERO(*pRetValue);
1341	RT_NOREF(pVCpu, pGVCpu, enmReg);
1342	return VERR_NOT_IMPLEMENTED;
1343	}
1344	# endif
1345	#endif
1346
1347
1348	#ifdef LOG_ENABLED
1349	/**
1350	* Get the virtual processor running status.
1351	*/
1352	DECLINLINE(VID_PROCESSOR_STATUS) nemHCWinCpuGetRunningStatus(PVMCPU pVCpu)
1353	{
1354	# ifdef IN_RING0
1355	NOREF(pVCpu);
1356	return VidProcessorStatusUndefined;
1357	# else
1358	RTERRVARS Saved;
1359	RTErrVarsSave(&Saved);
1360
1361	/*
1362	* This API is disabled in release builds, it seems. On build 17101 it requires
1363	* the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00
1364	*/
1365	VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined;
1366	NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus);
1367	AssertRC(rcNt);
1368
1369	RTErrVarsRestore(&Saved);
1370	return enmCpuStatus;
1371	# endif
1372	}
1373	#endif /* LOG_ENABLED */
1374
1375
1376	#if defined(NEM_WIN_USE_OUR_OWN_RUN_API) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1377	# ifdef IN_RING3 /* hopefully not needed in ring-0, as we'd need KTHREADs and KeAlertThread. */
1378	/**
1379	* Our own WHvCancelRunVirtualProcessor that can later be moved to ring-0.
1380	*
1381	* This is an experiment only.
1382	*
1383	* @returns VBox status code.
1384	* @param pVM The cross context VM structure.
1385	* @param pVCpu The cross context virtual CPU structure of the
1386	* calling EMT.
1387	*/
1388	NEM_TMPL_STATIC int nemHCWinCancelRunVirtualProcessor(PVM pVM, PVMCPU pVCpu)
1389	{
1390	/*
1391	* Work the state.
1392	*
1393	* From the looks of things, we should let the EMT call VidStopVirtualProcessor.
1394	* So, we just need to modify the state and kick the EMT if it's waiting on
1395	* messages. For the latter we use QueueUserAPC / KeAlterThread.
1396	*/
1397	for (;;)
1398	{
1399	VMCPUSTATE enmState = VMCPU_GET_STATE(pVCpu);
1400	switch (enmState)
1401	{
1402	case VMCPUSTATE_STARTED_EXEC_NEM:
1403	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM))
1404	{
1405	DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM -> CANCELED");
1406	Log8(("nemHCWinCancelRunVirtualProcessor: Switched %u to canceled state\n", pVCpu->idCpu));
1407	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelChangedState);
1408	return VINF_SUCCESS;
1409	}
1410	break;
1411
1412	case VMCPUSTATE_STARTED_EXEC_NEM_WAIT:
1413	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT))
1414	{
1415	DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM_WAIT -> CANCELED");
1416	# ifdef IN_RING0
1417	NTSTATUS rcNt = KeAlertThread(??);
1418	DBGFTRACE_CUSTOM(pVM, "KeAlertThread -> %#x", rcNt);
1419	# else
1420	NTSTATUS rcNt = NtAlertThread(pVCpu->nem.s.hNativeThreadHandle);
1421	DBGFTRACE_CUSTOM(pVM, "NtAlertThread -> %#x", rcNt);
1422	# endif
1423	Log8(("nemHCWinCancelRunVirtualProcessor: Alerted %u: %#x\n", pVCpu->idCpu, rcNt));
1424	Assert(rcNt == STATUS_SUCCESS);
1425	if (NT_SUCCESS(rcNt))
1426	{
1427	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelAlertedThread);
1428	return VINF_SUCCESS;
1429	}
1430	AssertLogRelMsgFailedReturn(("NtAlertThread failed: %#x\n", rcNt), RTErrConvertFromNtStatus(rcNt));
1431	}
1432	break;
1433
1434	default:
1435	return VINF_SUCCESS;
1436	}
1437
1438	ASMNopPause();
1439	RT_NOREF(pVM);
1440	}
1441	}
1442	# endif /* IN_RING3 */
1443	#endif /* NEM_WIN_USE_OUR_OWN_RUN_API \|\| NEM_WIN_WITH_RING0_RUNLOOP */
1444
1445
1446	#ifdef LOG_ENABLED
1447	/**
1448	* Logs the current CPU state.
1449	*/
1450	NEM_TMPL_STATIC void nemHCWinLogState(PVM pVM, PVMCPU pVCpu)
1451	{
1452	if (LogIs3Enabled())
1453	{
1454	# if 0 // def IN_RING3 - causes lazy state import assertions all over CPUM.
1455	char szRegs[4096];
1456	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
1457	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
1458	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
1459	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
1460	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
1461	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
1462	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
1463	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
1464	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
1465	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
1466	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
1467	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
1468	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
1469	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
1470	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
1471	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
1472	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
1473	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
1474	" efer=%016VR{efer}\n"
1475	" pat=%016VR{pat}\n"
1476	" sf_mask=%016VR{sf_mask}\n"
1477	"krnl_gs_base=%016VR{krnl_gs_base}\n"
1478	" lstar=%016VR{lstar}\n"
1479	" star=%016VR{star} cstar=%016VR{cstar}\n"
1480	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
1481	);
1482
1483	char szInstr[256];
1484	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
1485	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
1486	szInstr, sizeof(szInstr), NULL);
1487	Log3(("%s%s\n", szRegs, szInstr));
1488	# else
1489	/** @todo stat logging in ring-0 */
1490	RT_NOREF(pVM, pVCpu);
1491	# endif
1492	}
1493	}
1494	#endif /* LOG_ENABLED */
1495
1496
1497	/** Macro used by nemHCWinExecStateToLogStr and nemR3WinExecStateToLogStr. */
1498	#define SWITCH_IT(a_szPrefix) \
1499	do \
1500	switch (u)\
1501	{ \
1502	case 0x00: return a_szPrefix ""; \
1503	case 0x01: return a_szPrefix ",Pnd"; \
1504	case 0x02: return a_szPrefix ",Dbg"; \
1505	case 0x03: return a_szPrefix ",Pnd,Dbg"; \
1506	case 0x04: return a_szPrefix ",Shw"; \
1507	case 0x05: return a_szPrefix ",Pnd,Shw"; \
1508	case 0x06: return a_szPrefix ",Shw,Dbg"; \
1509	case 0x07: return a_szPrefix ",Pnd,Shw,Dbg"; \
1510	default: AssertFailedReturn("WTF?"); \
1511	} \
1512	while (0)
1513
1514	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1515	/**
1516	* Translates the execution stat bitfield into a short log string, VID version.
1517	*
1518	* @returns Read-only log string.
1519	* @param pMsgHdr The header which state to summarize.
1520	*/
1521	static const char nemHCWinExecStateToLogStr(HV_X64_INTERCEPT_MESSAGE_HEADER const pMsgHdr)
1522	{
1523	unsigned u = (unsigned)pMsgHdr->ExecutionState.InterruptionPending
1524	\| ((unsigned)pMsgHdr->ExecutionState.DebugActive << 1)
1525	\| ((unsigned)pMsgHdr->ExecutionState.InterruptShadow << 2);
1526	if (pMsgHdr->ExecutionState.EferLma)
1527	SWITCH_IT("LM");
1528	else if (pMsgHdr->ExecutionState.Cr0Pe)
1529	SWITCH_IT("PM");
1530	else
1531	SWITCH_IT("RM");
1532	}
1533	#elif defined(IN_RING3)
1534	/**
1535	* Translates the execution stat bitfield into a short log string, WinHv version.
1536	*
1537	* @returns Read-only log string.
1538	* @param pExitCtx The exit context which state to summarize.
1539	*/
1540	static const char nemR3WinExecStateToLogStr(WHV_VP_EXIT_CONTEXT const pExitCtx)
1541	{
1542	unsigned u = (unsigned)pExitCtx->ExecutionState.InterruptionPending
1543	\| ((unsigned)pExitCtx->ExecutionState.DebugActive << 1)
1544	\| ((unsigned)pExitCtx->ExecutionState.InterruptShadow << 2);
1545	if (pExitCtx->ExecutionState.EferLma)
1546	SWITCH_IT("LM");
1547	else if (pExitCtx->ExecutionState.Cr0Pe)
1548	SWITCH_IT("PM");
1549	else
1550	SWITCH_IT("RM");
1551	}
1552	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1553	#undef SWITCH_IT
1554
1555
1556	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1557	/**
1558	* Advances the guest RIP and clear EFLAGS.RF, VID version.
1559	*
1560	* This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
1561	*
1562	* @param pVCpu The cross context virtual CPU structure.
1563	* @param pExitCtx The exit context.
1564	* @param cbMinInstr The minimum instruction length, or 1 if not unknown.
1565	*/
1566	DECLINLINE(void)
1567	nemHCWinAdvanceGuestRipAndClearRF(PVMCPU pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr, uint8_t cbMinInstr)
1568	{
1569	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
1570
1571	/* Advance the RIP. */
1572	Assert(pMsgHdr->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
1573	pVCpu->cpum.GstCtx.rip += pMsgHdr->InstructionLength;
1574	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
1575
1576	/* Update interrupt inhibition. */
1577	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1578	{ /* likely */ }
1579	else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
1580	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1581	}
1582	#elif defined(IN_RING3)
1583	/**
1584	* Advances the guest RIP and clear EFLAGS.RF, WinHv version.
1585	*
1586	* This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
1587	*
1588	* @param pVCpu The cross context virtual CPU structure.
1589	* @param pExitCtx The exit context.
1590	* @param cbMinInstr The minimum instruction length, or 1 if not unknown.
1591	*/
1592	DECLINLINE(void) nemR3WinAdvanceGuestRipAndClearRF(PVMCPU pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx, uint8_t cbMinInstr)
1593	{
1594	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
1595
1596	/* Advance the RIP. */
1597	Assert(pExitCtx->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
1598	pVCpu->cpum.GstCtx.rip += pExitCtx->InstructionLength;
1599	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
1600
1601	/* Update interrupt inhibition. */
1602	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1603	{ /* likely */ }
1604	else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
1605	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1606	}
1607	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1608
1609
1610
1611	NEM_TMPL_STATIC DECLCALLBACK(int)
1612	nemHCWinUnmapOnePageCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, uint8_t pu2NemState, void pvUser)
1613	{
1614	RT_NOREF_PV(pvUser);
1615	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1616	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1617	AssertRC(rc);
1618	if (RT_SUCCESS(rc))
1619	#else
1620	RT_NOREF_PV(pVCpu);
1621	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1622	if (SUCCEEDED(hrc))
1623	#endif
1624	{
1625	Log5(("NEM GPA unmap all: %RGp (cMappedPages=%u)\n", GCPhys, pVM->nem.s.cMappedPages - 1));
1626	*pu2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1627	}
1628	else
1629	{
1630	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1631	LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
1632	#else
1633	LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1634	GCPhys, g_apszPageStates[*pu2NemState], hrc, hrc, RTNtLastStatusValue(),
1635	RTNtLastErrorValue(), pVM->nem.s.cMappedPages));
1636	#endif
1637	*pu2NemState = NEM_WIN_PAGE_STATE_NOT_SET;
1638	}
1639	if (pVM->nem.s.cMappedPages > 0)
1640	ASMAtomicDecU32(&pVM->nem.s.cMappedPages);
1641	return VINF_SUCCESS;
1642	}
1643
1644
1645	/**
1646	* State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
1647	* and nemHCWinHandleMemoryAccessPageCheckerCallback.
1648	*/
1649	typedef struct NEMHCWINHMACPCCSTATE
1650	{
1651	/** Input: Write access. */
1652	bool fWriteAccess;
1653	/** Output: Set if we did something. */
1654	bool fDidSomething;
1655	/** Output: Set it we should resume. */
1656	bool fCanResume;
1657	} NEMHCWINHMACPCCSTATE;
1658
1659	/**
1660	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1661	* Worker for nemR3WinHandleMemoryAccess; pvUser points to a
1662	* NEMHCWINHMACPCCSTATE structure. }
1663	*/
1664	NEM_TMPL_STATIC DECLCALLBACK(int)
1665	nemHCWinHandleMemoryAccessPageCheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1666	{
1667	NEMHCWINHMACPCCSTATE pState = (NEMHCWINHMACPCCSTATE )pvUser;
1668	pState->fDidSomething = false;
1669	pState->fCanResume = false;
1670
1671	/* If A20 is disabled, we may need to make another query on the masked
1672	page to get the correct protection information. */
1673	uint8_t u2State = pInfo->u2NemState;
1674	RTGCPHYS GCPhysSrc;
1675	if ( pVM->nem.s.fA20Enabled
1676	\|\| !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
1677	GCPhysSrc = GCPhys;
1678	else
1679	{
1680	GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20);
1681	PGMPHYSNEMPAGEINFO Info2;
1682	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL);
1683	AssertRCReturn(rc, rc);
1684
1685	*pInfo = Info2;
1686	pInfo->u2NemState = u2State;
1687	}
1688
1689	/*
1690	* Consolidate current page state with actual page protection and access type.
1691	* We don't really consider downgrades here, as they shouldn't happen.
1692	*/
1693	#ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1694	/** @todo Someone at microsoft please explain:
1695	* I'm not sure WTF was going on, but I ended up in a loop if I remapped a
1696	* readonly page as writable (unmap, then map again). Specifically, this was an
1697	* issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1. So, in
1698	* a hope to work around that we no longer pre-map anything, just unmap stuff
1699	* and do it lazily here. And here we will first unmap, restart, and then remap
1700	* with new protection or backing.
1701	*/
1702	#endif
1703	int rc;
1704	switch (u2State)
1705	{
1706	case NEM_WIN_PAGE_STATE_UNMAPPED:
1707	case NEM_WIN_PAGE_STATE_NOT_SET:
1708	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1709	{
1710	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1711	return VINF_SUCCESS;
1712	}
1713
1714	/* Don't bother remapping it if it's a write request to a non-writable page. */
1715	if ( pState->fWriteAccess
1716	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1717	{
1718	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1719	return VINF_SUCCESS;
1720	}
1721
1722	/* Map the page. */
1723	rc = nemHCNativeSetPhysPage(pVM,
1724	pVCpu,
1725	GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1726	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1727	pInfo->fNemProt,
1728	&u2State,
1729	true /fBackingState/);
1730	pInfo->u2NemState = u2State;
1731	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1732	GCPhys, g_apszPageStates[u2State], rc));
1733	pState->fDidSomething = true;
1734	pState->fCanResume = true;
1735	return rc;
1736
1737	case NEM_WIN_PAGE_STATE_READABLE:
1738	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1739	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1740	{
1741	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1742	return VINF_SUCCESS;
1743	}
1744
1745	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1746	/* Upgrade page to writable. */
1747	/** @todo test this*/
1748	if ( (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1749	&& pState->fWriteAccess)
1750	{
1751	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhys,
1752	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
1753	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
1754	AssertRC(rc);
1755	if (RT_SUCCESS(rc))
1756	{
1757	pInfo->u2NemState = NEM_WIN_PAGE_STATE_WRITABLE;
1758	pState->fDidSomething = true;
1759	pState->fCanResume = true;
1760	Log5(("NEM GPA write-upgrade/exit: %RGp (was %s, cMappedPages=%u)\n",
1761	GCPhys, g_apszPageStates[u2State], pVM->nem.s.cMappedPages));
1762	}
1763	}
1764	else
1765	{
1766	/* Need to emulate the acces. */
1767	AssertBreak(pInfo->fNemProt != NEM_PAGE_PROT_NONE); /* There should be no downgrades. */
1768	rc = VINF_SUCCESS;
1769	}
1770	return rc;
1771	#else
1772	break;
1773	#endif
1774
1775	case NEM_WIN_PAGE_STATE_WRITABLE:
1776	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1777	{
1778	if (pInfo->u2OldNemState == NEM_WIN_PAGE_STATE_WRITABLE)
1779	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
1780	else
1781	{
1782	pState->fCanResume = true;
1783	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
1784	GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
1785	}
1786	return VINF_SUCCESS;
1787	}
1788	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1789	AssertFailed(); /* There should be no downgrades. */
1790	#endif
1791	break;
1792
1793	default:
1794	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1795	}
1796
1797	/*
1798	* Unmap and restart the instruction.
1799	* If this fails, which it does every so often, just unmap everything for now.
1800	*/
1801	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1802	rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1803	AssertRC(rc);
1804	if (RT_SUCCESS(rc))
1805	#else
1806	/** @todo figure out whether we mess up the state or if it's WHv. */
1807	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1808	if (SUCCEEDED(hrc))
1809	#endif
1810	{
1811	pState->fDidSomething = true;
1812	pState->fCanResume = true;
1813	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1814	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
1815	Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
1816	return VINF_SUCCESS;
1817	}
1818	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1819	LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhys, rc));
1820	return rc;
1821	#else
1822	LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1823	GCPhys, g_apszPageStates[u2State], hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue(),
1824	pVM->nem.s.cMappedPages));
1825
1826	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinUnmapOnePageCallback, NULL);
1827	Log(("nemHCWinHandleMemoryAccessPageCheckerCallback: Unmapped all (cMappedPages=%u)\n", pVM->nem.s.cMappedPages));
1828
1829	pState->fDidSomething = true;
1830	pState->fCanResume = true;
1831	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1832	return VINF_SUCCESS;
1833	#endif
1834	}
1835
1836
1837
1838	#if defined(IN_RING0) && defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API)
1839	/**
1840	* Wrapper around nemR0WinImportState that converts VERR_NEM_FLUSH_TLB
1841	* into informational status codes and logs+asserts statuses.
1842	*
1843	* @returns VBox strict status code.
1844	* @param pGVM The global (ring-0) VM structure.
1845	* @param pGVCpu The global (ring-0) per CPU structure.
1846	* @param pVCpu The cross context per CPU structure.
1847	* @param fWhat What to import.
1848	* @param pszCaller Who is doing the importing.
1849	*/
1850	DECLINLINE(VBOXSTRICTRC) nemR0WinImportStateStrict(PGVM pGVM, PGVMCPU pGVCpu, PVMCPU pVCpu, uint64_t fWhat, const char *pszCaller)
1851	{
1852	int rc = nemR0WinImportState(pGVM, pGVCpu, &pVCpu->cpum.GstCtx, fWhat, true /fCanUpdateCr3/);
1853	if (RT_SUCCESS(rc))
1854	{
1855	Assert(rc == VINF_SUCCESS);
1856	return VINF_SUCCESS;
1857	}
1858
1859	if (rc == VERR_NEM_FLUSH_TLB)
1860	{
1861	Log4(("%s/%u: nemR0WinImportState -> %Rrc\n", pszCaller, pGVCpu->idCpu, -rc));
1862	return -rc;
1863	}
1864	RT_NOREF(pszCaller);
1865	AssertMsgFailedReturn(("%s/%u: nemR0WinImportState failed: %Rrc\n", pszCaller, pGVCpu->idCpu, rc), rc);
1866	}
1867	#endif /* IN_RING0 && NEM_WIN_TEMPLATE_MODE_OWN_RUN_API*/
1868
1869	#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3)
1870	/**
1871	* Wrapper around nemR0WinImportStateStrict and nemHCWinCopyStateFromHyperV.
1872	*
1873	* Unlike the wrapped APIs, this checks whether it's necessary.
1874	*
1875	* @returns VBox strict status code.
1876	* @param pGVM The global (ring-0) VM structure.
1877	* @param pGVCpu The global (ring-0) per CPU structure.
1878	* @param fWhat What to import.
1879	* @param pszCaller Who is doing the importing.
1880	*/
1881	DECLINLINE(VBOXSTRICTRC) nemHCWinImportStateIfNeededStrict(PVMCPU pVCpu, PGVMCPU pGVCpu, uint64_t fWhat, const char *pszCaller)
1882	{
1883	if (pVCpu->cpum.GstCtx.fExtrn & fWhat)
1884	{
1885	# ifdef IN_RING0
1886	return nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pVCpu, fWhat, pszCaller);
1887	# else
1888	RT_NOREF(pGVCpu, pszCaller);
1889	int rc = nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
1890	AssertRCReturn(rc, rc);
1891	# endif
1892	}
1893	return VINF_SUCCESS;
1894	}
1895	#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API \|\| IN_RING3 */
1896
1897	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1898	/**
1899	* Copies register state from the X64 intercept message header.
1900	*
1901	* ASSUMES no state copied yet.
1902	*
1903	* @param pVCpu The cross context per CPU structure.
1904	* @param pHdr The X64 intercept message header.
1905	* @sa nemR3WinCopyStateFromX64Header
1906	*/
1907	DECLINLINE(void) nemHCWinCopyStateFromX64Header(PVMCPU pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pHdr)
1908	{
1909	Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
1910	== (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT));
1911	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pHdr->CsSegment);
1912	pVCpu->cpum.GstCtx.rip = pHdr->Rip;
1913	pVCpu->cpum.GstCtx.rflags.u = pHdr->Rflags;
1914
1915	pVCpu->nem.s.fLastInterruptShadow = pHdr->ExecutionState.InterruptShadow;
1916	if (!pHdr->ExecutionState.InterruptShadow)
1917	{
1918	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1919	{ /* likely */ }
1920	else
1921	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1922	}
1923	else
1924	EMSetInhibitInterruptsPC(pVCpu, pHdr->Rip);
1925
1926	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT);
1927	}
1928	#elif defined(IN_RING3)
1929	/**
1930	* Copies register state from the (common) exit context.
1931	*
1932	* ASSUMES no state copied yet.
1933	*
1934	* @param pVCpu The cross context per CPU structure.
1935	* @param pExitCtx The common exit context.
1936	* @sa nemHCWinCopyStateFromX64Header
1937	*/
1938	DECLINLINE(void) nemR3WinCopyStateFromX64Header(PVMCPU pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx)
1939	{
1940	Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
1941	== (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT));
1942	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pExitCtx->Cs);
1943	pVCpu->cpum.GstCtx.rip = pExitCtx->Rip;
1944	pVCpu->cpum.GstCtx.rflags.u = pExitCtx->Rflags;
1945
1946	pVCpu->nem.s.fLastInterruptShadow = pExitCtx->ExecutionState.InterruptShadow;
1947	if (!pExitCtx->ExecutionState.InterruptShadow)
1948	{
1949	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1950	{ /* likely */ }
1951	else
1952	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1953	}
1954	else
1955	EMSetInhibitInterruptsPC(pVCpu, pExitCtx->Rip);
1956
1957	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT);
1958	}
1959	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1960
1961
1962	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1963	/**
1964	* Deals with memory intercept message.
1965	*
1966	* @returns Strict VBox status code.
1967	* @param pVM The cross context VM structure.
1968	* @param pVCpu The cross context per CPU structure.
1969	* @param pMsg The message.
1970	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
1971	* @sa nemR3WinHandleExitMemory
1972	*/
1973	NEM_TMPL_STATIC VBOXSTRICTRC
1974	nemHCWinHandleMessageMemory(PVM pVM, PVMCPU pVCpu, HV_X64_MEMORY_INTERCEPT_MESSAGE const *pMsg, PGVMCPU pGVCpu)
1975	{
1976	uint64_t const uHostTsc = ASMReadTSC();
1977	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
1978	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
1979	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
1980
1981	/*
1982	* Whatever we do, we must clear pending event injection upon resume.
1983	*/
1984	if (pMsg->Header.ExecutionState.InterruptionPending)
1985	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
1986
1987	# if 0 /* Experiment: 20K -> 34K exit/s. */
1988	if ( pMsg->Header.ExecutionState.EferLma
1989	&& pMsg->Header.CsSegment.Long
1990	&& pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
1991	{
1992	if ( pMsg->Header.Rip - (uint64_t)0xf65a < (uint64_t)(0xf662 - 0xf65a)
1993	&& pMsg->InstructionBytes[0] == 0x89
1994	&& pMsg->InstructionBytes[1] == 0x03)
1995	{
1996	pVCpu->cpum.GstCtx.rip = pMsg->Header.Rip + 2;
1997	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
1998	AssertMsg(pMsg->Header.InstructionLength == 2, ("%#x\n", pMsg->Header.InstructionLength));
1999	//Log(("%RX64 msg:\n%.80Rhxd\n", pVCpu->cpum.GstCtx.rip, pMsg));
2000	return VINF_SUCCESS;
2001	}
2002	}
2003	# endif
2004
2005	/*
2006	* Ask PGM for information about the given GCPhys. We need to check if we're
2007	* out of sync first.
2008	*/
2009	NEMHCWINHMACPCCSTATE State = { pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE, false, false };
2010	PGMPHYSNEMPAGEINFO Info;
2011	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pMsg->GuestPhysicalAddress, State.fWriteAccess, &Info,
2012	nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
2013	if (RT_SUCCESS(rc))
2014	{
2015	if (Info.fNemProt & ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2016	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
2017	{
2018	if (State.fCanResume)
2019	{
2020	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
2021	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2022	pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2023	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2024	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2025	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
2026	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
2027	return VINF_SUCCESS;
2028	}
2029	}
2030	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
2031	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2032	pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2033	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2034	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2035	}
2036	else
2037	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
2038	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2039	pMsg->GuestPhysicalAddress, rc, State.fDidSomething ? " modified-backing" : "",
2040	g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2041
2042	/*
2043	* Emulate the memory access, either access handler or special memory.
2044	*/
2045	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2046	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2047	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2048	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2049	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
2050	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2051	VBOXSTRICTRC rcStrict;
2052	# ifdef IN_RING0
2053	rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pVCpu,
2054	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES, "MemExit");
2055	if (rcStrict != VINF_SUCCESS)
2056	return rcStrict;
2057	# else
2058	rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2059	AssertRCReturn(rc, rc);
2060	NOREF(pGVCpu);
2061	# endif
2062
2063	if (pMsg->Reserved1)
2064	Log(("MemExit/Reserved1=%#x\n", pMsg->Reserved1));
2065	if (pMsg->Header.ExecutionState.Reserved0 \|\| pMsg->Header.ExecutionState.Reserved1)
2066	Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pMsg->Header.ExecutionState.Reserved0, pMsg->Header.ExecutionState.Reserved1));
2067
2068	if (!pExitRec)
2069	{
2070	//if (pMsg->InstructionByteCount > 0)
2071	// Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
2072	if (pMsg->InstructionByteCount > 0)
2073	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
2074	pMsg->InstructionBytes, pMsg->InstructionByteCount);
2075	else
2076	rcStrict = IEMExecOne(pVCpu);
2077	/** @todo do we need to do anything wrt debugging here? */
2078	}
2079	else
2080	{
2081	/* Frequent access or probing. */
2082	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2083	Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2084	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2085	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2086	}
2087	return rcStrict;
2088	}
2089	#elif defined(IN_RING3)
2090	/**
2091	* Deals with memory access exits (WHvRunVpExitReasonMemoryAccess).
2092	*
2093	* @returns Strict VBox status code.
2094	* @param pVM The cross context VM structure.
2095	* @param pVCpu The cross context per CPU structure.
2096	* @param pExit The VM exit information to handle.
2097	* @sa nemHCWinHandleMessageMemory
2098	*/
2099	NEM_TMPL_STATIC VBOXSTRICTRC
2100	nemR3WinHandleExitMemory(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2101	{
2102	uint64_t const uHostTsc = ASMReadTSC();
2103	Assert(pExit->MemoryAccess.AccessInfo.AccessType != 3);
2104
2105	/*
2106	* Whatever we do, we must clear pending event injection upon resume.
2107	*/
2108	if (pExit->VpContext.ExecutionState.InterruptionPending)
2109	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2110
2111	/*
2112	* Ask PGM for information about the given GCPhys. We need to check if we're
2113	* out of sync first.
2114	*/
2115	NEMHCWINHMACPCCSTATE State = { pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite, false, false };
2116	PGMPHYSNEMPAGEINFO Info;
2117	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pExit->MemoryAccess.Gpa, State.fWriteAccess, &Info,
2118	nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
2119	if (RT_SUCCESS(rc))
2120	{
2121	if (Info.fNemProt & ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2122	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
2123	{
2124	if (State.fCanResume)
2125	{
2126	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
2127	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2128	pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2129	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2130	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2131	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
2132	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
2133	return VINF_SUCCESS;
2134	}
2135	}
2136	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
2137	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2138	pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2139	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2140	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2141	}
2142	else
2143	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
2144	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2145	pExit->MemoryAccess.Gpa, rc, State.fDidSomething ? " modified-backing" : "",
2146	g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2147
2148	/*
2149	* Emulate the memory access, either access handler or special memory.
2150	*/
2151	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2152	pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2153	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2154	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2155	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
2156	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2157	rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2158	AssertRCReturn(rc, rc);
2159	if (pExit->VpContext.ExecutionState.Reserved0 \|\| pExit->VpContext.ExecutionState.Reserved1)
2160	Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pExit->VpContext.ExecutionState.Reserved0, pExit->VpContext.ExecutionState.Reserved1));
2161
2162	VBOXSTRICTRC rcStrict;
2163	if (!pExitRec)
2164	{
2165	//if (pMsg->InstructionByteCount > 0)
2166	// Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
2167	if (pExit->MemoryAccess.InstructionByteCount > 0)
2168	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
2169	pExit->MemoryAccess.InstructionBytes, pExit->MemoryAccess.InstructionByteCount);
2170	else
2171	rcStrict = IEMExecOne(pVCpu);
2172	/** @todo do we need to do anything wrt debugging here? */
2173	}
2174	else
2175	{
2176	/* Frequent access or probing. */
2177	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2178	Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2179	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2180	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2181	}
2182	return rcStrict;
2183	}
2184	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2185
2186
2187	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2188	/**
2189	* Deals with I/O port intercept message.
2190	*
2191	* @returns Strict VBox status code.
2192	* @param pVM The cross context VM structure.
2193	* @param pVCpu The cross context per CPU structure.
2194	* @param pMsg The message.
2195	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
2196	*/
2197	NEM_TMPL_STATIC VBOXSTRICTRC
2198	nemHCWinHandleMessageIoPort(PVM pVM, PVMCPU pVCpu, HV_X64_IO_PORT_INTERCEPT_MESSAGE const *pMsg, PGVMCPU pGVCpu)
2199	{
2200	/*
2201	* Assert message sanity.
2202	*/
2203	Assert( pMsg->AccessInfo.AccessSize == 1
2204	\|\| pMsg->AccessInfo.AccessSize == 2
2205	\|\| pMsg->AccessInfo.AccessSize == 4);
2206	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
2207	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2208	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2209	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2210	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2211	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2212	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRax, pMsg->Rax);
2213	if (pMsg->AccessInfo.StringOp)
2214	{
2215	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterDs, pMsg->DsSegment);
2216	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterEs, pMsg->EsSegment);
2217	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRcx, pMsg->Rcx);
2218	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRsi, pMsg->Rsi);
2219	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRdi, pMsg->Rdi);
2220	}
2221
2222	/*
2223	* Whatever we do, we must clear pending event injection upon resume.
2224	*/
2225	if (pMsg->Header.ExecutionState.InterruptionPending)
2226	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2227
2228	/*
2229	* Add history first to avoid two paths doing EMHistoryExec calls.
2230	*/
2231	VBOXSTRICTRC rcStrict;
2232	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2233	!pMsg->AccessInfo.StringOp
2234	? ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2235	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
2236	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
2237	: ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2238	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
2239	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
2240	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2241	if (!pExitRec)
2242	{
2243	if (!pMsg->AccessInfo.StringOp)
2244	{
2245	/*
2246	* Simple port I/O.
2247	*/
2248	static uint32_t const s_fAndMask[8] =
2249	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
2250	uint32_t const fAndMask = s_fAndMask[pMsg->AccessInfo.AccessSize];
2251
2252	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2253	if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2254	{
2255	rcStrict = IOMIOPortWrite(pVM, pVCpu, pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize);
2256	Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2257	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2258	pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
2259	if (IOM_SUCCESS(rcStrict))
2260	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
2261	# ifdef IN_RING0
2262	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
2263	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
2264	/** @todo check for debug breakpoints */ )
2265	return EMRZSetPendingIoPortWrite(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
2266	pMsg->AccessInfo.AccessSize, (uint32_t)pMsg->Rax & fAndMask);
2267	# endif
2268	else
2269	{
2270	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2271	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2272	}
2273	}
2274	else
2275	{
2276	uint32_t uValue = 0;
2277	rcStrict = IOMIOPortRead(pVM, pVCpu, pMsg->PortNumber, &uValue, pMsg->AccessInfo.AccessSize);
2278	Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2279	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2280	pMsg->PortNumber, pMsg->AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2281	if (IOM_SUCCESS(rcStrict))
2282	{
2283	if (pMsg->AccessInfo.AccessSize != 4)
2284	pVCpu->cpum.GstCtx.rax = (pMsg->Rax & ~(uint64_t)fAndMask) \| (uValue & fAndMask);
2285	else
2286	pVCpu->cpum.GstCtx.rax = uValue;
2287	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2288	Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pMsg->Rax, pVCpu->cpum.GstCtx.rax));
2289	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
2290	}
2291	else
2292	{
2293	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2294	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2295	# ifdef IN_RING0
2296	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
2297	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
2298	/** @todo check for debug breakpoints */ )
2299	return EMRZSetPendingIoPortRead(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
2300	pMsg->AccessInfo.AccessSize);
2301	# endif
2302	}
2303	}
2304	}
2305	else
2306	{
2307	/*
2308	* String port I/O.
2309	*/
2310	/** @todo Someone at Microsoft please explain how we can get the address mode
2311	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
2312	* getting the default mode, it can always be overridden by a prefix. This
2313	* forces us to interpret the instruction from opcodes, which is suboptimal.
2314	* Both AMD-V and VT-x includes the address size in the exit info, at least on
2315	* CPUs that are reasonably new.
2316	*
2317	* Of course, it's possible this is an undocumented and we just need to do some
2318	* experiments to figure out how it's communicated. Alternatively, we can scan
2319	* the opcode bytes for possible evil prefixes.
2320	*/
2321	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2322	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2323	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2324	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
2325	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
2326	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2327	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2328	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
2329	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
2330	# ifdef IN_RING0
2331	rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
2332	if (rcStrict != VINF_SUCCESS)
2333	return rcStrict;
2334	# else
2335	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2336	AssertRCReturn(rc, rc);
2337	RT_NOREF(pGVCpu);
2338	# endif
2339
2340	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
2341	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2342	pMsg->AccessInfo.RepPrefix ? "REP " : "",
2343	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUTS" : "INS",
2344	pMsg->PortNumber, pMsg->AccessInfo.AccessSize ));
2345	rcStrict = IEMExecOne(pVCpu);
2346	}
2347	if (IOM_SUCCESS(rcStrict))
2348	{
2349	/*
2350	* Do debug checks.
2351	*/
2352	if ( pMsg->Header.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
2353	\|\| (pMsg->Header.Rflags & X86_EFL_TF)
2354	\|\| DBGFBpIsHwIoArmed(pVM) )
2355	{
2356	/** @todo Debugging. */
2357	}
2358	}
2359	return rcStrict;
2360	}
2361
2362	/*
2363	* Frequent exit or something needing probing.
2364	* Get state and call EMHistoryExec.
2365	*/
2366	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2367	if (!pMsg->AccessInfo.StringOp)
2368	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2369	else
2370	{
2371	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2372	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2373	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
2374	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
2375	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2376	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
2377	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
2378	}
2379	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2380
2381	# ifdef IN_RING0
2382	rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
2383	if (rcStrict != VINF_SUCCESS)
2384	return rcStrict;
2385	# else
2386	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2387	AssertRCReturn(rc, rc);
2388	RT_NOREF(pGVCpu);
2389	# endif
2390
2391	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
2392	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2393	pMsg->AccessInfo.RepPrefix ? "REP " : "",
2394	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUT" : "IN",
2395	pMsg->AccessInfo.StringOp ? "S" : "",
2396	pMsg->PortNumber, pMsg->AccessInfo.AccessSize));
2397	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2398	Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2399	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2400	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2401	return rcStrict;
2402	}
2403	#elif defined(IN_RING3)
2404	/**
2405	* Deals with I/O port access exits (WHvRunVpExitReasonX64IoPortAccess).
2406	*
2407	* @returns Strict VBox status code.
2408	* @param pVM The cross context VM structure.
2409	* @param pVCpu The cross context per CPU structure.
2410	* @param pExit The VM exit information to handle.
2411	* @sa nemHCWinHandleMessageIoPort
2412	*/
2413	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitIoPort(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2414	{
2415	Assert( pExit->IoPortAccess.AccessInfo.AccessSize == 1
2416	\|\| pExit->IoPortAccess.AccessInfo.AccessSize == 2
2417	\|\| pExit->IoPortAccess.AccessInfo.AccessSize == 4);
2418
2419	/*
2420	* Whatever we do, we must clear pending event injection upon resume.
2421	*/
2422	if (pExit->VpContext.ExecutionState.InterruptionPending)
2423	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2424
2425	/*
2426	* Add history first to avoid two paths doing EMHistoryExec calls.
2427	*/
2428	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2429	!pExit->IoPortAccess.AccessInfo.StringOp
2430	? ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2431	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
2432	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
2433	: ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2434	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
2435	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
2436	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2437	if (!pExitRec)
2438	{
2439	VBOXSTRICTRC rcStrict;
2440	if (!pExit->IoPortAccess.AccessInfo.StringOp)
2441	{
2442	/*
2443	* Simple port I/O.
2444	*/
2445	static uint32_t const s_fAndMask[8] =
2446	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
2447	uint32_t const fAndMask = s_fAndMask[pExit->IoPortAccess.AccessInfo.AccessSize];
2448	if (pExit->IoPortAccess.AccessInfo.IsWrite)
2449	{
2450	rcStrict = IOMIOPortWrite(pVM, pVCpu, pExit->IoPortAccess.PortNumber,
2451	(uint32_t)pExit->IoPortAccess.Rax & fAndMask,
2452	pExit->IoPortAccess.AccessInfo.AccessSize);
2453	Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2454	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2455	pExit->IoPortAccess.PortNumber, (uint32_t)pExit->IoPortAccess.Rax & fAndMask,
2456	pExit->IoPortAccess.AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
2457	if (IOM_SUCCESS(rcStrict))
2458	{
2459	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2460	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
2461	}
2462	}
2463	else
2464	{
2465	uint32_t uValue = 0;
2466	rcStrict = IOMIOPortRead(pVM, pVCpu, pExit->IoPortAccess.PortNumber, &uValue,
2467	pExit->IoPortAccess.AccessInfo.AccessSize);
2468	Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2469	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2470	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2471	if (IOM_SUCCESS(rcStrict))
2472	{
2473	if (pExit->IoPortAccess.AccessInfo.AccessSize != 4)
2474	pVCpu->cpum.GstCtx.rax = (pExit->IoPortAccess.Rax & ~(uint64_t)fAndMask) \| (uValue & fAndMask);
2475	else
2476	pVCpu->cpum.GstCtx.rax = uValue;
2477	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2478	Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pExit->IoPortAccess.Rax, pVCpu->cpum.GstCtx.rax));
2479	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2480	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
2481	}
2482	}
2483	}
2484	else
2485	{
2486	/*
2487	* String port I/O.
2488	*/
2489	/** @todo Someone at Microsoft please explain how we can get the address mode
2490	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
2491	* getting the default mode, it can always be overridden by a prefix. This
2492	* forces us to interpret the instruction from opcodes, which is suboptimal.
2493	* Both AMD-V and VT-x includes the address size in the exit info, at least on
2494	* CPUs that are reasonably new.
2495	*
2496	* Of course, it's possible this is an undocumented and we just need to do some
2497	* experiments to figure out how it's communicated. Alternatively, we can scan
2498	* the opcode bytes for possible evil prefixes.
2499	*/
2500	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2501	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2502	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2503	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
2504	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
2505	pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
2506	pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
2507	pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
2508	pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
2509	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2510	AssertRCReturn(rc, rc);
2511
2512	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
2513	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2514	pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
2515	pExit->IoPortAccess.AccessInfo.IsWrite ? "OUTS" : "INS",
2516	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize ));
2517	rcStrict = IEMExecOne(pVCpu);
2518	}
2519	if (IOM_SUCCESS(rcStrict))
2520	{
2521	/*
2522	* Do debug checks.
2523	*/
2524	if ( pExit->VpContext.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
2525	\|\| (pExit->VpContext.Rflags & X86_EFL_TF)
2526	\|\| DBGFBpIsHwIoArmed(pVM) )
2527	{
2528	/** @todo Debugging. */
2529	}
2530	}
2531	return rcStrict;
2532	}
2533
2534	/*
2535	* Frequent exit or something needing probing.
2536	* Get state and call EMHistoryExec.
2537	*/
2538	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2539	if (!pExit->IoPortAccess.AccessInfo.StringOp)
2540	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2541	else
2542	{
2543	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2544	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2545	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
2546	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
2547	pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
2548	pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
2549	pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
2550	}
2551	pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
2552	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2553	AssertRCReturn(rc, rc);
2554	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
2555	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2556	pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
2557	pExit->IoPortAccess.AccessInfo.IsWrite ? "OUT" : "IN",
2558	pExit->IoPortAccess.AccessInfo.StringOp ? "S" : "",
2559	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize));
2560	VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2561	Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2562	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2563	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2564	return rcStrict;
2565	}
2566	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2567
2568
2569	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2570	/**
2571	* Deals with interrupt window message.
2572	*
2573	* @returns Strict VBox status code.
2574	* @param pVM The cross context VM structure.
2575	* @param pVCpu The cross context per CPU structure.
2576	* @param pMsg The message.
2577	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
2578	* @sa nemR3WinHandleExitInterruptWindow
2579	*/
2580	NEM_TMPL_STATIC VBOXSTRICTRC
2581	nemHCWinHandleMessageInterruptWindow(PVM pVM, PVMCPU pVCpu, HV_X64_INTERRUPT_WINDOW_MESSAGE const *pMsg, PGVMCPU pGVCpu)
2582	{
2583	/*
2584	* Assert message sanity.
2585	*/
2586	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE
2587	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ // READ & WRITE are probably not used here
2588	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2589	AssertMsg(pMsg->Type == HvX64PendingInterrupt \|\| pMsg->Type == HvX64PendingNmi, ("%#x\n", pMsg->Type));
2590
2591	/*
2592	* Just copy the state we've got and handle it in the loop for now.
2593	*/
2594	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2595	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2596
2597	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2598	Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d\n",
2599	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2600	pMsg->Type, RT_BOOL(pMsg->Header.Rflags & X86_EFL_IF), pMsg->Header.ExecutionState.InterruptShadow));
2601
2602	/** @todo call nemHCWinHandleInterruptFF */
2603	RT_NOREF(pVM, pGVCpu);
2604	return VINF_SUCCESS;
2605	}
2606	#elif defined(IN_RING3)
2607	/**
2608	* Deals with interrupt window exits (WHvRunVpExitReasonX64InterruptWindow).
2609	*
2610	* @returns Strict VBox status code.
2611	* @param pVM The cross context VM structure.
2612	* @param pVCpu The cross context per CPU structure.
2613	* @param pExit The VM exit information to handle.
2614	* @sa nemHCWinHandleMessageInterruptWindow
2615	*/
2616	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitInterruptWindow(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2617	{
2618	/*
2619	* Assert message sanity.
2620	*/
2621	AssertMsg( pExit->InterruptWindow.DeliverableType == WHvX64PendingInterrupt
2622	\|\| pExit->InterruptWindow.DeliverableType == WHvX64PendingNmi,
2623	("%#x\n", pExit->InterruptWindow.DeliverableType));
2624
2625	/*
2626	* Just copy the state we've got and handle it in the loop for now.
2627	*/
2628	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2629	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2630
2631	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2632	Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d\n",
2633	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2634	pExit->InterruptWindow.DeliverableType, RT_BOOL(pExit->VpContext.Rflags & X86_EFL_IF),
2635	pExit->VpContext.ExecutionState.InterruptShadow));
2636
2637	/** @todo call nemHCWinHandleInterruptFF */
2638	RT_NOREF(pVM);
2639	return VINF_SUCCESS;
2640	}
2641	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2642
2643
2644	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2645	/**
2646	* Deals with CPUID intercept message.
2647	*
2648	* @returns Strict VBox status code.
2649	* @param pVM The cross context VM structure.
2650	* @param pVCpu The cross context per CPU structure.
2651	* @param pMsg The message.
2652	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
2653	* @sa nemR3WinHandleExitCpuId
2654	*/
2655	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageCpuId(PVM pVM, PVMCPU pVCpu, HV_X64_CPUID_INTERCEPT_MESSAGE const *pMsg,
2656	PGVMCPU pGVCpu)
2657	{
2658	/* Check message register value sanity. */
2659	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2660	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2661	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2662	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2663	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRax, pMsg->Rax);
2664	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRcx, pMsg->Rcx);
2665	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRdx, pMsg->Rdx);
2666	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRbx, pMsg->Rbx);
2667
2668	/* Do exit history. */
2669	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
2670	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2671	if (!pExitRec)
2672	{
2673	/*
2674	* Soak up state and execute the instruction.
2675	*
2676	* Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
2677	* function and make everyone use it.
2678	*/
2679	/** @todo Combine implementations into IEMExecDecodedCpuId as this will
2680	* only get weirder with nested VT-x and AMD-V support. */
2681	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2682
2683	/* Copy in the low register values (top is always cleared). */
2684	pVCpu->cpum.GstCtx.rax = (uint32_t)pMsg->Rax;
2685	pVCpu->cpum.GstCtx.rcx = (uint32_t)pMsg->Rcx;
2686	pVCpu->cpum.GstCtx.rdx = (uint32_t)pMsg->Rdx;
2687	pVCpu->cpum.GstCtx.rbx = (uint32_t)pMsg->Rbx;
2688	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2689
2690	/* Get the correct values. */
2691	CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
2692	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
2693
2694	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
2695	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2696	pMsg->Rax, pMsg->Rcx, pMsg->Rdx, pMsg->Rbx,
2697	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx,
2698	pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
2699
2700	/* Move RIP and we're done. */
2701	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2702
2703	return VINF_SUCCESS;
2704	}
2705
2706	/*
2707	* Frequent exit or something needing probing.
2708	* Get state and call EMHistoryExec.
2709	*/
2710	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2711	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2712	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2713	pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
2714	pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
2715	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2716	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
2717	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2718	pMsg->Rax, pMsg->Rcx, pMsg->Rdx, pMsg->Rbx,
2719	pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
2720	# ifdef IN_RING0
2721	VBOXSTRICTRC rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "CpuIdExit");
2722	if (rcStrict != VINF_SUCCESS)
2723	return rcStrict;
2724	RT_NOREF(pVM);
2725	# else
2726	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2727	AssertRCReturn(rc, rc);
2728	RT_NOREF(pGVCpu);
2729	# endif
2730	VBOXSTRICTRC rcStrictExec = EMHistoryExec(pVCpu, pExitRec, 0);
2731	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2732	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2733	VBOXSTRICTRC_VAL(rcStrictExec), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2734	return rcStrictExec;
2735	}
2736	#elif defined(IN_RING3)
2737	/**
2738	* Deals with CPUID exits (WHvRunVpExitReasonX64Cpuid).
2739	*
2740	* @returns Strict VBox status code.
2741	* @param pVM The cross context VM structure.
2742	* @param pVCpu The cross context per CPU structure.
2743	* @param pExit The VM exit information to handle.
2744	* @sa nemHCWinHandleMessageCpuId
2745	*/
2746	NEM_TMPL_STATIC VBOXSTRICTRC
2747	nemR3WinHandleExitCpuId(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2748	{
2749	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
2750	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2751	if (!pExitRec)
2752	{
2753	/*
2754	* Soak up state and execute the instruction.
2755	*
2756	* Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
2757	* function and make everyone use it.
2758	*/
2759	/** @todo Combine implementations into IEMExecDecodedCpuId as this will
2760	* only get weirder with nested VT-x and AMD-V support. */
2761	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2762
2763	/* Copy in the low register values (top is always cleared). */
2764	pVCpu->cpum.GstCtx.rax = (uint32_t)pExit->CpuidAccess.Rax;
2765	pVCpu->cpum.GstCtx.rcx = (uint32_t)pExit->CpuidAccess.Rcx;
2766	pVCpu->cpum.GstCtx.rdx = (uint32_t)pExit->CpuidAccess.Rdx;
2767	pVCpu->cpum.GstCtx.rbx = (uint32_t)pExit->CpuidAccess.Rbx;
2768	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2769
2770	/* Get the correct values. */
2771	CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
2772	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
2773
2774	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
2775	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2776	pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx,
2777	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx,
2778	pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
2779
2780	/* Move RIP and we're done. */
2781	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
2782
2783	RT_NOREF_PV(pVM);
2784	return VINF_SUCCESS;
2785	}
2786
2787	/*
2788	* Frequent exit or something needing probing.
2789	* Get state and call EMHistoryExec.
2790	*/
2791	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2792	pVCpu->cpum.GstCtx.rax = pExit->CpuidAccess.Rax;
2793	pVCpu->cpum.GstCtx.rcx = pExit->CpuidAccess.Rcx;
2794	pVCpu->cpum.GstCtx.rdx = pExit->CpuidAccess.Rdx;
2795	pVCpu->cpum.GstCtx.rbx = pExit->CpuidAccess.Rbx;
2796	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2797	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
2798	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2799	pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx,
2800	pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
2801	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2802	AssertRCReturn(rc, rc);
2803	VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2804	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2805	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2806	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2807	return rcStrict;
2808	}
2809	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2810
2811
2812	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2813	/**
2814	* Deals with MSR intercept message.
2815	*
2816	* @returns Strict VBox status code.
2817	* @param pVCpu The cross context per CPU structure.
2818	* @param pMsg The message.
2819	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
2820	* @sa nemR3WinHandleExitMsr
2821	*/
2822	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageMsr(PVMCPU pVCpu, HV_X64_MSR_INTERCEPT_MESSAGE const *pMsg, PGVMCPU pGVCpu)
2823	{
2824	/*
2825	* A wee bit of sanity first.
2826	*/
2827	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
2828	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2829	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2830	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2831	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2832	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2833	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRax, pMsg->Rax);
2834	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRdx, pMsg->Rdx);
2835
2836	/*
2837	* Check CPL as that's common to both RDMSR and WRMSR.
2838	*/
2839	VBOXSTRICTRC rcStrict;
2840	if (pMsg->Header.ExecutionState.Cpl == 0)
2841	{
2842	/*
2843	* Get all the MSR state. Since we're getting EFER, we also need to
2844	* get CR0, CR4 and CR3.
2845	*/
2846	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2847	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2848	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
2849	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2850	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2851
2852	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2853	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, pGVCpu,
2854	(!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
2855	\| CPUMCTX_EXTRN_ALL_MSRS \| CPUMCTX_EXTRN_CR0
2856	\| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4,
2857	"MSRs");
2858	if (rcStrict == VINF_SUCCESS)
2859	{
2860	if (!pExitRec)
2861	{
2862	/*
2863	* Handle writes.
2864	*/
2865	if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2866	{
2867	rcStrict = CPUMSetGuestMsr(pVCpu, pMsg->MsrNumber, RT_MAKE_U64((uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx));
2868	Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n",
2869	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2870	pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
2871	if (rcStrict == VINF_SUCCESS)
2872	{
2873	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2874	return VINF_SUCCESS;
2875	}
2876	# ifndef IN_RING3
2877	/* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
2878	if (rcStrict == VERR_CPUM_RAISE_GP_0)
2879	rcStrict = VINF_CPUM_R3_MSR_WRITE;
2880	return rcStrict;
2881	# else
2882	LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n",
2883	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2884	pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
2885	# endif
2886	}
2887	/*
2888	* Handle reads.
2889	*/
2890	else
2891	{
2892	uint64_t uValue = 0;
2893	rcStrict = CPUMQueryGuestMsr(pVCpu, pMsg->MsrNumber, &uValue);
2894	Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
2895	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2896	pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2897	if (rcStrict == VINF_SUCCESS)
2898	{
2899	pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
2900	pVCpu->cpum.GstCtx.rdx = uValue >> 32;
2901	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
2902	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2903	return VINF_SUCCESS;
2904	}
2905	# ifndef IN_RING3
2906	/* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
2907	if (rcStrict == VERR_CPUM_RAISE_GP_0)
2908	rcStrict = VINF_CPUM_R3_MSR_READ;
2909	return rcStrict;
2910	# else
2911	LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
2912	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2913	pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2914	# endif
2915	}
2916	}
2917	else
2918	{
2919	/*
2920	* Handle frequent exit or something needing probing.
2921	*/
2922	Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
2923	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2924	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD", pMsg->MsrNumber));
2925	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2926	Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2927	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2928	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2929	return rcStrict;
2930	}
2931	}
2932	else
2933	{
2934	LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
2935	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2936	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD",
2937	pMsg->MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
2938	return rcStrict;
2939	}
2940	}
2941	else if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2942	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n",
2943	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2944	pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx ));
2945	else
2946	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n",
2947	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2948	pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber));
2949
2950	/*
2951	* If we get down here, we're supposed to #GP(0).
2952	*/
2953	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, pGVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL_MSRS, "MSR");
2954	if (rcStrict == VINF_SUCCESS)
2955	{
2956	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
2957	if (rcStrict == VINF_IEM_RAISED_XCPT)
2958	rcStrict = VINF_SUCCESS;
2959	else if (rcStrict != VINF_SUCCESS)
2960	Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
2961	}
2962	return rcStrict;
2963	}
2964	#elif defined(IN_RING3)
2965	/**
2966	* Deals with MSR access exits (WHvRunVpExitReasonX64MsrAccess).
2967	*
2968	* @returns Strict VBox status code.
2969	* @param pVM The cross context VM structure.
2970	* @param pVCpu The cross context per CPU structure.
2971	* @param pExit The VM exit information to handle.
2972	* @sa nemHCWinHandleMessageMsr
2973	*/
2974	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitMsr(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2975	{
2976	/*
2977	* Check CPL as that's common to both RDMSR and WRMSR.
2978	*/
2979	VBOXSTRICTRC rcStrict;
2980	if (pExit->VpContext.ExecutionState.Cpl == 0)
2981	{
2982	/*
2983	* Get all the MSR state. Since we're getting EFER, we also need to
2984	* get CR0, CR4 and CR3.
2985	*/
2986	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2987	pExit->MsrAccess.AccessInfo.IsWrite
2988	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
2989	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2990	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2991	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2992	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NULL,
2993	(!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
2994	\| CPUMCTX_EXTRN_ALL_MSRS \| CPUMCTX_EXTRN_CR0
2995	\| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4,
2996	"MSRs");
2997	if (rcStrict == VINF_SUCCESS)
2998	{
2999	if (!pExitRec)
3000	{
3001	/*
3002	* Handle writes.
3003	*/
3004	if (pExit->MsrAccess.AccessInfo.IsWrite)
3005	{
3006	rcStrict = CPUMSetGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber,
3007	RT_MAKE_U64((uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx));
3008	Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3009	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
3010	(uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
3011	if (rcStrict == VINF_SUCCESS)
3012	{
3013	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
3014	return VINF_SUCCESS;
3015	}
3016	LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n", pVCpu->idCpu,
3017	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3018	pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx,
3019	VBOXSTRICTRC_VAL(rcStrict) ));
3020	}
3021	/*
3022	* Handle reads.
3023	*/
3024	else
3025	{
3026	uint64_t uValue = 0;
3027	rcStrict = CPUMQueryGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber, &uValue);
3028	Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu,
3029	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3030	pExit->MsrAccess.MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
3031	if (rcStrict == VINF_SUCCESS)
3032	{
3033	pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
3034	pVCpu->cpum.GstCtx.rdx = uValue >> 32;
3035	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
3036	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
3037	return VINF_SUCCESS;
3038	}
3039	LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3040	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
3041	uValue, VBOXSTRICTRC_VAL(rcStrict) ));
3042	}
3043	}
3044	else
3045	{
3046	/*
3047	* Handle frequent exit or something needing probing.
3048	*/
3049	Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
3050	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3051	pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber));
3052	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
3053	Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
3054	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3055	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
3056	return rcStrict;
3057	}
3058	}
3059	else
3060	{
3061	LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
3062	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3063	pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
3064	return rcStrict;
3065	}
3066	}
3067	else if (pExit->MsrAccess.AccessInfo.IsWrite)
3068	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3069	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
3070	pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx ));
3071	else
3072	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3073	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
3074	pExit->MsrAccess.MsrNumber));
3075
3076	/*
3077	* If we get down here, we're supposed to #GP(0).
3078	*/
3079	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NULL,
3080	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL_MSRS, "MSR");
3081	if (rcStrict == VINF_SUCCESS)
3082	{
3083	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
3084	if (rcStrict == VINF_IEM_RAISED_XCPT)
3085	rcStrict = VINF_SUCCESS;
3086	else if (rcStrict != VINF_SUCCESS)
3087	Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
3088	}
3089
3090	RT_NOREF_PV(pVM);
3091	return rcStrict;
3092	}
3093	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3094
3095
3096	/**
3097	* Worker for nemHCWinHandleMessageException & nemR3WinHandleExitException that
3098	* checks if the given opcodes are of interest at all.
3099	*
3100	* @returns true if interesting, false if not.
3101	* @param cbOpcodes Number of opcode bytes available.
3102	* @param pbOpcodes The opcode bytes.
3103	* @param f64BitMode Whether we're in 64-bit mode.
3104	*/
3105	DECLINLINE(bool) nemHcWinIsInterestingUndefinedOpcode(uint8_t cbOpcodes, uint8_t const *pbOpcodes, bool f64BitMode)
3106	{
3107	/*
3108	* Currently only interested in VMCALL and VMMCALL.
3109	*/
3110	while (cbOpcodes >= 3)
3111	{
3112	switch (pbOpcodes[0])
3113	{
3114	case 0x0f:
3115	switch (pbOpcodes[1])
3116	{
3117	case 0x01:
3118	switch (pbOpcodes[2])
3119	{
3120	case 0xc1: /* 0f 01 c1 VMCALL */
3121	return true;
3122	case 0xd9: /* 0f 01 d9 VMMCALL */
3123	return true;
3124	default:
3125	break;
3126	}
3127	break;
3128	}
3129	break;
3130
3131	default:
3132	return false;
3133
3134	/* prefixes */
3135	case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47:
3136	case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
3137	if (!f64BitMode)
3138	return false;
3139	RT_FALL_THRU();
3140	case X86_OP_PRF_CS:
3141	case X86_OP_PRF_SS:
3142	case X86_OP_PRF_DS:
3143	case X86_OP_PRF_ES:
3144	case X86_OP_PRF_FS:
3145	case X86_OP_PRF_GS:
3146	case X86_OP_PRF_SIZE_OP:
3147	case X86_OP_PRF_SIZE_ADDR:
3148	case X86_OP_PRF_LOCK:
3149	case X86_OP_PRF_REPZ:
3150	case X86_OP_PRF_REPNZ:
3151	cbOpcodes--;
3152	pbOpcodes++;
3153	continue;
3154	}
3155	break;
3156	}
3157	return false;
3158	}
3159
3160
3161	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3162	/**
3163	* Copies state included in a exception intercept message.
3164	*
3165	* @param pVCpu The cross context per CPU structure.
3166	* @param pMsg The message.
3167	* @param fClearXcpt Clear pending exception.
3168	*/
3169	DECLINLINE(void)
3170	nemHCWinCopyStateFromExceptionMessage(PVMCPU pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg, bool fClearXcpt)
3171	{
3172	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
3173	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_DS
3174	\| (fClearXcpt ? CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT : 0) );
3175	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
3176	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
3177	pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
3178	pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
3179	pVCpu->cpum.GstCtx.rsp = pMsg->Rsp;
3180	pVCpu->cpum.GstCtx.rbp = pMsg->Rbp;
3181	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
3182	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
3183	pVCpu->cpum.GstCtx.r8 = pMsg->R8;
3184	pVCpu->cpum.GstCtx.r9 = pMsg->R9;
3185	pVCpu->cpum.GstCtx.r10 = pMsg->R10;
3186	pVCpu->cpum.GstCtx.r11 = pMsg->R11;
3187	pVCpu->cpum.GstCtx.r12 = pMsg->R12;
3188	pVCpu->cpum.GstCtx.r13 = pMsg->R13;
3189	pVCpu->cpum.GstCtx.r14 = pMsg->R14;
3190	pVCpu->cpum.GstCtx.r15 = pMsg->R15;
3191	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
3192	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ss, pMsg->SsSegment);
3193	}
3194	#elif defined(IN_RING3)
3195	/**
3196	* Copies state included in a exception intercept exit.
3197	*
3198	* @param pVCpu The cross context per CPU structure.
3199	* @param pExit The VM exit information.
3200	* @param fClearXcpt Clear pending exception.
3201	*/
3202	DECLINLINE(void) nemR3WinCopyStateFromExceptionMessage(PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit, bool fClearXcpt)
3203	{
3204	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3205	if (fClearXcpt)
3206	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
3207	}
3208	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3209
3210
3211	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3212	/**
3213	* Deals with exception intercept message (HvMessageTypeX64ExceptionIntercept).
3214	*
3215	* @returns Strict VBox status code.
3216	* @param pVCpu The cross context per CPU structure.
3217	* @param pMsg The message.
3218	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
3219	* @sa nemR3WinHandleExitMsr
3220	*/
3221	NEM_TMPL_STATIC VBOXSTRICTRC
3222	nemHCWinHandleMessageException(PVMCPU pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg, PGVMCPU pGVCpu)
3223	{
3224	/*
3225	* Assert sanity.
3226	*/
3227	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
3228	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
3229	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
3230	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
3231	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRip, pMsg->Header.Rip);
3232	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
3233	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
3234	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterDs, pMsg->DsSegment);
3235	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterSs, pMsg->SsSegment);
3236	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRax, pMsg->Rax);
3237	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRcx, pMsg->Rcx);
3238	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRdx, pMsg->Rdx);
3239	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRbx, pMsg->Rbx);
3240	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRsp, pMsg->Rsp);
3241	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRbp, pMsg->Rbp);
3242	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRsi, pMsg->Rsi);
3243	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRdi, pMsg->Rdi);
3244	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR8, pMsg->R8);
3245	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR9, pMsg->R9);
3246	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR10, pMsg->R10);
3247	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR11, pMsg->R11);
3248	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR12, pMsg->R12);
3249	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR13, pMsg->R13);
3250	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR14, pMsg->R14);
3251	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterR15, pMsg->R15);
3252
3253	/*
3254	* Get most of the register state since we'll end up making IEM inject the
3255	* event. The exception isn't normally flaged as a pending event, so duh.
3256	*
3257	* Note! We can optimize this later with event injection.
3258	*/
3259	Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n",
3260	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
3261	pMsg->ExceptionVector, pMsg->ErrorCode, pMsg->ExceptionParameter));
3262	nemHCWinCopyStateFromExceptionMessage(pVCpu, pMsg, true /fClearXcpt/);
3263	uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
3264	if (pMsg->ExceptionVector == X86_XCPT_DB)
3265	fWhat \|= CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_DR6;
3266	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, pGVCpu, fWhat, "Xcpt");
3267	if (rcStrict != VINF_SUCCESS)
3268	return rcStrict;
3269
3270	/*
3271	* Handle the intercept.
3272	*/
3273	TRPMEVENT enmEvtType = TRPM_TRAP;
3274	switch (pMsg->ExceptionVector)
3275	{
3276	/*
3277	* We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
3278	* and need to turn them over to GIM.
3279	*
3280	* Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
3281	* #UD for handling non-native hypercall instructions. (IEM will
3282	* decode both and let the GIM provider decide whether to accept it.)
3283	*/
3284	case X86_XCPT_UD:
3285	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
3286	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
3287	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3288
3289	if (nemHcWinIsInterestingUndefinedOpcode(pMsg->InstructionByteCount, pMsg->InstructionBytes,
3290	pMsg->Header.ExecutionState.EferLma && pMsg->Header.CsSegment.Long ))
3291	{
3292	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
3293	pMsg->InstructionBytes, pMsg->InstructionByteCount);
3294	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
3295	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
3296	nemHCWinExecStateToLogStr(&pMsg->Header), VBOXSTRICTRC_VAL(rcStrict) ));
3297	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
3298	return rcStrict;
3299	}
3300	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
3301	pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->InstructionByteCount, pMsg->InstructionBytes ));
3302	break;
3303
3304	/*
3305	* Filter debug exceptions.
3306	*/
3307	case X86_XCPT_DB:
3308	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
3309	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
3310	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3311	Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
3312	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header) ));
3313	break;
3314
3315	case X86_XCPT_BP:
3316	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
3317	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
3318	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3319	Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
3320	pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->Header.InstructionLength));
3321	enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
3322	break;
3323
3324	/* This shouldn't happen. */
3325	default:
3326	AssertLogRelMsgFailedReturn(("ExceptionVector=%#x\n", pMsg->ExceptionVector), VERR_IEM_IPE_6);
3327	}
3328
3329	/*
3330	* Inject it.
3331	*/
3332	rcStrict = IEMInjectTrap(pVCpu, pMsg->ExceptionVector, enmEvtType, pMsg->ErrorCode,
3333	pMsg->ExceptionParameter /??/, pMsg->Header.InstructionLength);
3334	Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
3335	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
3336	nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->ExceptionVector, VBOXSTRICTRC_VAL(rcStrict) ));
3337	return rcStrict;
3338	}
3339	#elif defined(IN_RING3)
3340	/**
3341	* Deals with MSR access exits (WHvRunVpExitReasonException).
3342	*
3343	* @returns Strict VBox status code.
3344	* @param pVM The cross context VM structure.
3345	* @param pVCpu The cross context per CPU structure.
3346	* @param pExit The VM exit information to handle.
3347	* @sa nemR3WinHandleExitException
3348	*/
3349	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitException(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3350	{
3351	/*
3352	* Get most of the register state since we'll end up making IEM inject the
3353	* event. The exception isn't normally flaged as a pending event, so duh.
3354	*
3355	* Note! We can optimize this later with event injection.
3356	*/
3357	Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3358	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType,
3359	pExit->VpException.ErrorCode, pExit->VpException.ExceptionParameter ));
3360	nemR3WinCopyStateFromExceptionMessage(pVCpu, pExit, true /fClearXcpt/);
3361	uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
3362	if (pExit->VpException.ExceptionType == X86_XCPT_DB)
3363	fWhat \|= CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_DR6;
3364	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NULL, fWhat, "Xcpt");
3365	if (rcStrict != VINF_SUCCESS)
3366	return rcStrict;
3367
3368	/*
3369	* Handle the intercept.
3370	*/
3371	TRPMEVENT enmEvtType = TRPM_TRAP;
3372	switch (pExit->VpException.ExceptionType)
3373	{
3374	/*
3375	* We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
3376	* and need to turn them over to GIM.
3377	*
3378	* Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
3379	* #UD for handling non-native hypercall instructions. (IEM will
3380	* decode both and let the GIM provider decide whether to accept it.)
3381	*/
3382	case X86_XCPT_UD:
3383	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
3384	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
3385	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3386	if (nemHcWinIsInterestingUndefinedOpcode(pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes,
3387	pExit->VpContext.ExecutionState.EferLma && pExit->VpContext.Cs.Long ))
3388	{
3389	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
3390	pExit->VpException.InstructionBytes,
3391	pExit->VpException.InstructionByteCount);
3392	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
3393	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
3394	nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
3395	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
3396	return rcStrict;
3397	}
3398
3399	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu,
3400	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3401	pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes ));
3402	break;
3403
3404	/*
3405	* Filter debug exceptions.
3406	*/
3407	case X86_XCPT_DB:
3408	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
3409	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
3410	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3411	Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
3412	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext) ));
3413	break;
3414
3415	case X86_XCPT_BP:
3416	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
3417	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
3418	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3419	Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3420	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.InstructionLength));
3421	enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
3422	break;
3423
3424	/* This shouldn't happen. */
3425	default:
3426	AssertLogRelMsgFailedReturn(("ExceptionType=%#x\n", pExit->VpException.ExceptionType), VERR_IEM_IPE_6);
3427	}
3428
3429	/*
3430	* Inject it.
3431	*/
3432	rcStrict = IEMInjectTrap(pVCpu, pExit->VpException.ExceptionType, enmEvtType, pExit->VpException.ErrorCode,
3433	pExit->VpException.ExceptionParameter /??/, pExit->VpContext.InstructionLength);
3434	Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
3435	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
3436	nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType, VBOXSTRICTRC_VAL(rcStrict) ));
3437
3438	RT_NOREF_PV(pVM);
3439	return rcStrict;
3440	}
3441	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3442
3443
3444	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3445	/**
3446	* Deals with unrecoverable exception (triple fault).
3447	*
3448	* Seen WRMSR 0x201 (IA32_MTRR_PHYSMASK0) writes from grub / debian9 ending up
3449	* here too. So we'll leave it to IEM to decide.
3450	*
3451	* @returns Strict VBox status code.
3452	* @param pVCpu The cross context per CPU structure.
3453	* @param pMsgHdr The message header.
3454	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
3455	* @sa nemR3WinHandleExitUnrecoverableException
3456	*/
3457	NEM_TMPL_STATIC VBOXSTRICTRC
3458	nemHCWinHandleMessageUnrecoverableException(PVMCPU pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr, PGVMCPU pGVCpu)
3459	{
3460	/* Check message register value sanity. */
3461	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, pGVCpu, HvX64RegisterCs, pMsgHdr->CsSegment);
3462	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRip, pMsgHdr->Rip);
3463	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterRflags, pMsgHdr->Rflags);
3464	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, pGVCpu, HvX64RegisterCr8, (uint64_t)pMsgHdr->Cr8);
3465
3466	# if 0
3467	/*
3468	* Just copy the state we've got and handle it in the loop for now.
3469	*/
3470	nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
3471	Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n",
3472	pVCpu->idCpu, pMsgHdr->CsSegment.Selector, pMsgHdr->Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsgHdr->Rflags));
3473	return VINF_EM_TRIPLE_FAULT;
3474	# else
3475	/*
3476	* Let IEM decide whether this is really it.
3477	*/
3478	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
3479	pMsgHdr->Rip + pMsgHdr->CsSegment.Base, ASMReadTSC());
3480	nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
3481	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, pGVCpu,
3482	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL, "TripleExit");
3483	if (rcStrict == VINF_SUCCESS)
3484	{
3485	rcStrict = IEMExecOne(pVCpu);
3486	if (rcStrict == VINF_SUCCESS)
3487	{
3488	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3489	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags ));
3490	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
3491	return VINF_SUCCESS;
3492	}
3493	if (rcStrict == VINF_EM_TRIPLE_FAULT)
3494	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3495	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3496	else
3497	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3498	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3499	}
3500	else
3501	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3502	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3503	return rcStrict;
3504	# endif
3505	}
3506	#elif defined(IN_RING3)
3507	/**
3508	* Deals with MSR access exits (WHvRunVpExitReasonUnrecoverableException).
3509	*
3510	* @returns Strict VBox status code.
3511	* @param pVM The cross context VM structure.
3512	* @param pVCpu The cross context per CPU structure.
3513	* @param pExit The VM exit information to handle.
3514	* @sa nemHCWinHandleMessageUnrecoverableException
3515	*/
3516	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitUnrecoverableException(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3517	{
3518	# if 0
3519	/*
3520	* Just copy the state we've got and handle it in the loop for now.
3521	*/
3522	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3523	Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3524	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
3525	RT_NOREF_PV(pVM);
3526	return VINF_EM_TRIPLE_FAULT;
3527	# else
3528	/*
3529	* Let IEM decide whether this is really it.
3530	*/
3531	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
3532	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3533	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3534	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NULL,
3535	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL, "TripleExit");
3536	if (rcStrict == VINF_SUCCESS)
3537	{
3538	rcStrict = IEMExecOne(pVCpu);
3539	if (rcStrict == VINF_SUCCESS)
3540	{
3541	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3542	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
3543	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
3544	return VINF_SUCCESS;
3545	}
3546	if (rcStrict == VINF_EM_TRIPLE_FAULT)
3547	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3548	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3549	else
3550	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3551	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3552	}
3553	else
3554	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3555	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3556	RT_NOREF_PV(pVM);
3557	return rcStrict;
3558	# endif
3559
3560	}
3561	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3562
3563
3564	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3565	/**
3566	* Handles messages (VM exits).
3567	*
3568	* @returns Strict VBox status code.
3569	* @param pVM The cross context VM structure.
3570	* @param pVCpu The cross context per CPU structure.
3571	* @param pMappingHeader The message slot mapping.
3572	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
3573	* @sa nemR3WinHandleExit
3574	*/
3575	NEM_TMPL_STATIC VBOXSTRICTRC
3576	nemHCWinHandleMessage(PVM pVM, PVMCPU pVCpu, VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader, PGVMCPU pGVCpu)
3577	{
3578	if (pMappingHeader->enmVidMsgType == VidMessageHypervisorMessage)
3579	{
3580	AssertMsg(pMappingHeader->cbMessage == HV_MESSAGE_SIZE, ("%#x\n", pMappingHeader->cbMessage));
3581	HV_MESSAGE const pMsg = (HV_MESSAGE const )(pMappingHeader + 1);
3582	switch (pMsg->Header.MessageType)
3583	{
3584	case HvMessageTypeUnmappedGpa:
3585	Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
3586	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
3587	return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pGVCpu);
3588
3589	case HvMessageTypeGpaIntercept:
3590	Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
3591	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemIntercept);
3592	return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pGVCpu);
3593
3594	case HvMessageTypeX64IoPortIntercept:
3595	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64IoPortIntercept));
3596	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
3597	return nemHCWinHandleMessageIoPort(pVM, pVCpu, &pMsg->X64IoPortIntercept, pGVCpu);
3598
3599	case HvMessageTypeX64Halt:
3600	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
3601	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
3602	pMsg->X64InterceptHeader.Rip + pMsg->X64InterceptHeader.CsSegment.Base, ASMReadTSC());
3603	Log4(("HaltExit\n"));
3604	return VINF_EM_HALT;
3605
3606	case HvMessageTypeX64InterruptWindow:
3607	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterruptWindow));
3608	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
3609	return nemHCWinHandleMessageInterruptWindow(pVM, pVCpu, &pMsg->X64InterruptWindow, pGVCpu);
3610
3611	case HvMessageTypeX64CpuidIntercept:
3612	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64CpuIdIntercept));
3613	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
3614	return nemHCWinHandleMessageCpuId(pVM, pVCpu, &pMsg->X64CpuIdIntercept, pGVCpu);
3615
3616	case HvMessageTypeX64MsrIntercept:
3617	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64MsrIntercept));
3618	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
3619	return nemHCWinHandleMessageMsr(pVCpu, &pMsg->X64MsrIntercept, pGVCpu);
3620
3621	case HvMessageTypeX64ExceptionIntercept:
3622	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64ExceptionIntercept));
3623	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
3624	return nemHCWinHandleMessageException(pVCpu, &pMsg->X64ExceptionIntercept, pGVCpu);
3625
3626	case HvMessageTypeUnrecoverableException:
3627	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterceptHeader));
3628	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
3629	return nemHCWinHandleMessageUnrecoverableException(pVCpu, &pMsg->X64InterceptHeader, pGVCpu);
3630
3631	case HvMessageTypeInvalidVpRegisterValue:
3632	case HvMessageTypeUnsupportedFeature:
3633	case HvMessageTypeTlbPageSizeMismatch:
3634	LogRel(("Unimplemented msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3635	AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n%.32Rhxd\n", pMsg->Header.MessageType, pMsg),
3636	VERR_NEM_IPE_3);
3637
3638	case HvMessageTypeX64ApicEoi:
3639	case HvMessageTypeX64LegacyFpError:
3640	case HvMessageTypeX64RegisterIntercept:
3641	case HvMessageTypeApicEoi:
3642	case HvMessageTypeFerrAsserted:
3643	case HvMessageTypeEventLogBufferComplete:
3644	case HvMessageTimerExpired:
3645	LogRel(("Unexpected msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3646	AssertLogRelMsgFailedReturn(("Unexpected message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
3647	VERR_NEM_IPE_3);
3648
3649	default:
3650	LogRel(("Unknown msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3651	AssertLogRelMsgFailedReturn(("Unknown message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
3652	VERR_NEM_IPE_3);
3653	}
3654	}
3655	else
3656	AssertLogRelMsgFailedReturn(("Unexpected VID message type on CPU #%u: %#x LB %u\n",
3657	pVCpu->idCpu, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage),
3658	VERR_NEM_IPE_4);
3659	}
3660	#elif defined(IN_RING3)
3661	/**
3662	* Handles VM exits.
3663	*
3664	* @returns Strict VBox status code.
3665	* @param pVM The cross context VM structure.
3666	* @param pVCpu The cross context per CPU structure.
3667	* @param pExit The VM exit information to handle.
3668	* @sa nemHCWinHandleMessage
3669	*/
3670	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExit(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3671	{
3672	switch (pExit->ExitReason)
3673	{
3674	case WHvRunVpExitReasonMemoryAccess:
3675	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
3676	return nemR3WinHandleExitMemory(pVM, pVCpu, pExit);
3677
3678	case WHvRunVpExitReasonX64IoPortAccess:
3679	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
3680	return nemR3WinHandleExitIoPort(pVM, pVCpu, pExit);
3681
3682	case WHvRunVpExitReasonX64Halt:
3683	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
3684	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
3685	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3686	Log4(("HaltExit\n"));
3687	return VINF_EM_HALT;
3688
3689	case WHvRunVpExitReasonCanceled:
3690	return VINF_SUCCESS;
3691
3692	case WHvRunVpExitReasonX64InterruptWindow:
3693	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
3694	return nemR3WinHandleExitInterruptWindow(pVM, pVCpu, pExit);
3695
3696	case WHvRunVpExitReasonX64Cpuid:
3697	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
3698	return nemR3WinHandleExitCpuId(pVM, pVCpu, pExit);
3699
3700	case WHvRunVpExitReasonX64MsrAccess:
3701	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
3702	return nemR3WinHandleExitMsr(pVM, pVCpu, pExit);
3703
3704	case WHvRunVpExitReasonException:
3705	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
3706	return nemR3WinHandleExitException(pVM, pVCpu, pExit);
3707
3708	case WHvRunVpExitReasonUnrecoverableException:
3709	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
3710	return nemR3WinHandleExitUnrecoverableException(pVM, pVCpu, pExit);
3711
3712	case WHvRunVpExitReasonUnsupportedFeature:
3713	case WHvRunVpExitReasonInvalidVpRegisterValue:
3714	LogRel(("Unimplemented exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
3715	AssertLogRelMsgFailedReturn(("Unexpected exit on CPU #%u: %#x\n%.32Rhxd\n",
3716	pVCpu->idCpu, pExit->ExitReason, pExit), VERR_NEM_IPE_3);
3717
3718	/* Undesired exits: */
3719	case WHvRunVpExitReasonNone:
3720	default:
3721	LogRel(("Unknown exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
3722	AssertLogRelMsgFailedReturn(("Unknown exit on CPU #%u: %#x!\n", pVCpu->idCpu, pExit->ExitReason), VERR_NEM_IPE_3);
3723	}
3724	}
3725	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3726
3727
3728	#ifdef IN_RING0
3729	/**
3730	* Perform an I/O control operation on the partition handle (VID.SYS),
3731	* restarting on alert-like behaviour.
3732	*
3733	* @returns NT status code.
3734	* @param pGVM The ring-0 VM structure.
3735	* @param pGVCpu The ring-0 CPU structure.
3736	* @param pVCpu The calling cross context CPU structure.
3737	* @param fFlags The wait flags.
3738	* @param cMillies The timeout in milliseconds
3739	*/
3740	static NTSTATUS nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(PGVM pGVM, PGVMCPU pGVCpu, PVMCPU pVCpu,
3741	uint32_t fFlags, uint32_t cMillies)
3742	{
3743	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
3744	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = fFlags;
3745	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
3746	NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
3747	&pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
3748	pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
3749	NULL, 0);
3750	if (rcNt == STATUS_SUCCESS)
3751	{ /* likely */ }
3752	/*
3753	* Generally, if we get down here, we have been interrupted between ACK'ing
3754	* a message and waiting for the next due to a NtAlertThread call. So, we
3755	* should stop ACK'ing the previous message and get on waiting on the next.
3756	* See similar stuff in nemHCWinRunGC().
3757	*/
3758	else if ( rcNt == STATUS_TIMEOUT
3759	\|\| rcNt == STATUS_ALERTED /* just in case */
3760	\|\| rcNt == STATUS_KERNEL_APC /* just in case */
3761	\|\| rcNt == STATUS_USER_APC /* just in case */)
3762	{
3763	DBGFTRACE_CUSTOM(pVCpu->CTX_SUFF(pVM), "IoCtlMessageSlotHandleAndGetNextRestart/1 %#x (f=%#x)", rcNt, fFlags);
3764	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPendingAlerts);
3765	Assert(fFlags & VID_MSHAGN_F_GET_NEXT_MESSAGE);
3766
3767	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pVCpu->idCpu;
3768	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = fFlags & ~VID_MSHAGN_F_HANDLE_MESSAGE;
3769	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
3770	rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
3771	&pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
3772	pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
3773	NULL, 0);
3774	DBGFTRACE_CUSTOM(pVCpu->CTX_SUFF(pVM), "IoCtlMessageSlotHandleAndGetNextRestart/2 %#x", rcNt);
3775	}
3776	return rcNt;
3777	}
3778
3779	#endif /* IN_RING0 */
3780
3781
3782	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3783	/**
3784	* Worker for nemHCWinRunGC that stops the execution on the way out.
3785	*
3786	* The CPU was running the last time we checked, no there are no messages that
3787	* needs being marked handled/whatever. Caller checks this.
3788	*
3789	* @returns rcStrict on success, error status on failure.
3790	* @param pVM The cross context VM structure.
3791	* @param pVCpu The cross context per CPU structure.
3792	* @param rcStrict The nemHCWinRunGC return status. This is a little
3793	* bit unnecessary, except in internal error cases,
3794	* since we won't need to stop the CPU if we took an
3795	* exit.
3796	* @param pMappingHeader The message slot mapping.
3797	* @param pGVM The global (ring-0) VM structure (NULL in r3).
3798	* @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
3799	*/
3800	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinStopCpu(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict,
3801	VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader,
3802	PGVM pGVM, PGVMCPU pGVCpu)
3803	{
3804	# ifdef DBGFTRACE_ENABLED
3805	HV_MESSAGE const volatile pMsgForTrace = (HV_MESSAGE const volatile )(pMappingHeader + 1);
3806	# endif
3807
3808	/*
3809	* Try stopping the processor. If we're lucky we manage to do this before it
3810	* does another VM exit.
3811	*/
3812	DBGFTRACE_CUSTOM(pVM, "nemStop#0");
3813	# ifdef IN_RING0
3814	pVCpu->nem.s.uIoCtlBuf.idCpu = pGVCpu->idCpu;
3815	NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlStopVirtualProcessor.uFunction,
3816	&pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
3817	NULL, 0);
3818	if (NT_SUCCESS(rcNt))
3819	{
3820	DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay (%#x)", rcNt);
3821	Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
3822	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
3823	return rcStrict;
3824	}
3825	# else
3826	BOOL fRet = VidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu);
3827	if (fRet)
3828	{
3829	DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay");
3830	Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
3831	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
3832	return rcStrict;
3833	}
3834	RT_NOREF(pGVM, pGVCpu);
3835	# endif
3836
3837	/*
3838	* Dang. The CPU stopped by itself and we got a couple of message to deal with.
3839	*/
3840	# ifdef IN_RING0
3841	DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", rcNt);
3842	AssertLogRelMsgReturn(rcNt == ERROR_VID_STOP_PENDING, ("rcNt=%#x\n", rcNt),
3843	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3844	# else
3845	DWORD dwErr = RTNtLastErrorValue();
3846	DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", dwErr);
3847	AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u (%#x)\n", dwErr, dwErr),
3848	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3849	# endif
3850	Log8(("nemHCWinStopCpu: Stopping CPU #%u pending...\n", pVCpu->idCpu));
3851	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPending);
3852
3853	/*
3854	* First message: Exit or similar, sometimes VidMessageStopRequestComplete.
3855	* Note! We can safely ASSUME that rcStrict isn't an important information one.
3856	*/
3857	# ifdef IN_RING0
3858	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pGVM, pGVCpu, pVCpu, VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3859	DBGFTRACE_CUSTOM(pVM, "nemStop#1: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
3860	pMsgForTrace->Header.MessageType);
3861	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
3862	("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
3863	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3864	# else
3865	BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
3866	VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3867	DBGFTRACE_CUSTOM(pVM, "nemStop#1: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
3868	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3869	AssertLogRelMsgReturn(fWait, ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
3870	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3871	# endif
3872
3873	VID_MESSAGE_TYPE enmVidMsgType = pMappingHeader->enmVidMsgType;
3874	if (enmVidMsgType != VidMessageStopRequestComplete)
3875	{
3876	VBOXSTRICTRC rcStrict2 = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, pGVCpu);
3877	if (rcStrict2 != VINF_SUCCESS && RT_SUCCESS(rcStrict))
3878	rcStrict = rcStrict2;
3879	DBGFTRACE_CUSTOM(pVM, "nemStop#1: handled %#x -> %d", pMsgForTrace->Header.MessageType, VBOXSTRICTRC_VAL(rcStrict));
3880
3881	/*
3882	* Mark it as handled and get the stop request completed message, then mark
3883	* that as handled too. CPU is back into fully stopped stated then.
3884	*/
3885	# ifdef IN_RING0
3886	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pGVM, pGVCpu, pVCpu,
3887	VID_MSHAGN_F_HANDLE_MESSAGE \| VID_MSHAGN_F_GET_NEXT_MESSAGE,
3888	30000 /ms/);
3889	DBGFTRACE_CUSTOM(pVM, "nemStop#2: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
3890	pMsgForTrace->Header.MessageType);
3891	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
3892	("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
3893	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3894	# else
3895	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
3896	VID_MSHAGN_F_HANDLE_MESSAGE \| VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3897	DBGFTRACE_CUSTOM(pVM, "nemStop#2: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
3898	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3899	AssertLogRelMsgReturn(fWait, ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
3900	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3901	# endif
3902
3903	/* It should be a stop request completed message. */
3904	enmVidMsgType = pMappingHeader->enmVidMsgType;
3905	AssertLogRelMsgReturn(enmVidMsgType == VidMessageStopRequestComplete,
3906	("Unexpected 2nd message following ERROR_VID_STOP_PENDING: %#x LB %#x\n",
3907	enmVidMsgType, pMappingHeader->cbMessage),
3908	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3909
3910	/*
3911	* Mark the VidMessageStopRequestComplete message as handled.
3912	*/
3913	# ifdef IN_RING0
3914	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pGVM, pGVCpu, pVCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /ms/);
3915	DBGFTRACE_CUSTOM(pVM, "nemStop#3: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType,
3916	pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3917	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
3918	("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
3919	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3920	# else
3921	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /ms/);
3922	DBGFTRACE_CUSTOM(pVM, "nemStop#3: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
3923	pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3924	AssertLogRelMsgReturn(fWait, ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
3925	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3926	# endif
3927	Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict) ));
3928	}
3929	else
3930	{
3931	/** @todo I'm not so sure about this now... */
3932	DBGFTRACE_CUSTOM(pVM, "nemStop#9: %#x %#x %#x", pMappingHeader->enmVidMsgType,
3933	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3934	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPendingOdd);
3935	Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc) - 1st VidMessageSlotHandleAndGetNext got VidMessageStopRequestComplete.\n",
3936	VBOXSTRICTRC_VAL(rcStrict) ));
3937	}
3938	return rcStrict;
3939	}
3940	#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3941
3942	#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3)
3943
3944	/**
3945	* Deals with pending interrupt related force flags, may inject interrupt.
3946	*
3947	* @returns VBox strict status code.
3948	* @param pVM The cross context VM structure.
3949	* @param pVCpu The cross context per CPU structure.
3950	* @param pGVCpu The global (ring-0) per CPU structure.
3951	* @param pfInterruptWindows Where to return interrupt window flags.
3952	*/
3953	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleInterruptFF(PVM pVM, PVMCPU pVCpu, PGVMCPU pGVCpu, uint8_t *pfInterruptWindows)
3954	{
3955	Assert(!TRPMHasTrap(pVCpu));
3956	RT_NOREF_PV(pVM);
3957
3958	/*
3959	* First update APIC. We ASSUME this won't need TPR/CR8.
3960	*/
3961	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3962	{
3963	APICUpdatePendingInterrupts(pVCpu);
3964	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
3965	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
3966	return VINF_SUCCESS;
3967	}
3968
3969	/*
3970	* We don't currently implement SMIs.
3971	*/
3972	AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
3973
3974	/*
3975	* Check if we've got the minimum of state required for deciding whether we
3976	* can inject interrupts and NMIs. If we don't have it, get all we might require
3977	* for injection via IEM.
3978	*/
3979	bool const fPendingNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3980	uint64_t fNeedExtrn = CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
3981	\| (fPendingNmi ? CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI : 0);
3982	if (pVCpu->cpum.GstCtx.fExtrn & fNeedExtrn)
3983	{
3984	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, pGVCpu,
3985	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "IntFF");
3986	if (rcStrict != VINF_SUCCESS)
3987	return rcStrict;
3988	}
3989	bool const fInhibitInterrupts = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
3990	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip;
3991
3992	/*
3993	* NMI? Try deliver it first.
3994	*/
3995	if (fPendingNmi)
3996	{
3997	if ( !fInhibitInterrupts
3998	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3999	{
4000	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, pGVCpu,
4001	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
4002	if (rcStrict == VINF_SUCCESS)
4003	{
4004	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
4005	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_NMI, TRPM_HARDWARE_INT, 0, 0, 0);
4006	Log8(("Injected NMI on %u (%d)\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4007	}
4008	return rcStrict;
4009	}
4010	*pfInterruptWindows \|= NEM_WIN_INTW_F_NMI;
4011	Log8(("NMI window pending on %u\n", pVCpu->idCpu));
4012	}
4013
4014	/*
4015	* APIC or PIC interrupt?
4016	*/
4017	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
4018	{
4019	if ( !fInhibitInterrupts
4020	&& pVCpu->cpum.GstCtx.rflags.Bits.u1IF)
4021	{
4022	AssertCompile(NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT & CPUMCTX_EXTRN_APIC_TPR);
4023	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, pGVCpu,
4024	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
4025	if (rcStrict == VINF_SUCCESS)
4026	{
4027	uint8_t bInterrupt;
4028	int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
4029	if (RT_SUCCESS(rc))
4030	{
4031	rcStrict = IEMInjectTrap(pVCpu, bInterrupt, TRPM_HARDWARE_INT, 0, 0, 0);
4032	Log8(("Injected interrupt %#x on %u (%d)\n", bInterrupt, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4033	}
4034	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
4035	{
4036	pfInterruptWindows \|= (bInterrupt >> 4 /??*/) << NEM_WIN_INTW_F_PRIO_SHIFT;
4037	Log8(("VERR_APIC_INTR_MASKED_BY_TPR: pfInterruptWindows=%#x\n", pfInterruptWindows));
4038	}
4039	else
4040	Log8(("PDMGetInterrupt failed -> %d\n", rc));
4041	}
4042	return rcStrict;
4043	}
4044	*pfInterruptWindows \|= NEM_WIN_INTW_F_REGULAR;
4045	Log8(("Interrupt window pending on %u\n", pVCpu->idCpu));
4046	}
4047
4048	return VINF_SUCCESS;
4049	}
4050
4051
4052	/**
4053	* Inner NEM runloop for windows.
4054	*
4055	* @returns Strict VBox status code.
4056	* @param pVM The cross context VM structure.
4057	* @param pVCpu The cross context per CPU structure.
4058	* @param pGVM The ring-0 VM structure (NULL in ring-3).
4059	* @param pGVCpu The ring-0 per CPU structure (NULL in ring-3).
4060	*/
4061	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVM pVM, PVMCPU pVCpu, PGVM pGVM, PGVMCPU pGVCpu)
4062	{
4063	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags));
4064	# ifdef LOG_ENABLED
4065	if (LogIs3Enabled())
4066	nemHCWinLogState(pVM, pVCpu);
4067	# endif
4068	# ifdef IN_RING0
4069	Assert(pVCpu->idCpu == pGVCpu->idCpu);
4070	# endif
4071
4072	/*
4073	* Try switch to NEM runloop state.
4074	*/
4075	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
4076	{ /* likely */ }
4077	else
4078	{
4079	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4080	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
4081	return VINF_SUCCESS;
4082	}
4083
4084	/*
4085	* The run loop.
4086	*
4087	* Current approach to state updating to use the sledgehammer and sync
4088	* everything every time. This will be optimized later.
4089	*/
4090	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4091	VID_MESSAGE_MAPPING_HEADER volatile pMappingHeader = (VID_MESSAGE_MAPPING_HEADER volatile )pVCpu->nem.s.pvMsgSlotMapping;
4092	# endif
4093	const bool fSingleStepping = DBGFIsStepping(pVCpu);
4094	// const uint32_t fCheckVmFFs = !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK
4095	// : VM_FF_HP_R0_PRE_HM_STEP_MASK;
4096	// const uint32_t fCheckCpuFFs = !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK;
4097	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4098	for (unsigned iLoop = 0;; iLoop++)
4099	{
4100	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4101	/*
4102	* Hack alert!
4103	*/
4104	uint32_t const cMappedPages = pVM->nem.s.cMappedPages;
4105	if (cMappedPages >= 4000)
4106	{
4107	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinWHvUnmapOnePageCallback, NULL);
4108	Log(("nemHCWinRunGC: Unmapped all; cMappedPages=%u -> %u\n", cMappedPages, pVM->nem.s.cMappedPages));
4109	}
4110	# endif
4111
4112	/*
4113	* Pending interrupts or such? Need to check and deal with this prior
4114	* to the state syncing.
4115	*/
4116	pVCpu->nem.s.fDesiredInterruptWindows = 0;
4117	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_UPDATE_APIC \| VMCPU_FF_INTERRUPT_PIC
4118	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4119	{
4120	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4121	/* Make sure the CPU isn't executing. */
4122	if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
4123	{
4124	pVCpu->nem.s.fHandleAndGetFlags = 0;
4125	rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader, pGVM, pGVCpu);
4126	if (rcStrict == VINF_SUCCESS)
4127	{ /* likely */ }
4128	else
4129	{
4130	LogFlow(("NEM/%u: breaking: nemHCWinStopCpu -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4131	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4132	break;
4133	}
4134	}
4135	# endif
4136
4137	/* Try inject interrupt. */
4138	rcStrict = nemHCWinHandleInterruptFF(pVM, pVCpu, pGVCpu, &pVCpu->nem.s.fDesiredInterruptWindows);
4139	if (rcStrict == VINF_SUCCESS)
4140	{ /* likely */ }
4141	else
4142	{
4143	LogFlow(("NEM/%u: breaking: nemHCWinHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4144	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4145	break;
4146	}
4147	}
4148
4149	/*
4150	* Ensure that hyper-V has the whole state.
4151	* (We always update the interrupt windows settings when active as hyper-V seems
4152	* to forget about it after an exit.)
4153	*/
4154	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK))
4155	!= (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK)
4156	\|\| ( ( pVCpu->nem.s.fDesiredInterruptWindows
4157	\|\| pVCpu->nem.s.fCurrentInterruptWindows != pVCpu->nem.s.fDesiredInterruptWindows)
4158	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4159	&& pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */
4160	# endif
4161	)
4162	)
4163	{
4164	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4165	AssertMsg(pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */,
4166	("%#x fExtrn=%#RX64 (%#RX64) fDesiredInterruptWindows=%d fCurrentInterruptWindows=%#x vs %#x\n",
4167	pVCpu->nem.s.fHandleAndGetFlags, pVCpu->cpum.GstCtx.fExtrn, ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK),
4168	pVCpu->nem.s.fDesiredInterruptWindows, pVCpu->nem.s.fCurrentInterruptWindows, pVCpu->nem.s.fDesiredInterruptWindows));
4169	# endif
4170	# ifdef IN_RING0
4171	int rc2 = nemR0WinExportState(pGVM, pGVCpu, &pVCpu->cpum.GstCtx);
4172	# else
4173	int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu);
4174	RT_NOREF(pGVM, pGVCpu);
4175	# endif
4176	AssertRCReturn(rc2, rc2);
4177	}
4178
4179	/*
4180	* Poll timers and run for a bit.
4181	*
4182	* With the VID approach (ring-0 or ring-3) we can specify a timeout here,
4183	* so we take the time of the next timer event and uses that as a deadline.
4184	* The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
4185	*/
4186	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
4187	* the whole polling job when timers have changed... */
4188	uint64_t offDeltaIgnored;
4189	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
4190	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4191	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4192	{
4193	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4194	if (pVCpu->nem.s.fHandleAndGetFlags)
4195	{ /* Very likely that the CPU does NOT need starting (pending msg, running). */ }
4196	else
4197	{
4198	# ifdef IN_RING0
4199	pVCpu->nem.s.uIoCtlBuf.idCpu = pGVCpu->idCpu;
4200	NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlStartVirtualProcessor.uFunction,
4201	&pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
4202	NULL, 0);
4203	LogFlow(("NEM/%u: IoCtlStartVirtualProcessor -> %#x\n", pVCpu->idCpu, rcNt));
4204	AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("VidStartVirtualProcessor failed for CPU #%u: %#x\n", pGVCpu->idCpu, rcNt),
4205	VERR_NEM_IPE_5);
4206	# else
4207	AssertLogRelMsgReturn(g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
4208	("VidStartVirtualProcessor failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
4209	pVCpu->idCpu, RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()),
4210	VERR_NEM_IPE_5);
4211	# endif
4212	pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
4213	}
4214	# endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
4215
4216	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
4217	{
4218	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4219	uint64_t const nsNow = RTTimeNanoTS();
4220	int64_t const cNsNextTimerEvt = nsNow - nsNextTimerEvt;
4221	uint32_t cMsWait;
4222	if (cNsNextTimerEvt < 100000 /* ns */)
4223	cMsWait = 0;
4224	else if ((uint64_t)cNsNextTimerEvt < RT_NS_1SEC)
4225	{
4226	if ((uint32_t)cNsNextTimerEvt < 2*RT_NS_1MS)
4227	cMsWait = 1;
4228	else
4229	cMsWait = ((uint32_t)cNsNextTimerEvt - 100000 /ns/) / RT_NS_1MS;
4230	}
4231	else
4232	cMsWait = RT_MS_1SEC;
4233	# ifdef IN_RING0
4234	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
4235	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = pVCpu->nem.s.fHandleAndGetFlags;
4236	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMsWait;
4237	NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
4238	&pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
4239	pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
4240	NULL, 0);
4241	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4242	if (rcNt == STATUS_SUCCESS)
4243	# else
4244	BOOL fRet = VidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
4245	pVCpu->nem.s.fHandleAndGetFlags, cMsWait);
4246	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4247	if (fRet)
4248	# endif
4249	# else
4250	WHV_RUN_VP_EXIT_CONTEXT ExitReason;
4251	RT_ZERO(ExitReason);
4252	HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
4253	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4254	if (SUCCEEDED(hrc))
4255	# endif
4256	{
4257	/*
4258	* Deal with the message.
4259	*/
4260	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4261	rcStrict = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, pGVCpu);
4262	pVCpu->nem.s.fHandleAndGetFlags \|= VID_MSHAGN_F_HANDLE_MESSAGE;
4263	# else
4264	rcStrict = nemR3WinHandleExit(pVM, pVCpu, &ExitReason);
4265	# endif
4266	if (rcStrict == VINF_SUCCESS)
4267	{ /* hopefully likely */ }
4268	else
4269	{
4270	LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4271	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4272	break;
4273	}
4274	}
4275	else
4276	{
4277	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4278
4279	/* VID.SYS merges STATUS_ALERTED and STATUS_USER_APC into STATUS_TIMEOUT,
4280	so after NtAlertThread we end up here with a STATUS_TIMEOUT. And yeah,
4281	the error code conversion is into WAIT_XXX, i.e. NT status codes. */
4282	# ifndef IN_RING0
4283	DWORD rcNt = GetLastError();
4284	# endif
4285	LogFlow(("NEM/%u: VidMessageSlotHandleAndGetNext -> %#x\n", pVCpu->idCpu, rcNt));
4286	AssertLogRelMsgReturn( rcNt == STATUS_TIMEOUT
4287	\|\| rcNt == STATUS_ALERTED /* just in case */
4288	\|\| rcNt == STATUS_USER_APC /* ditto */
4289	\|\| rcNt == STATUS_KERNEL_APC /* ditto */
4290	, ("VidMessageSlotHandleAndGetNext failed for CPU #%u: %#x (%u)\n",
4291	pVCpu->idCpu, rcNt, rcNt),
4292	VERR_NEM_IPE_0);
4293	pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
4294	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatGetMsgTimeout);
4295	# else
4296	AssertLogRelMsgFailedReturn(("WHvRunVirtualProcessor failed for CPU #%u: %#x (%u)\n",
4297	pVCpu->idCpu, hrc, GetLastError()),
4298	VERR_NEM_IPE_0);
4299	# endif
4300	}
4301
4302	/*
4303	* If no relevant FFs are pending, loop.
4304	*/
4305	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
4306	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4307	continue;
4308
4309	/** @todo Try handle pending flags, not just return to EM loops. Take care
4310	* not to set important RCs here unless we've handled a message. */
4311	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
4312	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
4313	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
4314	}
4315	else
4316	{
4317	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
4318	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
4319	}
4320	}
4321	else
4322	{
4323	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
4324	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
4325	}
4326	break;
4327	} /* the run loop */
4328
4329
4330	/*
4331	* If the CPU is running, make sure to stop it before we try sync back the
4332	* state and return to EM. We don't sync back the whole state if we can help it.
4333	*/
4334	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4335	if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
4336	{
4337	pVCpu->nem.s.fHandleAndGetFlags = 0;
4338	rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader, pGVM, pGVCpu);
4339	}
4340	# endif
4341
4342	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
4343	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4344
4345	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
4346	{
4347	/* Try anticipate what we might need. */
4348	uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI;
4349	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
4350	\|\| RT_FAILURE(rcStrict))
4351	fImport = CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
4352	# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
4353	else if ( rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
4354	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
4355	fImport = CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT;
4356	else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
4357	fImport = CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT;
4358	# endif
4359	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
4360	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4361	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
4362
4363	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
4364	{
4365	# ifdef IN_RING0
4366	int rc2 = nemR0WinImportState(pGVM, pGVCpu, &pVCpu->cpum.GstCtx, fImport \| CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT,
4367	true /fCanUpdateCr3/);
4368	if (RT_SUCCESS(rc2))
4369	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4370	else if (rc2 == VERR_NEM_FLUSH_TLB)
4371	{
4372	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4373	if (rcStrict == VINF_SUCCESS \|\| rcStrict == -rc2)
4374	rcStrict = -rc2;
4375	else
4376	{
4377	pVCpu->nem.s.rcPending = -rc2;
4378	LogFlow(("NEM/%u: rcPending=%Rrc (rcStrict=%Rrc)\n", pVCpu->idCpu, rc2, VBOXSTRICTRC_VAL(rcStrict) ));
4379	}
4380	}
4381	# else
4382	int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, fImport \| CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
4383	if (RT_SUCCESS(rc2))
4384	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4385	# endif
4386	else if (RT_SUCCESS(rcStrict))
4387	rcStrict = rc2;
4388	if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
4389	pVCpu->cpum.GstCtx.fExtrn = 0;
4390	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
4391	}
4392	else
4393	{
4394	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4395	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
4396	}
4397	}
4398	else
4399	{
4400	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4401	pVCpu->cpum.GstCtx.fExtrn = 0;
4402	}
4403
4404	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
4405	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
4406	return rcStrict;
4407	}
4408
4409	#endif /* defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3) */
4410
4411	/**
4412	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
4413	*/
4414	NEM_TMPL_STATIC DECLCALLBACK(int) nemHCWinUnsetForA20CheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys,
4415	PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
4416	{
4417	/* We'll just unmap the memory. */
4418	if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
4419	{
4420	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4421	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
4422	AssertRC(rc);
4423	if (RT_SUCCESS(rc))
4424	#else
4425	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
4426	if (SUCCEEDED(hrc))
4427	#endif
4428	{
4429	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4430	Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
4431	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
4432	}
4433	else
4434	{
4435	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4436	LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
4437	return rc;
4438	#else
4439	LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4440	GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4441	return VERR_NEM_IPE_2;
4442	#endif
4443	}
4444	}
4445	RT_NOREF(pVCpu, pvUser);
4446	return VINF_SUCCESS;
4447	}
4448
4449
4450	/**
4451	* Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
4452	*
4453	* @returns The PGMPhysNemQueryPageInfo result.
4454	* @param pVM The cross context VM structure.
4455	* @param pVCpu The cross context virtual CPU structure.
4456	* @param GCPhys The page to unmap.
4457	*/
4458	NEM_TMPL_STATIC int nemHCWinUnmapPageForA20Gate(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4459	{
4460	PGMPHYSNEMPAGEINFO Info;
4461	return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /fMakeWritable/, &Info,
4462	nemHCWinUnsetForA20CheckerCallback, NULL);
4463	}
4464
4465
4466	void nemHCNativeNotifyHandlerPhysicalRegister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
4467	{
4468	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
4469	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
4470	}
4471
4472
4473	void nemHCNativeNotifyHandlerPhysicalDeregister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
4474	int fRestoreAsRAM, bool fRestoreAsRAM2)
4475	{
4476	Log5(("nemHCNativeNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d fRestoreAsRAM=%d fRestoreAsRAM2=%d\n",
4477	GCPhys, cb, enmKind, fRestoreAsRAM, fRestoreAsRAM2));
4478	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb); NOREF(fRestoreAsRAM); NOREF(fRestoreAsRAM2);
4479	}
4480
4481
4482	void nemHCNativeNotifyHandlerPhysicalModify(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
4483	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
4484	{
4485	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
4486	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
4487	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
4488	}
4489
4490
4491	/**
4492	* Worker that maps pages into Hyper-V.
4493	*
4494	* This is used by the PGM physical page notifications as well as the memory
4495	* access VMEXIT handlers.
4496	*
4497	* @returns VBox status code.
4498	* @param pVM The cross context VM structure.
4499	* @param pVCpu The cross context virtual CPU structure of the
4500	* calling EMT.
4501	* @param GCPhysSrc The source page address.
4502	* @param GCPhysDst The hyper-V destination page. This may differ from
4503	* GCPhysSrc when A20 is disabled.
4504	* @param fPageProt NEM_PAGE_PROT_XXX.
4505	* @param pu2State Our page state (input/output).
4506	* @param fBackingChanged Set if the page backing is being changed.
4507	* @thread EMT(pVCpu)
4508	*/
4509	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
4510	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
4511	{
4512	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4513	/*
4514	* When using the hypercalls instead of the ring-3 APIs, we don't need to
4515	* unmap memory before modifying it. We still want to track the state though,
4516	* since unmap will fail when called an unmapped page and we don't want to redo
4517	* upgrades/downgrades.
4518	*/
4519	uint8_t const u2OldState = *pu2State;
4520	int rc;
4521	if (fPageProt == NEM_PAGE_PROT_NONE)
4522	{
4523	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
4524	{
4525	rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4526	if (RT_SUCCESS(rc))
4527	{
4528	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4529	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4530	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4531	}
4532	else
4533	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4534	}
4535	else
4536	rc = VINF_SUCCESS;
4537	}
4538	else if (fPageProt & NEM_PAGE_PROT_WRITE)
4539	{
4540	if (u2OldState != NEM_WIN_PAGE_STATE_WRITABLE \|\| fBackingChanged)
4541	{
4542	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4543	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
4544	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4545	if (RT_SUCCESS(rc))
4546	{
4547	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4548	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
4549	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
4550	Log5(("NEM GPA writable/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4551	NOREF(cMappedPages);
4552	}
4553	else
4554	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4555	}
4556	else
4557	rc = VINF_SUCCESS;
4558	}
4559	else
4560	{
4561	if (u2OldState != NEM_WIN_PAGE_STATE_READABLE \|\| fBackingChanged)
4562	{
4563	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4564	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4565	if (RT_SUCCESS(rc))
4566	{
4567	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4568	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
4569	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
4570	Log5(("NEM GPA read+exec/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4571	NOREF(cMappedPages);
4572	}
4573	else
4574	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4575	}
4576	else
4577	rc = VINF_SUCCESS;
4578	}
4579
4580	return VINF_SUCCESS;
4581
4582	#else
4583	/*
4584	* Looks like we need to unmap a page before we can change the backing
4585	* or even modify the protection. This is going to be REALLY efficient.
4586	* PGM lends us two bits to keep track of the state here.
4587	*/
4588	uint8_t const u2OldState = *pu2State;
4589	uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
4590	: fPageProt & NEM_PAGE_PROT_READ ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED;
4591	if ( fBackingChanged
4592	\|\| u2NewState != u2OldState)
4593	{
4594	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
4595	{
4596	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4597	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4598	AssertRC(rc);
4599	if (RT_SUCCESS(rc))
4600	{
4601	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4602	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4603	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
4604	{
4605	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
4606	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4607	return VINF_SUCCESS;
4608	}
4609	}
4610	else
4611	{
4612	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4613	return rc;
4614	}
4615	# else
4616	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE);
4617	if (SUCCEEDED(hrc))
4618	{
4619	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4620	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4621	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
4622	{
4623	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
4624	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4625	return VINF_SUCCESS;
4626	}
4627	}
4628	else
4629	{
4630	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4631	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4632	return VERR_NEM_INIT_FAILED;
4633	}
4634	# endif
4635	}
4636	}
4637
4638	/*
4639	* Writeable mapping?
4640	*/
4641	if (fPageProt & NEM_PAGE_PROT_WRITE)
4642	{
4643	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4644	int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4645	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
4646	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4647	AssertRC(rc);
4648	if (RT_SUCCESS(rc))
4649	{
4650	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4651	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4652	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4653	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4654	return VINF_SUCCESS;
4655	}
4656	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4657	return rc;
4658	# else
4659	void *pvPage;
4660	int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
4661	if (RT_SUCCESS(rc))
4662	{
4663	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE,
4664	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute \| WHvMapGpaRangeFlagWrite);
4665	if (SUCCEEDED(hrc))
4666	{
4667	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4668	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4669	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4670	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4671	return VINF_SUCCESS;
4672	}
4673	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4674	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4675	return VERR_NEM_INIT_FAILED;
4676	}
4677	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
4678	return rc;
4679	# endif
4680	}
4681
4682	if (fPageProt & NEM_PAGE_PROT_READ)
4683	{
4684	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4685	int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4686	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4687	AssertRC(rc);
4688	if (RT_SUCCESS(rc))
4689	{
4690	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4691	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4692	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4693	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4694	return VINF_SUCCESS;
4695	}
4696	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4697	return rc;
4698	# else
4699	const void *pvPage;
4700	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
4701	if (RT_SUCCESS(rc))
4702	{
4703	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE,
4704	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute);
4705	if (SUCCEEDED(hrc))
4706	{
4707	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4708	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4709	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4710	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4711	return VINF_SUCCESS;
4712	}
4713	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4714	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4715	return VERR_NEM_INIT_FAILED;
4716	}
4717	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
4718	return rc;
4719	# endif
4720	}
4721
4722	/* We already unmapped it above. */
4723	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4724	return VINF_SUCCESS;
4725	#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
4726	}
4727
4728
4729	NEM_TMPL_STATIC int nemHCJustUnmapPageFromHyperV(PVM pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
4730	{
4731	if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED)
4732	{
4733	Log5(("nemHCJustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst));
4734	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4735	return VINF_SUCCESS;
4736	}
4737
4738	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4739	PVMCPU pVCpu = VMMGetCpu(pVM);
4740	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4741	AssertRC(rc);
4742	if (RT_SUCCESS(rc))
4743	{
4744	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4745	Log5(("NEM GPA unmapped/just: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[*pu2State], cMappedPages));
4746	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4747	return VINF_SUCCESS;
4748	}
4749	LogRel(("nemHCJustUnmapPageFromHyperV/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4750	return rc;
4751	#else
4752	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
4753	if (SUCCEEDED(hrc))
4754	{
4755	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4756	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4757	Log5(("nemHCJustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
4758	return VINF_SUCCESS;
4759	}
4760	LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n",
4761	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4762	return VERR_NEM_IPE_6;
4763	#endif
4764	}
4765
4766
4767	int nemHCNativeNotifyPhysPageAllocated(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
4768	PGMPAGETYPE enmType, uint8_t *pu2State)
4769	{
4770	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4771	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
4772	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
4773
4774	int rc;
4775	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4776	PVMCPU pVCpu = VMMGetCpu(pVM);
4777	if ( pVM->nem.s.fA20Enabled
4778	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4779	rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4780	else
4781	{
4782	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
4783	rc = nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
4784	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys) && RT_SUCCESS(rc))
4785	rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4786
4787	}
4788	#else
4789	RT_NOREF_PV(fPageProt);
4790	if ( pVM->nem.s.fA20Enabled
4791	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4792	rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4793	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4794	rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4795	else
4796	rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */
4797	#endif
4798	return rc;
4799	}
4800
4801
4802	void nemHCNativeNotifyPhysPageProtChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
4803	PGMPAGETYPE enmType, uint8_t *pu2State)
4804	{
4805	Log5(("nemHCNativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4806	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
4807	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
4808
4809	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4810	PVMCPU pVCpu = VMMGetCpu(pVM);
4811	if ( pVM->nem.s.fA20Enabled
4812	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4813	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
4814	else
4815	{
4816	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
4817	nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
4818	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4819	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
4820	}
4821	#else
4822	RT_NOREF_PV(fPageProt);
4823	if ( pVM->nem.s.fA20Enabled
4824	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4825	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4826	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4827	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4828	/* else: ignore since we've got the alias page at this address. */
4829	#endif
4830	}
4831
4832
4833	void nemHCNativeNotifyPhysPageChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
4834	uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
4835	{
4836	Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4837	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
4838	RT_NOREF_PV(HCPhysPrev); RT_NOREF_PV(HCPhysNew); RT_NOREF_PV(enmType);
4839
4840	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4841	PVMCPU pVCpu = VMMGetCpu(pVM);
4842	if ( pVM->nem.s.fA20Enabled
4843	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4844	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4845	else
4846	{
4847	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
4848	nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
4849	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4850	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4851	}
4852	#else
4853	RT_NOREF_PV(fPageProt);
4854	if ( pVM->nem.s.fA20Enabled
4855	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4856	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4857	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4858	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4859	/* else: ignore since we've got the alias page at this address. */
4860	#endif
4861	}
4862

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h@ 75249

Download in other formats: