NEMAllNativeTemplate-win.cpp.h@ 92561

Last change on this file since 92561 was 92541, checked in by vboxsync, 3 years ago
VMM: Nested VMX: bugref:10092 Allow forcing mapping/unmapping of CR3 even when the paging mode deosn't actually change. This is required for VMX/SVM guest transitions.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 241.4 KB

Line
1	/* $Id: NEMAllNativeTemplate-win.cpp.h 92541 2021-11-22 06:35:38Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, Windows code template ring-0/3.
4	*/
5
6	/*
7	* Copyright (C) 2018-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Defined Constants And Macros *
21	*********************************************************************************************************************************/
22	/** Copy back a segment from hyper-V. */
23	#define NEM_WIN_COPY_BACK_SEG(a_Dst, a_Src) \
24	do { \
25	(a_Dst).u64Base = (a_Src).Base; \
26	(a_Dst).u32Limit = (a_Src).Limit; \
27	(a_Dst).ValidSel = (a_Dst).Sel = (a_Src).Selector; \
28	(a_Dst).Attr.u = (a_Src).Attributes; \
29	(a_Dst).fFlags = CPUMSELREG_FLAGS_VALID; \
30	} while (0)
31
32	/** @def NEMWIN_ASSERT_MSG_REG_VAL
33	* Asserts the correctness of a register value in a message/context.
34	*/
35	#if 0
36	# define NEMWIN_NEED_GET_REGISTER
37	# if defined(IN_RING0) \|\| defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
38	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \
39	do { \
40	HV_REGISTER_VALUE TmpVal; \
41	nemHCWinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
42	AssertMsg(a_Expr, a_Msg); \
43	} while (0)
44	# else
45	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \
46	do { \
47	WHV_REGISTER_VALUE TmpVal; \
48	nemR3WinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
49	AssertMsg(a_Expr, a_Msg); \
50	} while (0)
51	# endif
52	#else
53	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) do { } while (0)
54	#endif
55
56	/** @def NEMWIN_ASSERT_MSG_REG_VAL
57	* Asserts the correctness of a 64-bit register value in a message/context.
58	*/
59	#define NEMWIN_ASSERT_MSG_REG_VAL64(a_pVCpu, a_enmReg, a_u64Val) \
60	NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, (a_u64Val) == TmpVal.Reg64, \
61	(#a_u64Val "=%#RX64, expected %#RX64\n", (a_u64Val), TmpVal.Reg64))
62	/** @def NEMWIN_ASSERT_MSG_REG_VAL
63	* Asserts the correctness of a segment register value in a message/context.
64	*/
65	#define NEMWIN_ASSERT_MSG_REG_SEG(a_pVCpu, a_enmReg, a_SReg) \
66	NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, \
67	(a_SReg).Base == TmpVal.Segment.Base \
68	&& (a_SReg).Limit == TmpVal.Segment.Limit \
69	&& (a_SReg).Selector == TmpVal.Segment.Selector \
70	&& (a_SReg).Attributes == TmpVal.Segment.Attributes, \
71	( #a_SReg "=%#RX16 {%#RX64 LB %#RX32,%#RX16} expected %#RX16 {%#RX64 LB %#RX32,%#RX16}\n", \
72	(a_SReg).Selector, (a_SReg).Base, (a_SReg).Limit, (a_SReg).Attributes, \
73	TmpVal.Segment.Selector, TmpVal.Segment.Base, TmpVal.Segment.Limit, TmpVal.Segment.Attributes))
74
75
76	/*********************************************************************************************************************************
77	* Global Variables *
78	*********************************************************************************************************************************/
79	/** NEM_WIN_PAGE_STATE_XXX names. */
80	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
81
82	/** HV_INTERCEPT_ACCESS_TYPE names. */
83	static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" };
84
85
86	/*********************************************************************************************************************************
87	* Internal Functions *
88	*********************************************************************************************************************************/
89	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
90	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged);
91
92
93
94	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
95
96	/**
97	* Wrapper around VMMR0_DO_NEM_MAP_PAGES for a single page.
98	*
99	* @returns VBox status code.
100	* @param pVM The cross context VM structure.
101	* @param pVCpu The cross context virtual CPU structure of the caller.
102	* @param GCPhysSrc The source page. Does not need to be page aligned.
103	* @param GCPhysDst The destination page. Same as @a GCPhysSrc except for
104	* when A20 is disabled.
105	* @param fFlags HV_MAP_GPA_XXX.
106	*/
107	DECLINLINE(int) nemHCWinHypercallMapPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fFlags)
108	{
109	#ifdef IN_RING0
110	/** @todo optimize further, caller generally has the physical address. */
111	return nemR0WinMapPages(pVM, pVCpu,
112	GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
113	GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
114	1, fFlags);
115	#else
116	pVCpu->nem.s.Hypercall.MapPages.GCPhysSrc = GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
117	pVCpu->nem.s.Hypercall.MapPages.GCPhysDst = GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
118	pVCpu->nem.s.Hypercall.MapPages.cPages = 1;
119	pVCpu->nem.s.Hypercall.MapPages.fFlags = fFlags;
120	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_MAP_PAGES, 0, NULL);
121	#endif
122	}
123
124
125	/**
126	* Wrapper around VMMR0_DO_NEM_UNMAP_PAGES for a single page.
127	*
128	* @returns VBox status code.
129	* @param pVM The cross context VM structure.
130	* @param pVCpu The cross context virtual CPU structure of the caller.
131	* @param GCPhys The page to unmap. Does not need to be page aligned.
132	*/
133	DECLINLINE(int) nemHCWinHypercallUnmapPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys)
134	{
135	# ifdef IN_RING0
136	return nemR0WinUnmapPages(pVM, pVCpu, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, 1);
137	# else
138	pVCpu->nem.s.Hypercall.UnmapPages.GCPhys = GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
139	pVCpu->nem.s.Hypercall.UnmapPages.cPages = 1;
140	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_UNMAP_PAGES, 0, NULL);
141	# endif
142	}
143
144	#endif /* NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
145	#ifndef IN_RING0
146
147	NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVMCC pVM, PVMCPUCC pVCpu)
148	{
149	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
150	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
151	if (pVM->nem.s.fUseRing0Runloop)
152	# endif
153	{
154	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPORT_STATE, 0, NULL);
155	AssertLogRelRCReturn(rc, rc);
156	return rc;
157	}
158	# endif
159	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
160
161	/*
162	* The following is very similar to what nemR0WinExportState() does.
163	*/
164	WHV_REGISTER_NAME aenmNames[128];
165	WHV_REGISTER_VALUE aValues[128];
166
167	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK);
168	if ( !fWhat
169	&& pVCpu->nem.s.fCurrentInterruptWindows == pVCpu->nem.s.fDesiredInterruptWindows)
170	return VINF_SUCCESS;
171	uintptr_t iReg = 0;
172
173	# define ADD_REG64(a_enmName, a_uValue) do { \
174	aenmNames[iReg] = (a_enmName); \
175	aValues[iReg].Reg128.High64 = 0; \
176	aValues[iReg].Reg64 = (a_uValue); \
177	iReg++; \
178	} while (0)
179	# define ADD_REG128(a_enmName, a_uValueLo, a_uValueHi) do { \
180	aenmNames[iReg] = (a_enmName); \
181	aValues[iReg].Reg128.Low64 = (a_uValueLo); \
182	aValues[iReg].Reg128.High64 = (a_uValueHi); \
183	iReg++; \
184	} while (0)
185
186	/* GPRs */
187	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
188	{
189	if (fWhat & CPUMCTX_EXTRN_RAX)
190	ADD_REG64(WHvX64RegisterRax, pVCpu->cpum.GstCtx.rax);
191	if (fWhat & CPUMCTX_EXTRN_RCX)
192	ADD_REG64(WHvX64RegisterRcx, pVCpu->cpum.GstCtx.rcx);
193	if (fWhat & CPUMCTX_EXTRN_RDX)
194	ADD_REG64(WHvX64RegisterRdx, pVCpu->cpum.GstCtx.rdx);
195	if (fWhat & CPUMCTX_EXTRN_RBX)
196	ADD_REG64(WHvX64RegisterRbx, pVCpu->cpum.GstCtx.rbx);
197	if (fWhat & CPUMCTX_EXTRN_RSP)
198	ADD_REG64(WHvX64RegisterRsp, pVCpu->cpum.GstCtx.rsp);
199	if (fWhat & CPUMCTX_EXTRN_RBP)
200	ADD_REG64(WHvX64RegisterRbp, pVCpu->cpum.GstCtx.rbp);
201	if (fWhat & CPUMCTX_EXTRN_RSI)
202	ADD_REG64(WHvX64RegisterRsi, pVCpu->cpum.GstCtx.rsi);
203	if (fWhat & CPUMCTX_EXTRN_RDI)
204	ADD_REG64(WHvX64RegisterRdi, pVCpu->cpum.GstCtx.rdi);
205	if (fWhat & CPUMCTX_EXTRN_R8_R15)
206	{
207	ADD_REG64(WHvX64RegisterR8, pVCpu->cpum.GstCtx.r8);
208	ADD_REG64(WHvX64RegisterR9, pVCpu->cpum.GstCtx.r9);
209	ADD_REG64(WHvX64RegisterR10, pVCpu->cpum.GstCtx.r10);
210	ADD_REG64(WHvX64RegisterR11, pVCpu->cpum.GstCtx.r11);
211	ADD_REG64(WHvX64RegisterR12, pVCpu->cpum.GstCtx.r12);
212	ADD_REG64(WHvX64RegisterR13, pVCpu->cpum.GstCtx.r13);
213	ADD_REG64(WHvX64RegisterR14, pVCpu->cpum.GstCtx.r14);
214	ADD_REG64(WHvX64RegisterR15, pVCpu->cpum.GstCtx.r15);
215	}
216	}
217
218	/* RIP & Flags */
219	if (fWhat & CPUMCTX_EXTRN_RIP)
220	ADD_REG64(WHvX64RegisterRip, pVCpu->cpum.GstCtx.rip);
221	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
222	ADD_REG64(WHvX64RegisterRflags, pVCpu->cpum.GstCtx.rflags.u);
223
224	/* Segments */
225	# define ADD_SEG(a_enmName, a_SReg) \
226	do { \
227	aenmNames[iReg] = a_enmName; \
228	aValues[iReg].Segment.Base = (a_SReg).u64Base; \
229	aValues[iReg].Segment.Limit = (a_SReg).u32Limit; \
230	aValues[iReg].Segment.Selector = (a_SReg).Sel; \
231	aValues[iReg].Segment.Attributes = (a_SReg).Attr.u; \
232	iReg++; \
233	} while (0)
234	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
235	{
236	if (fWhat & CPUMCTX_EXTRN_ES)
237	ADD_SEG(WHvX64RegisterEs, pVCpu->cpum.GstCtx.es);
238	if (fWhat & CPUMCTX_EXTRN_CS)
239	ADD_SEG(WHvX64RegisterCs, pVCpu->cpum.GstCtx.cs);
240	if (fWhat & CPUMCTX_EXTRN_SS)
241	ADD_SEG(WHvX64RegisterSs, pVCpu->cpum.GstCtx.ss);
242	if (fWhat & CPUMCTX_EXTRN_DS)
243	ADD_SEG(WHvX64RegisterDs, pVCpu->cpum.GstCtx.ds);
244	if (fWhat & CPUMCTX_EXTRN_FS)
245	ADD_SEG(WHvX64RegisterFs, pVCpu->cpum.GstCtx.fs);
246	if (fWhat & CPUMCTX_EXTRN_GS)
247	ADD_SEG(WHvX64RegisterGs, pVCpu->cpum.GstCtx.gs);
248	}
249
250	/* Descriptor tables & task segment. */
251	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
252	{
253	if (fWhat & CPUMCTX_EXTRN_LDTR)
254	ADD_SEG(WHvX64RegisterLdtr, pVCpu->cpum.GstCtx.ldtr);
255	if (fWhat & CPUMCTX_EXTRN_TR)
256	ADD_SEG(WHvX64RegisterTr, pVCpu->cpum.GstCtx.tr);
257	if (fWhat & CPUMCTX_EXTRN_IDTR)
258	{
259	aenmNames[iReg] = WHvX64RegisterIdtr;
260	aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.idtr.cbIdt;
261	aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.idtr.pIdt;
262	iReg++;
263	}
264	if (fWhat & CPUMCTX_EXTRN_GDTR)
265	{
266	aenmNames[iReg] = WHvX64RegisterGdtr;
267	aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.gdtr.cbGdt;
268	aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.gdtr.pGdt;
269	iReg++;
270	}
271	}
272
273	/* Control registers. */
274	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
275	{
276	if (fWhat & CPUMCTX_EXTRN_CR0)
277	ADD_REG64(WHvX64RegisterCr0, pVCpu->cpum.GstCtx.cr0);
278	if (fWhat & CPUMCTX_EXTRN_CR2)
279	ADD_REG64(WHvX64RegisterCr2, pVCpu->cpum.GstCtx.cr2);
280	if (fWhat & CPUMCTX_EXTRN_CR3)
281	ADD_REG64(WHvX64RegisterCr3, pVCpu->cpum.GstCtx.cr3);
282	if (fWhat & CPUMCTX_EXTRN_CR4)
283	ADD_REG64(WHvX64RegisterCr4, pVCpu->cpum.GstCtx.cr4);
284	}
285	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
286	ADD_REG64(WHvX64RegisterCr8, CPUMGetGuestCR8(pVCpu));
287
288	/* Debug registers. */
289	/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
290	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
291	{
292	ADD_REG64(WHvX64RegisterDr0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
293	ADD_REG64(WHvX64RegisterDr1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
294	ADD_REG64(WHvX64RegisterDr2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
295	ADD_REG64(WHvX64RegisterDr3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
296	}
297	if (fWhat & CPUMCTX_EXTRN_DR6)
298	ADD_REG64(WHvX64RegisterDr6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
299	if (fWhat & CPUMCTX_EXTRN_DR7)
300	ADD_REG64(WHvX64RegisterDr7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
301
302	/* Floating point state. */
303	if (fWhat & CPUMCTX_EXTRN_X87)
304	{
305	ADD_REG128(WHvX64RegisterFpMmx0, pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[1]);
306	ADD_REG128(WHvX64RegisterFpMmx1, pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[1]);
307	ADD_REG128(WHvX64RegisterFpMmx2, pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[1]);
308	ADD_REG128(WHvX64RegisterFpMmx3, pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[1]);
309	ADD_REG128(WHvX64RegisterFpMmx4, pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[1]);
310	ADD_REG128(WHvX64RegisterFpMmx5, pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[1]);
311	ADD_REG128(WHvX64RegisterFpMmx6, pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[1]);
312	ADD_REG128(WHvX64RegisterFpMmx7, pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[1]);
313
314	aenmNames[iReg] = WHvX64RegisterFpControlStatus;
315	aValues[iReg].FpControlStatus.FpControl = pVCpu->cpum.GstCtx.XState.x87.FCW;
316	aValues[iReg].FpControlStatus.FpStatus = pVCpu->cpum.GstCtx.XState.x87.FSW;
317	aValues[iReg].FpControlStatus.FpTag = pVCpu->cpum.GstCtx.XState.x87.FTW;
318	aValues[iReg].FpControlStatus.Reserved = pVCpu->cpum.GstCtx.XState.x87.FTW >> 8;
319	aValues[iReg].FpControlStatus.LastFpOp = pVCpu->cpum.GstCtx.XState.x87.FOP;
320	aValues[iReg].FpControlStatus.LastFpRip = (pVCpu->cpum.GstCtx.XState.x87.FPUIP)
321	\| ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.CS << 32)
322	\| ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.Rsrvd1 << 48);
323	iReg++;
324
325	aenmNames[iReg] = WHvX64RegisterXmmControlStatus;
326	aValues[iReg].XmmControlStatus.LastFpRdp = (pVCpu->cpum.GstCtx.XState.x87.FPUDP)
327	\| ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.DS << 32)
328	\| ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.Rsrvd2 << 48);
329	aValues[iReg].XmmControlStatus.XmmStatusControl = pVCpu->cpum.GstCtx.XState.x87.MXCSR;
330	aValues[iReg].XmmControlStatus.XmmStatusControlMask = pVCpu->cpum.GstCtx.XState.x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
331	iReg++;
332	}
333
334	/* Vector state. */
335	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
336	{
337	ADD_REG128(WHvX64RegisterXmm0, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Hi);
338	ADD_REG128(WHvX64RegisterXmm1, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Hi);
339	ADD_REG128(WHvX64RegisterXmm2, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Hi);
340	ADD_REG128(WHvX64RegisterXmm3, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Hi);
341	ADD_REG128(WHvX64RegisterXmm4, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Hi);
342	ADD_REG128(WHvX64RegisterXmm5, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Hi);
343	ADD_REG128(WHvX64RegisterXmm6, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Hi);
344	ADD_REG128(WHvX64RegisterXmm7, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Hi);
345	ADD_REG128(WHvX64RegisterXmm8, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Hi);
346	ADD_REG128(WHvX64RegisterXmm9, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Hi);
347	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Hi);
348	ADD_REG128(WHvX64RegisterXmm11, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Hi);
349	ADD_REG128(WHvX64RegisterXmm12, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Hi);
350	ADD_REG128(WHvX64RegisterXmm13, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Hi);
351	ADD_REG128(WHvX64RegisterXmm14, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Hi);
352	ADD_REG128(WHvX64RegisterXmm15, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Hi);
353	}
354
355	/* MSRs */
356	// WHvX64RegisterTsc - don't touch
357	if (fWhat & CPUMCTX_EXTRN_EFER)
358	ADD_REG64(WHvX64RegisterEfer, pVCpu->cpum.GstCtx.msrEFER);
359	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
360	ADD_REG64(WHvX64RegisterKernelGsBase, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
361	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
362	{
363	ADD_REG64(WHvX64RegisterSysenterCs, pVCpu->cpum.GstCtx.SysEnter.cs);
364	ADD_REG64(WHvX64RegisterSysenterEip, pVCpu->cpum.GstCtx.SysEnter.eip);
365	ADD_REG64(WHvX64RegisterSysenterEsp, pVCpu->cpum.GstCtx.SysEnter.esp);
366	}
367	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
368	{
369	ADD_REG64(WHvX64RegisterStar, pVCpu->cpum.GstCtx.msrSTAR);
370	ADD_REG64(WHvX64RegisterLstar, pVCpu->cpum.GstCtx.msrLSTAR);
371	ADD_REG64(WHvX64RegisterCstar, pVCpu->cpum.GstCtx.msrCSTAR);
372	ADD_REG64(WHvX64RegisterSfmask, pVCpu->cpum.GstCtx.msrSFMASK);
373	}
374	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
375	{
376	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
377	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
378	ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
379	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
380	{
381	ADD_REG64(WHvX64RegisterApicBase, APICGetBaseMsrNoCheck(pVCpu));
382	ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
383	#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
384	ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
385	#endif
386	ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
387	ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
388	ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
389	ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
390	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000, pCtxMsrs->msr.MtrrFix4K_C0000);
391	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000, pCtxMsrs->msr.MtrrFix4K_C8000);
392	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000, pCtxMsrs->msr.MtrrFix4K_D0000);
393	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000, pCtxMsrs->msr.MtrrFix4K_D8000);
394	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000, pCtxMsrs->msr.MtrrFix4K_E0000);
395	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000, pCtxMsrs->msr.MtrrFix4K_E8000);
396	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000, pCtxMsrs->msr.MtrrFix4K_F0000);
397	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000, pCtxMsrs->msr.MtrrFix4K_F8000);
398	#if 0 /** @todo these registers aren't available? Might explain something.. .*/
399	const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
400	if (enmCpuVendor != CPUMCPUVENDOR_AMD)
401	{
402	ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
403	ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
404	}
405	#endif
406	}
407	}
408
409	/* event injection (clear it). */
410	if (fWhat & CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)
411	ADD_REG64(WHvRegisterPendingInterruption, 0);
412
413	/* Interruptibility state. This can get a little complicated since we get
414	half of the state via HV_X64_VP_EXECUTION_STATE. */
415	if ( (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
416	== (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI) )
417	{
418	ADD_REG64(WHvRegisterInterruptState, 0);
419	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
420	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
421	aValues[iReg - 1].InterruptState.InterruptShadow = 1;
422	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
423	aValues[iReg - 1].InterruptState.NmiMasked = 1;
424	}
425	else if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
426	{
427	if ( pVCpu->nem.s.fLastInterruptShadow
428	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
429	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip))
430	{
431	ADD_REG64(WHvRegisterInterruptState, 0);
432	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
433	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
434	aValues[iReg - 1].InterruptState.InterruptShadow = 1;
435	/** @todo Retrieve NMI state, currently assuming it's zero. (yes this may happen on I/O) */
436	//if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
437	// aValues[iReg - 1].InterruptState.NmiMasked = 1;
438	}
439	}
440	else
441	Assert(!(fWhat & CPUMCTX_EXTRN_INHIBIT_NMI));
442
443	/* Interrupt windows. Always set if active as Hyper-V seems to be forgetful. */
444	uint8_t const fDesiredIntWin = pVCpu->nem.s.fDesiredInterruptWindows;
445	if ( fDesiredIntWin
446	\|\| pVCpu->nem.s.fCurrentInterruptWindows != fDesiredIntWin)
447	{
448	pVCpu->nem.s.fCurrentInterruptWindows = pVCpu->nem.s.fDesiredInterruptWindows;
449	Log8(("Setting WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin=%X\n", fDesiredIntWin));
450	ADD_REG64(WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin);
451	Assert(aValues[iReg - 1].DeliverabilityNotifications.NmiNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_NMI));
452	Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_REGULAR));
453	Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptPriority == (unsigned)((fDesiredIntWin & NEM_WIN_INTW_F_PRIO_MASK) >> NEM_WIN_INTW_F_PRIO_SHIFT));
454	}
455
456	/// @todo WHvRegisterPendingEvent
457
458	/*
459	* Set the registers.
460	*/
461	Assert(iReg < RT_ELEMENTS(aValues));
462	Assert(iReg < RT_ELEMENTS(aenmNames));
463	# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
464	Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
465	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
466	# endif
467	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
468	if (SUCCEEDED(hrc))
469	{
470	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK \| CPUMCTX_EXTRN_KEEPER_NEM;
471	return VINF_SUCCESS;
472	}
473	AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
474	pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
475	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
476	return VERR_INTERNAL_ERROR;
477
478	# undef ADD_REG64
479	# undef ADD_REG128
480	# undef ADD_SEG
481
482	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
483	}
484
485
486	NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
487	{
488	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
489	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
490	if (pVM->nem.s.fUseRing0Runloop)
491	# endif
492	{
493	/* See NEMR0ImportState */
494	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_IMPORT_STATE, fWhat, NULL);
495	if (RT_SUCCESS(rc))
496	return rc;
497	if (rc == VERR_NEM_FLUSH_TLB)
498	{
499	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /fGlobal/, false /fPdpesMapped/);
500	return rc;
501	}
502	AssertLogRelRCReturn(rc, rc);
503	return rc;
504	}
505	# endif
506	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
507	WHV_REGISTER_NAME aenmNames[128];
508
509	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
510	uintptr_t iReg = 0;
511
512	/* GPRs */
513	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
514	{
515	if (fWhat & CPUMCTX_EXTRN_RAX)
516	aenmNames[iReg++] = WHvX64RegisterRax;
517	if (fWhat & CPUMCTX_EXTRN_RCX)
518	aenmNames[iReg++] = WHvX64RegisterRcx;
519	if (fWhat & CPUMCTX_EXTRN_RDX)
520	aenmNames[iReg++] = WHvX64RegisterRdx;
521	if (fWhat & CPUMCTX_EXTRN_RBX)
522	aenmNames[iReg++] = WHvX64RegisterRbx;
523	if (fWhat & CPUMCTX_EXTRN_RSP)
524	aenmNames[iReg++] = WHvX64RegisterRsp;
525	if (fWhat & CPUMCTX_EXTRN_RBP)
526	aenmNames[iReg++] = WHvX64RegisterRbp;
527	if (fWhat & CPUMCTX_EXTRN_RSI)
528	aenmNames[iReg++] = WHvX64RegisterRsi;
529	if (fWhat & CPUMCTX_EXTRN_RDI)
530	aenmNames[iReg++] = WHvX64RegisterRdi;
531	if (fWhat & CPUMCTX_EXTRN_R8_R15)
532	{
533	aenmNames[iReg++] = WHvX64RegisterR8;
534	aenmNames[iReg++] = WHvX64RegisterR9;
535	aenmNames[iReg++] = WHvX64RegisterR10;
536	aenmNames[iReg++] = WHvX64RegisterR11;
537	aenmNames[iReg++] = WHvX64RegisterR12;
538	aenmNames[iReg++] = WHvX64RegisterR13;
539	aenmNames[iReg++] = WHvX64RegisterR14;
540	aenmNames[iReg++] = WHvX64RegisterR15;
541	}
542	}
543
544	/* RIP & Flags */
545	if (fWhat & CPUMCTX_EXTRN_RIP)
546	aenmNames[iReg++] = WHvX64RegisterRip;
547	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
548	aenmNames[iReg++] = WHvX64RegisterRflags;
549
550	/* Segments */
551	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
552	{
553	if (fWhat & CPUMCTX_EXTRN_ES)
554	aenmNames[iReg++] = WHvX64RegisterEs;
555	if (fWhat & CPUMCTX_EXTRN_CS)
556	aenmNames[iReg++] = WHvX64RegisterCs;
557	if (fWhat & CPUMCTX_EXTRN_SS)
558	aenmNames[iReg++] = WHvX64RegisterSs;
559	if (fWhat & CPUMCTX_EXTRN_DS)
560	aenmNames[iReg++] = WHvX64RegisterDs;
561	if (fWhat & CPUMCTX_EXTRN_FS)
562	aenmNames[iReg++] = WHvX64RegisterFs;
563	if (fWhat & CPUMCTX_EXTRN_GS)
564	aenmNames[iReg++] = WHvX64RegisterGs;
565	}
566
567	/* Descriptor tables. */
568	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
569	{
570	if (fWhat & CPUMCTX_EXTRN_LDTR)
571	aenmNames[iReg++] = WHvX64RegisterLdtr;
572	if (fWhat & CPUMCTX_EXTRN_TR)
573	aenmNames[iReg++] = WHvX64RegisterTr;
574	if (fWhat & CPUMCTX_EXTRN_IDTR)
575	aenmNames[iReg++] = WHvX64RegisterIdtr;
576	if (fWhat & CPUMCTX_EXTRN_GDTR)
577	aenmNames[iReg++] = WHvX64RegisterGdtr;
578	}
579
580	/* Control registers. */
581	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
582	{
583	if (fWhat & CPUMCTX_EXTRN_CR0)
584	aenmNames[iReg++] = WHvX64RegisterCr0;
585	if (fWhat & CPUMCTX_EXTRN_CR2)
586	aenmNames[iReg++] = WHvX64RegisterCr2;
587	if (fWhat & CPUMCTX_EXTRN_CR3)
588	aenmNames[iReg++] = WHvX64RegisterCr3;
589	if (fWhat & CPUMCTX_EXTRN_CR4)
590	aenmNames[iReg++] = WHvX64RegisterCr4;
591	}
592	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
593	aenmNames[iReg++] = WHvX64RegisterCr8;
594
595	/* Debug registers. */
596	if (fWhat & CPUMCTX_EXTRN_DR7)
597	aenmNames[iReg++] = WHvX64RegisterDr7;
598	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
599	{
600	if (!(fWhat & CPUMCTX_EXTRN_DR7) && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_DR7))
601	{
602	fWhat \|= CPUMCTX_EXTRN_DR7;
603	aenmNames[iReg++] = WHvX64RegisterDr7;
604	}
605	aenmNames[iReg++] = WHvX64RegisterDr0;
606	aenmNames[iReg++] = WHvX64RegisterDr1;
607	aenmNames[iReg++] = WHvX64RegisterDr2;
608	aenmNames[iReg++] = WHvX64RegisterDr3;
609	}
610	if (fWhat & CPUMCTX_EXTRN_DR6)
611	aenmNames[iReg++] = WHvX64RegisterDr6;
612
613	/* Floating point state. */
614	if (fWhat & CPUMCTX_EXTRN_X87)
615	{
616	aenmNames[iReg++] = WHvX64RegisterFpMmx0;
617	aenmNames[iReg++] = WHvX64RegisterFpMmx1;
618	aenmNames[iReg++] = WHvX64RegisterFpMmx2;
619	aenmNames[iReg++] = WHvX64RegisterFpMmx3;
620	aenmNames[iReg++] = WHvX64RegisterFpMmx4;
621	aenmNames[iReg++] = WHvX64RegisterFpMmx5;
622	aenmNames[iReg++] = WHvX64RegisterFpMmx6;
623	aenmNames[iReg++] = WHvX64RegisterFpMmx7;
624	aenmNames[iReg++] = WHvX64RegisterFpControlStatus;
625	}
626	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
627	aenmNames[iReg++] = WHvX64RegisterXmmControlStatus;
628
629	/* Vector state. */
630	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
631	{
632	aenmNames[iReg++] = WHvX64RegisterXmm0;
633	aenmNames[iReg++] = WHvX64RegisterXmm1;
634	aenmNames[iReg++] = WHvX64RegisterXmm2;
635	aenmNames[iReg++] = WHvX64RegisterXmm3;
636	aenmNames[iReg++] = WHvX64RegisterXmm4;
637	aenmNames[iReg++] = WHvX64RegisterXmm5;
638	aenmNames[iReg++] = WHvX64RegisterXmm6;
639	aenmNames[iReg++] = WHvX64RegisterXmm7;
640	aenmNames[iReg++] = WHvX64RegisterXmm8;
641	aenmNames[iReg++] = WHvX64RegisterXmm9;
642	aenmNames[iReg++] = WHvX64RegisterXmm10;
643	aenmNames[iReg++] = WHvX64RegisterXmm11;
644	aenmNames[iReg++] = WHvX64RegisterXmm12;
645	aenmNames[iReg++] = WHvX64RegisterXmm13;
646	aenmNames[iReg++] = WHvX64RegisterXmm14;
647	aenmNames[iReg++] = WHvX64RegisterXmm15;
648	}
649
650	/* MSRs */
651	// WHvX64RegisterTsc - don't touch
652	if (fWhat & CPUMCTX_EXTRN_EFER)
653	aenmNames[iReg++] = WHvX64RegisterEfer;
654	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
655	aenmNames[iReg++] = WHvX64RegisterKernelGsBase;
656	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
657	{
658	aenmNames[iReg++] = WHvX64RegisterSysenterCs;
659	aenmNames[iReg++] = WHvX64RegisterSysenterEip;
660	aenmNames[iReg++] = WHvX64RegisterSysenterEsp;
661	}
662	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
663	{
664	aenmNames[iReg++] = WHvX64RegisterStar;
665	aenmNames[iReg++] = WHvX64RegisterLstar;
666	aenmNames[iReg++] = WHvX64RegisterCstar;
667	aenmNames[iReg++] = WHvX64RegisterSfmask;
668	}
669
670	//#ifdef LOG_ENABLED
671	// const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
672	//#endif
673	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
674	aenmNames[iReg++] = WHvX64RegisterTscAux;
675	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
676	{
677	aenmNames[iReg++] = WHvX64RegisterApicBase; /// @todo APIC BASE
678	aenmNames[iReg++] = WHvX64RegisterPat;
679	#if 0 /def LOG_ENABLED/ /** @todo Check if WHvX64RegisterMsrMtrrCap works... */
680	aenmNames[iReg++] = WHvX64RegisterMsrMtrrCap;
681	#endif
682	aenmNames[iReg++] = WHvX64RegisterMsrMtrrDefType;
683	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix64k00000;
684	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16k80000;
685	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16kA0000;
686	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC0000;
687	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC8000;
688	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD0000;
689	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD8000;
690	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE0000;
691	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE8000;
692	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF0000;
693	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF8000;
694	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
695	//#ifdef LOG_ENABLED
696	// if (enmCpuVendor != CPUMCPUVENDOR_AMD)
697	// aenmNames[iReg++] = HvX64RegisterIa32FeatureControl;
698	//#endif
699	}
700
701	/* Interruptibility. */
702	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
703	{
704	aenmNames[iReg++] = WHvRegisterInterruptState;
705	aenmNames[iReg++] = WHvX64RegisterRip;
706	}
707
708	/* event injection */
709	aenmNames[iReg++] = WHvRegisterPendingInterruption;
710	aenmNames[iReg++] = WHvRegisterPendingEvent0; /** @todo renamed to WHvRegisterPendingEvent */
711
712	size_t const cRegs = iReg;
713	Assert(cRegs < RT_ELEMENTS(aenmNames));
714
715	/*
716	* Get the registers.
717	*/
718	WHV_REGISTER_VALUE aValues[128];
719	RT_ZERO(aValues);
720	Assert(RT_ELEMENTS(aValues) >= cRegs);
721	Assert(RT_ELEMENTS(aenmNames) >= cRegs);
722	# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
723	Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
724	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
725	# endif
726	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, (uint32_t)cRegs, aValues);
727	AssertLogRelMsgReturn(SUCCEEDED(hrc),
728	("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
729	pVM->nem.s.hPartition, pVCpu->idCpu, cRegs, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
730	, VERR_NEM_GET_REGISTERS_FAILED);
731
732	iReg = 0;
733	# define GET_REG64(a_DstVar, a_enmName) do { \
734	Assert(aenmNames[iReg] == (a_enmName)); \
735	(a_DstVar) = aValues[iReg].Reg64; \
736	iReg++; \
737	} while (0)
738	# define GET_REG64_LOG7(a_DstVar, a_enmName, a_szLogName) do { \
739	Assert(aenmNames[iReg] == (a_enmName)); \
740	if ((a_DstVar) != aValues[iReg].Reg64) \
741	Log7(("NEM/%u: " a_szLogName " changed %RX64 -> %RX64\n", pVCpu->idCpu, (a_DstVar), aValues[iReg].Reg64)); \
742	(a_DstVar) = aValues[iReg].Reg64; \
743	iReg++; \
744	} while (0)
745	# define GET_REG128(a_DstVarLo, a_DstVarHi, a_enmName) do { \
746	Assert(aenmNames[iReg] == a_enmName); \
747	(a_DstVarLo) = aValues[iReg].Reg128.Low64; \
748	(a_DstVarHi) = aValues[iReg].Reg128.High64; \
749	iReg++; \
750	} while (0)
751	# define GET_SEG(a_SReg, a_enmName) do { \
752	Assert(aenmNames[iReg] == (a_enmName)); \
753	NEM_WIN_COPY_BACK_SEG(a_SReg, aValues[iReg].Segment); \
754	iReg++; \
755	} while (0)
756
757	/* GPRs */
758	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
759	{
760	if (fWhat & CPUMCTX_EXTRN_RAX)
761	GET_REG64(pVCpu->cpum.GstCtx.rax, WHvX64RegisterRax);
762	if (fWhat & CPUMCTX_EXTRN_RCX)
763	GET_REG64(pVCpu->cpum.GstCtx.rcx, WHvX64RegisterRcx);
764	if (fWhat & CPUMCTX_EXTRN_RDX)
765	GET_REG64(pVCpu->cpum.GstCtx.rdx, WHvX64RegisterRdx);
766	if (fWhat & CPUMCTX_EXTRN_RBX)
767	GET_REG64(pVCpu->cpum.GstCtx.rbx, WHvX64RegisterRbx);
768	if (fWhat & CPUMCTX_EXTRN_RSP)
769	GET_REG64(pVCpu->cpum.GstCtx.rsp, WHvX64RegisterRsp);
770	if (fWhat & CPUMCTX_EXTRN_RBP)
771	GET_REG64(pVCpu->cpum.GstCtx.rbp, WHvX64RegisterRbp);
772	if (fWhat & CPUMCTX_EXTRN_RSI)
773	GET_REG64(pVCpu->cpum.GstCtx.rsi, WHvX64RegisterRsi);
774	if (fWhat & CPUMCTX_EXTRN_RDI)
775	GET_REG64(pVCpu->cpum.GstCtx.rdi, WHvX64RegisterRdi);
776	if (fWhat & CPUMCTX_EXTRN_R8_R15)
777	{
778	GET_REG64(pVCpu->cpum.GstCtx.r8, WHvX64RegisterR8);
779	GET_REG64(pVCpu->cpum.GstCtx.r9, WHvX64RegisterR9);
780	GET_REG64(pVCpu->cpum.GstCtx.r10, WHvX64RegisterR10);
781	GET_REG64(pVCpu->cpum.GstCtx.r11, WHvX64RegisterR11);
782	GET_REG64(pVCpu->cpum.GstCtx.r12, WHvX64RegisterR12);
783	GET_REG64(pVCpu->cpum.GstCtx.r13, WHvX64RegisterR13);
784	GET_REG64(pVCpu->cpum.GstCtx.r14, WHvX64RegisterR14);
785	GET_REG64(pVCpu->cpum.GstCtx.r15, WHvX64RegisterR15);
786	}
787	}
788
789	/* RIP & Flags */
790	if (fWhat & CPUMCTX_EXTRN_RIP)
791	GET_REG64(pVCpu->cpum.GstCtx.rip, WHvX64RegisterRip);
792	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
793	GET_REG64(pVCpu->cpum.GstCtx.rflags.u, WHvX64RegisterRflags);
794
795	/* Segments */
796	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
797	{
798	if (fWhat & CPUMCTX_EXTRN_ES)
799	GET_SEG(pVCpu->cpum.GstCtx.es, WHvX64RegisterEs);
800	if (fWhat & CPUMCTX_EXTRN_CS)
801	GET_SEG(pVCpu->cpum.GstCtx.cs, WHvX64RegisterCs);
802	if (fWhat & CPUMCTX_EXTRN_SS)
803	GET_SEG(pVCpu->cpum.GstCtx.ss, WHvX64RegisterSs);
804	if (fWhat & CPUMCTX_EXTRN_DS)
805	GET_SEG(pVCpu->cpum.GstCtx.ds, WHvX64RegisterDs);
806	if (fWhat & CPUMCTX_EXTRN_FS)
807	GET_SEG(pVCpu->cpum.GstCtx.fs, WHvX64RegisterFs);
808	if (fWhat & CPUMCTX_EXTRN_GS)
809	GET_SEG(pVCpu->cpum.GstCtx.gs, WHvX64RegisterGs);
810	}
811
812	/* Descriptor tables and the task segment. */
813	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
814	{
815	if (fWhat & CPUMCTX_EXTRN_LDTR)
816	GET_SEG(pVCpu->cpum.GstCtx.ldtr, WHvX64RegisterLdtr);
817
818	if (fWhat & CPUMCTX_EXTRN_TR)
819	{
820	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
821	avoid to trigger sanity assertions around the code, always fix this. */
822	GET_SEG(pVCpu->cpum.GstCtx.tr, WHvX64RegisterTr);
823	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
824	{
825	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
826	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
827	break;
828	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
829	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
830	break;
831	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
832	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
833	break;
834	}
835	}
836	if (fWhat & CPUMCTX_EXTRN_IDTR)
837	{
838	Assert(aenmNames[iReg] == WHvX64RegisterIdtr);
839	pVCpu->cpum.GstCtx.idtr.cbIdt = aValues[iReg].Table.Limit;
840	pVCpu->cpum.GstCtx.idtr.pIdt = aValues[iReg].Table.Base;
841	iReg++;
842	}
843	if (fWhat & CPUMCTX_EXTRN_GDTR)
844	{
845	Assert(aenmNames[iReg] == WHvX64RegisterGdtr);
846	pVCpu->cpum.GstCtx.gdtr.cbGdt = aValues[iReg].Table.Limit;
847	pVCpu->cpum.GstCtx.gdtr.pGdt = aValues[iReg].Table.Base;
848	iReg++;
849	}
850	}
851
852	/* Control registers. */
853	bool fMaybeChangedMode = false;
854	bool fUpdateCr3 = false;
855	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
856	{
857	if (fWhat & CPUMCTX_EXTRN_CR0)
858	{
859	Assert(aenmNames[iReg] == WHvX64RegisterCr0);
860	if (pVCpu->cpum.GstCtx.cr0 != aValues[iReg].Reg64)
861	{
862	CPUMSetGuestCR0(pVCpu, aValues[iReg].Reg64);
863	fMaybeChangedMode = true;
864	}
865	iReg++;
866	}
867	if (fWhat & CPUMCTX_EXTRN_CR2)
868	GET_REG64(pVCpu->cpum.GstCtx.cr2, WHvX64RegisterCr2);
869	if (fWhat & CPUMCTX_EXTRN_CR3)
870	{
871	if (pVCpu->cpum.GstCtx.cr3 != aValues[iReg].Reg64)
872	{
873	CPUMSetGuestCR3(pVCpu, aValues[iReg].Reg64);
874	fUpdateCr3 = true;
875	}
876	iReg++;
877	}
878	if (fWhat & CPUMCTX_EXTRN_CR4)
879	{
880	if (pVCpu->cpum.GstCtx.cr4 != aValues[iReg].Reg64)
881	{
882	CPUMSetGuestCR4(pVCpu, aValues[iReg].Reg64);
883	fMaybeChangedMode = true;
884	}
885	iReg++;
886	}
887	}
888	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
889	{
890	Assert(aenmNames[iReg] == WHvX64RegisterCr8);
891	APICSetTpr(pVCpu, (uint8_t)aValues[iReg].Reg64 << 4);
892	iReg++;
893	}
894
895	/* Debug registers. */
896	if (fWhat & CPUMCTX_EXTRN_DR7)
897	{
898	Assert(aenmNames[iReg] == WHvX64RegisterDr7);
899	if (pVCpu->cpum.GstCtx.dr[7] != aValues[iReg].Reg64)
900	CPUMSetGuestDR7(pVCpu, aValues[iReg].Reg64);
901	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
902	iReg++;
903	}
904	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
905	{
906	Assert(aenmNames[iReg] == WHvX64RegisterDr0);
907	Assert(aenmNames[iReg+3] == WHvX64RegisterDr3);
908	if (pVCpu->cpum.GstCtx.dr[0] != aValues[iReg].Reg64)
909	CPUMSetGuestDR0(pVCpu, aValues[iReg].Reg64);
910	iReg++;
911	if (pVCpu->cpum.GstCtx.dr[1] != aValues[iReg].Reg64)
912	CPUMSetGuestDR1(pVCpu, aValues[iReg].Reg64);
913	iReg++;
914	if (pVCpu->cpum.GstCtx.dr[2] != aValues[iReg].Reg64)
915	CPUMSetGuestDR2(pVCpu, aValues[iReg].Reg64);
916	iReg++;
917	if (pVCpu->cpum.GstCtx.dr[3] != aValues[iReg].Reg64)
918	CPUMSetGuestDR3(pVCpu, aValues[iReg].Reg64);
919	iReg++;
920	}
921	if (fWhat & CPUMCTX_EXTRN_DR6)
922	{
923	Assert(aenmNames[iReg] == WHvX64RegisterDr6);
924	if (pVCpu->cpum.GstCtx.dr[6] != aValues[iReg].Reg64)
925	CPUMSetGuestDR6(pVCpu, aValues[iReg].Reg64);
926	iReg++;
927	}
928
929	/* Floating point state. */
930	if (fWhat & CPUMCTX_EXTRN_X87)
931	{
932	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[1], WHvX64RegisterFpMmx0);
933	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[1], WHvX64RegisterFpMmx1);
934	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[1], WHvX64RegisterFpMmx2);
935	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[1], WHvX64RegisterFpMmx3);
936	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[1], WHvX64RegisterFpMmx4);
937	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[1], WHvX64RegisterFpMmx5);
938	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[1], WHvX64RegisterFpMmx6);
939	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[1], WHvX64RegisterFpMmx7);
940
941	Assert(aenmNames[iReg] == WHvX64RegisterFpControlStatus);
942	pVCpu->cpum.GstCtx.XState.x87.FCW = aValues[iReg].FpControlStatus.FpControl;
943	pVCpu->cpum.GstCtx.XState.x87.FSW = aValues[iReg].FpControlStatus.FpStatus;
944	pVCpu->cpum.GstCtx.XState.x87.FTW = aValues[iReg].FpControlStatus.FpTag
945	/\| (aValues[iReg].FpControlStatus.Reserved << 8)/;
946	pVCpu->cpum.GstCtx.XState.x87.FOP = aValues[iReg].FpControlStatus.LastFpOp;
947	pVCpu->cpum.GstCtx.XState.x87.FPUIP = (uint32_t)aValues[iReg].FpControlStatus.LastFpRip;
948	pVCpu->cpum.GstCtx.XState.x87.CS = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 32);
949	pVCpu->cpum.GstCtx.XState.x87.Rsrvd1 = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 48);
950	iReg++;
951	}
952
953	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
954	{
955	Assert(aenmNames[iReg] == WHvX64RegisterXmmControlStatus);
956	if (fWhat & CPUMCTX_EXTRN_X87)
957	{
958	pVCpu->cpum.GstCtx.XState.x87.FPUDP = (uint32_t)aValues[iReg].XmmControlStatus.LastFpRdp;
959	pVCpu->cpum.GstCtx.XState.x87.DS = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 32);
960	pVCpu->cpum.GstCtx.XState.x87.Rsrvd2 = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 48);
961	}
962	pVCpu->cpum.GstCtx.XState.x87.MXCSR = aValues[iReg].XmmControlStatus.XmmStatusControl;
963	pVCpu->cpum.GstCtx.XState.x87.MXCSR_MASK = aValues[iReg].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
964	iReg++;
965	}
966
967	/* Vector state. */
968	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
969	{
970	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Hi, WHvX64RegisterXmm0);
971	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Hi, WHvX64RegisterXmm1);
972	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Hi, WHvX64RegisterXmm2);
973	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Hi, WHvX64RegisterXmm3);
974	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Hi, WHvX64RegisterXmm4);
975	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Hi, WHvX64RegisterXmm5);
976	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Hi, WHvX64RegisterXmm6);
977	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Hi, WHvX64RegisterXmm7);
978	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Hi, WHvX64RegisterXmm8);
979	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Hi, WHvX64RegisterXmm9);
980	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Hi, WHvX64RegisterXmm10);
981	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Hi, WHvX64RegisterXmm11);
982	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Hi, WHvX64RegisterXmm12);
983	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Hi, WHvX64RegisterXmm13);
984	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Hi, WHvX64RegisterXmm14);
985	GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Hi, WHvX64RegisterXmm15);
986	}
987
988	/* MSRs */
989	// WHvX64RegisterTsc - don't touch
990	if (fWhat & CPUMCTX_EXTRN_EFER)
991	{
992	Assert(aenmNames[iReg] == WHvX64RegisterEfer);
993	if (aValues[iReg].Reg64 != pVCpu->cpum.GstCtx.msrEFER)
994	{
995	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, aValues[iReg].Reg64));
996	if ((aValues[iReg].Reg64 ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
997	PGMNotifyNxeChanged(pVCpu, RT_BOOL(aValues[iReg].Reg64 & MSR_K6_EFER_NXE));
998	pVCpu->cpum.GstCtx.msrEFER = aValues[iReg].Reg64;
999	fMaybeChangedMode = true;
1000	}
1001	iReg++;
1002	}
1003	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1004	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrKERNELGSBASE, WHvX64RegisterKernelGsBase, "MSR KERNEL_GS_BASE");
1005	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1006	{
1007	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.cs, WHvX64RegisterSysenterCs, "MSR SYSENTER.CS");
1008	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.eip, WHvX64RegisterSysenterEip, "MSR SYSENTER.EIP");
1009	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.esp, WHvX64RegisterSysenterEsp, "MSR SYSENTER.ESP");
1010	}
1011	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1012	{
1013	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSTAR, WHvX64RegisterStar, "MSR STAR");
1014	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrLSTAR, WHvX64RegisterLstar, "MSR LSTAR");
1015	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrCSTAR, WHvX64RegisterCstar, "MSR CSTAR");
1016	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSFMASK, WHvX64RegisterSfmask, "MSR SFMASK");
1017	}
1018	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
1019	{
1020	PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1021	if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
1022	GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX");
1023	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1024	{
1025	Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
1026	const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
1027	if (aValues[iReg].Reg64 != uOldBase)
1028	{
1029	Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
1030	pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
1031	int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
1032	AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
1033	}
1034	iReg++;
1035
1036	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
1037	#if 0 /def LOG_ENABLED/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
1038	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
1039	#endif
1040	GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType, WHvX64RegisterMsrMtrrDefType, "MSR MTRR_DEF_TYPE");
1041	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
1042	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
1043	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
1044	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000, WHvX64RegisterMsrMtrrFix4kC0000, "MSR MTRR_FIX_4K_C0000");
1045	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000, WHvX64RegisterMsrMtrrFix4kC8000, "MSR MTRR_FIX_4K_C8000");
1046	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000, WHvX64RegisterMsrMtrrFix4kD0000, "MSR MTRR_FIX_4K_D0000");
1047	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000, WHvX64RegisterMsrMtrrFix4kD8000, "MSR MTRR_FIX_4K_D8000");
1048	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000, WHvX64RegisterMsrMtrrFix4kE0000, "MSR MTRR_FIX_4K_E0000");
1049	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000, WHvX64RegisterMsrMtrrFix4kE8000, "MSR MTRR_FIX_4K_E8000");
1050	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000, WHvX64RegisterMsrMtrrFix4kF0000, "MSR MTRR_FIX_4K_F0000");
1051	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000, WHvX64RegisterMsrMtrrFix4kF8000, "MSR MTRR_FIX_4K_F8000");
1052	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
1053	}
1054	}
1055
1056	/* Interruptibility. */
1057	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
1058	{
1059	Assert(aenmNames[iReg] == WHvRegisterInterruptState);
1060	Assert(aenmNames[iReg + 1] == WHvX64RegisterRip);
1061
1062	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_INT))
1063	{
1064	pVCpu->nem.s.fLastInterruptShadow = aValues[iReg].InterruptState.InterruptShadow;
1065	if (aValues[iReg].InterruptState.InterruptShadow)
1066	EMSetInhibitInterruptsPC(pVCpu, aValues[iReg + 1].Reg64);
1067	else
1068	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1069	}
1070
1071	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_NMI))
1072	{
1073	if (aValues[iReg].InterruptState.NmiMasked)
1074	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
1075	else
1076	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
1077	}
1078
1079	fWhat \|= CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI;
1080	iReg += 2;
1081	}
1082
1083	/* Event injection. */
1084	/// @todo WHvRegisterPendingInterruption
1085	Assert(aenmNames[iReg] == WHvRegisterPendingInterruption);
1086	if (aValues[iReg].PendingInterruption.InterruptionPending)
1087	{
1088	Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
1089	aValues[iReg].PendingInterruption.InterruptionType, aValues[iReg].PendingInterruption.InterruptionVector,
1090	aValues[iReg].PendingInterruption.DeliverErrorCode, aValues[iReg].PendingInterruption.ErrorCode,
1091	aValues[iReg].PendingInterruption.InstructionLength, aValues[iReg].PendingInterruption.NestedEvent));
1092	AssertMsg((aValues[iReg].PendingInterruption.AsUINT64 & UINT64_C(0xfc00)) == 0,
1093	("%#RX64\n", aValues[iReg].PendingInterruption.AsUINT64));
1094	}
1095
1096	/// @todo WHvRegisterPendingEvent0 (renamed to WHvRegisterPendingEvent).
1097
1098	/* Almost done, just update extrn flags and maybe change PGM mode. */
1099	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1100	if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
1101	pVCpu->cpum.GstCtx.fExtrn = 0;
1102
1103	/* Typical. */
1104	if (!fMaybeChangedMode && !fUpdateCr3)
1105	return VINF_SUCCESS;
1106
1107	/*
1108	* Slow.
1109	*/
1110	if (fMaybeChangedMode)
1111	{
1112	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
1113	false /* fForce */);
1114	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1115	}
1116
1117	if (fUpdateCr3)
1118	{
1119	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3, false /fPdpesMapped/);
1120	if (rc == VINF_SUCCESS)
1121	{ /* likely */ }
1122	else
1123	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1124	}
1125
1126	return VINF_SUCCESS;
1127	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1128	}
1129
1130	#endif /* !IN_RING0 */
1131
1132
1133	/**
1134	* Interface for importing state on demand (used by IEM).
1135	*
1136	* @returns VBox status code.
1137	* @param pVCpu The cross context CPU structure.
1138	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1139	*/
1140	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1141	{
1142	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
1143
1144	#ifdef IN_RING0
1145	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1146	return nemR0WinImportState(pVCpu->pGVM, pVCpu, &pVCpu->cpum.GstCtx, fWhat, true /fCanUpdateCr3/);
1147	# else
1148	RT_NOREF(pVCpu, fWhat);
1149	return VERR_NOT_IMPLEMENTED;
1150	# endif
1151	#else
1152	return nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
1153	#endif
1154	}
1155
1156
1157	/**
1158	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
1159	*
1160	* @returns VBox status code.
1161	* @param pVCpu The cross context CPU structure.
1162	* @param pcTicks Where to return the CPU tick count.
1163	* @param puAux Where to return the TSC_AUX register value.
1164	*/
1165	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
1166	{
1167	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
1168
1169	#ifdef IN_RING3
1170	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1171	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1172	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1173
1174	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1175	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
1176	if (pVM->nem.s.fUseRing0Runloop)
1177	# endif
1178	{
1179	/* Call ring-0 and get the values. */
1180	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_QUERY_CPU_TICK, 0, NULL);
1181	AssertLogRelRCReturn(rc, rc);
1182	*pcTicks = pVCpu->nem.s.Hypercall.QueryCpuTick.cTicks;
1183	if (puAux)
1184	*puAux = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX
1185	? pVCpu->nem.s.Hypercall.QueryCpuTick.uAux : CPUMGetGuestTscAux(pVCpu);
1186	return VINF_SUCCESS;
1187	}
1188	# endif
1189	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1190	/* Call the offical API. */
1191	WHV_REGISTER_NAME aenmNames[2] = { WHvX64RegisterTsc, WHvX64RegisterTscAux };
1192	WHV_REGISTER_VALUE aValues[2] = { {0, 0}, {0, 0} };
1193	Assert(RT_ELEMENTS(aenmNames) == RT_ELEMENTS(aValues));
1194	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, 2, aValues);
1195	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1196	("WHvGetVirtualProcessorRegisters(%p, %u,{tsc,tsc_aux},2,) -> %Rhrc (Last=%#x/%u)\n",
1197	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1198	, VERR_NEM_GET_REGISTERS_FAILED);
1199	*pcTicks = aValues[0].Reg64;
1200	if (puAux)
1201	*pcTicks = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX ? aValues[0].Reg64 : CPUMGetGuestTscAux(pVCpu);
1202	return VINF_SUCCESS;
1203	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1204	#else /* IN_RING0 */
1205	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1206	int rc = nemR0WinQueryCpuTick(pVCpu->pGVM, pVCpu, pcTicks, puAux);
1207	if (RT_SUCCESS(rc) && puAux && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX))
1208	*puAux = CPUMGetGuestTscAux(pVCpu);
1209	return rc;
1210	# else
1211	RT_NOREF(pVCpu, pcTicks, puAux);
1212	return VERR_NOT_IMPLEMENTED;
1213	# endif
1214	#endif /* IN_RING0 */
1215	}
1216
1217
1218	/**
1219	* Resumes CPU clock (TSC) on all virtual CPUs.
1220	*
1221	* This is called by TM when the VM is started, restored, resumed or similar.
1222	*
1223	* @returns VBox status code.
1224	* @param pVM The cross context VM structure.
1225	* @param pVCpu The cross context CPU structure of the calling EMT.
1226	* @param uPausedTscValue The TSC value at the time of pausing.
1227	*/
1228	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
1229	{
1230	#ifdef IN_RING0
1231	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1232	return nemR0WinResumeCpuTickOnAll(pVM, pVCpu, uPausedTscValue);
1233	# else
1234	RT_NOREF(pVM, pVCpu, uPausedTscValue);
1235	return VERR_NOT_IMPLEMENTED;
1236	# endif
1237	#else /* IN_RING3 */
1238	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1239	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1240
1241	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1242	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
1243	if (pVM->nem.s.fUseRing0Runloop)
1244	# endif
1245	{
1246	/* Call ring-0 and do it all there. */
1247	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL, uPausedTscValue, NULL);
1248	}
1249	# endif
1250	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1251	/*
1252	* Call the offical API to do the job.
1253	*/
1254	if (pVM->cCpus > 1)
1255	RTThreadYield(); /* Try decrease the chance that we get rescheduled in the middle. */
1256
1257	/* Start with the first CPU. */
1258	WHV_REGISTER_NAME enmName = WHvX64RegisterTsc;
1259	WHV_REGISTER_VALUE Value = {0, 0};
1260	Value.Reg64 = uPausedTscValue;
1261	uint64_t const uFirstTsc = ASMReadTSC();
1262	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, 0 /iCpu/, &enmName, 1, &Value);
1263	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1264	("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64) -> %Rhrc (Last=%#x/%u)\n",
1265	pVM->nem.s.hPartition, uPausedTscValue, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1266	, VERR_NEM_SET_TSC);
1267
1268	/* Do the other CPUs, adjusting for elapsed TSC and keeping finger crossed
1269	that we don't introduce too much drift here. */
1270	for (VMCPUID iCpu = 1; iCpu < pVM->cCpus; iCpu++)
1271	{
1272	Assert(enmName == WHvX64RegisterTsc);
1273	const uint64_t offDelta = (ASMReadTSC() - uFirstTsc);
1274	Value.Reg64 = uPausedTscValue + offDelta;
1275	hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, iCpu, &enmName, 1, &Value);
1276	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1277	("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64 + %#RX64) -> %Rhrc (Last=%#x/%u)\n",
1278	pVM->nem.s.hPartition, iCpu, uPausedTscValue, offDelta, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1279	, VERR_NEM_SET_TSC);
1280	}
1281
1282	return VINF_SUCCESS;
1283	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1284	#endif /* IN_RING3 */
1285	}
1286
1287	#ifdef NEMWIN_NEED_GET_REGISTER
1288	# if defined(IN_RING0) \|\| defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
1289	/** Worker for assertion macro. */
1290	NEM_TMPL_STATIC int nemHCWinGetRegister(PVMCPUCC pVCpu, PGVMCPU pGVCpu, uint32_t enmReg, HV_REGISTER_VALUE *pRetValue)
1291	{
1292	RT_ZERO(*pRetValue);
1293	# ifdef IN_RING3
1294	RT_NOREF(pVCpu, pGVCpu, enmReg);
1295	return VERR_NOT_IMPLEMENTED;
1296	# else
1297	NOREF(pVCpu);
1298
1299	/*
1300	* Hypercall parameters.
1301	*/
1302	HV_INPUT_GET_VP_REGISTERS pInput = (HV_INPUT_GET_VP_REGISTERS )pGVCpu->nem.s.HypercallData.pbPage;
1303	AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
1304	AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API);
1305
1306	pInput->PartitionId = pVCpu->pGVM->nemr0.s.idHvPartition;
1307	pInput->VpIndex = pVCpu->idCpu;
1308	pInput->fFlags = 0;
1309	pInput->Names[0] = (HV_REGISTER_NAME)enmReg;
1310
1311	size_t const cbInput = RT_ALIGN_Z(RT_UOFFSETOF(HV_INPUT_GET_VP_REGISTERS, Names[1]), 32);
1312	HV_REGISTER_VALUE paValues = (HV_REGISTER_VALUE )((uint8_t *)pInput + cbInput);
1313	RT_BZERO(paValues, sizeof(paValues[0]) * 1);
1314
1315	/*
1316	* Make the hypercall and copy out the value.
1317	*/
1318	uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallGetVpRegisters, 1),
1319	pGVCpu->nem.s.HypercallData.HCPhysPage,
1320	pGVCpu->nem.s.HypercallData.HCPhysPage + cbInput);
1321	AssertLogRelMsgReturn(uResult == HV_MAKE_CALL_REP_RET(1), ("uResult=%RX64 cRegs=%#x\n", uResult, 1),
1322	VERR_NEM_GET_REGISTERS_FAILED);
1323
1324	*pRetValue = paValues[0];
1325	return VINF_SUCCESS;
1326	# endif
1327	}
1328	# else
1329	/** Worker for assertion macro. */
1330	NEM_TMPL_STATIC int nemR3WinGetRegister(PVMCPUCC a_pVCpu, uint32_t a_enmReg, WHV_REGISTER_VALUE pValue)
1331	{
1332	RT_ZERO(*pRetValue);
1333	RT_NOREF(pVCpu, pGVCpu, enmReg);
1334	return VERR_NOT_IMPLEMENTED;
1335	}
1336	# endif
1337	#endif
1338
1339
1340	#ifdef LOG_ENABLED
1341	/**
1342	* Get the virtual processor running status.
1343	*/
1344	DECLINLINE(VID_PROCESSOR_STATUS) nemHCWinCpuGetRunningStatus(PVMCPUCC pVCpu)
1345	{
1346	# ifdef IN_RING0
1347	NOREF(pVCpu);
1348	return VidProcessorStatusUndefined;
1349	# else
1350	RTERRVARS Saved;
1351	RTErrVarsSave(&Saved);
1352
1353	/*
1354	* This API is disabled in release builds, it seems. On build 17101 it requires
1355	* the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00
1356	*/
1357	VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined;
1358	NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus);
1359	AssertRC(rcNt);
1360
1361	RTErrVarsRestore(&Saved);
1362	return enmCpuStatus;
1363	# endif
1364	}
1365	#endif /* LOG_ENABLED */
1366
1367
1368	#if defined(NEM_WIN_USE_OUR_OWN_RUN_API) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1369	# ifdef IN_RING3 /* hopefully not needed in ring-0, as we'd need KTHREADs and KeAlertThread. */
1370	/**
1371	* Our own WHvCancelRunVirtualProcessor that can later be moved to ring-0.
1372	*
1373	* This is an experiment only.
1374	*
1375	* @returns VBox status code.
1376	* @param pVM The cross context VM structure.
1377	* @param pVCpu The cross context virtual CPU structure of the
1378	* calling EMT.
1379	*/
1380	NEM_TMPL_STATIC int nemHCWinCancelRunVirtualProcessor(PVMCC pVM, PVMCPUCC pVCpu)
1381	{
1382	/*
1383	* Work the state.
1384	*
1385	* From the looks of things, we should let the EMT call VidStopVirtualProcessor.
1386	* So, we just need to modify the state and kick the EMT if it's waiting on
1387	* messages. For the latter we use QueueUserAPC / KeAlterThread.
1388	*/
1389	for (;;)
1390	{
1391	VMCPUSTATE enmState = VMCPU_GET_STATE(pVCpu);
1392	switch (enmState)
1393	{
1394	case VMCPUSTATE_STARTED_EXEC_NEM:
1395	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM))
1396	{
1397	DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM -> CANCELED");
1398	Log8(("nemHCWinCancelRunVirtualProcessor: Switched %u to canceled state\n", pVCpu->idCpu));
1399	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelChangedState);
1400	return VINF_SUCCESS;
1401	}
1402	break;
1403
1404	case VMCPUSTATE_STARTED_EXEC_NEM_WAIT:
1405	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT))
1406	{
1407	DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM_WAIT -> CANCELED");
1408	# ifdef IN_RING0
1409	NTSTATUS rcNt = KeAlertThread(??);
1410	DBGFTRACE_CUSTOM(pVM, "KeAlertThread -> %#x", rcNt);
1411	# else
1412	NTSTATUS rcNt = NtAlertThread(pVCpu->nem.s.hNativeThreadHandle);
1413	DBGFTRACE_CUSTOM(pVM, "NtAlertThread -> %#x", rcNt);
1414	# endif
1415	Log8(("nemHCWinCancelRunVirtualProcessor: Alerted %u: %#x\n", pVCpu->idCpu, rcNt));
1416	Assert(rcNt == STATUS_SUCCESS);
1417	if (NT_SUCCESS(rcNt))
1418	{
1419	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelAlertedThread);
1420	return VINF_SUCCESS;
1421	}
1422	AssertLogRelMsgFailedReturn(("NtAlertThread failed: %#x\n", rcNt), RTErrConvertFromNtStatus(rcNt));
1423	}
1424	break;
1425
1426	default:
1427	return VINF_SUCCESS;
1428	}
1429
1430	ASMNopPause();
1431	RT_NOREF(pVM);
1432	}
1433	}
1434	# endif /* IN_RING3 */
1435	#endif /* NEM_WIN_USE_OUR_OWN_RUN_API \|\| NEM_WIN_WITH_RING0_RUNLOOP */
1436
1437
1438	#ifdef LOG_ENABLED
1439	/**
1440	* Logs the current CPU state.
1441	*/
1442	NEM_TMPL_STATIC void nemHCWinLogState(PVMCC pVM, PVMCPUCC pVCpu)
1443	{
1444	if (LogIs3Enabled())
1445	{
1446	# if 0 // def IN_RING3 - causes lazy state import assertions all over CPUM.
1447	char szRegs[4096];
1448	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
1449	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
1450	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
1451	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
1452	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
1453	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
1454	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
1455	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
1456	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
1457	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
1458	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
1459	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
1460	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
1461	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
1462	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
1463	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
1464	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
1465	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
1466	" efer=%016VR{efer}\n"
1467	" pat=%016VR{pat}\n"
1468	" sf_mask=%016VR{sf_mask}\n"
1469	"krnl_gs_base=%016VR{krnl_gs_base}\n"
1470	" lstar=%016VR{lstar}\n"
1471	" star=%016VR{star} cstar=%016VR{cstar}\n"
1472	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
1473	);
1474
1475	char szInstr[256];
1476	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
1477	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
1478	szInstr, sizeof(szInstr), NULL);
1479	Log3(("%s%s\n", szRegs, szInstr));
1480	# else
1481	/** @todo stat logging in ring-0 */
1482	RT_NOREF(pVM, pVCpu);
1483	# endif
1484	}
1485	}
1486	#endif /* LOG_ENABLED */
1487
1488
1489	/** Macro used by nemHCWinExecStateToLogStr and nemR3WinExecStateToLogStr. */
1490	#define SWITCH_IT(a_szPrefix) \
1491	do \
1492	switch (u)\
1493	{ \
1494	case 0x00: return a_szPrefix ""; \
1495	case 0x01: return a_szPrefix ",Pnd"; \
1496	case 0x02: return a_szPrefix ",Dbg"; \
1497	case 0x03: return a_szPrefix ",Pnd,Dbg"; \
1498	case 0x04: return a_szPrefix ",Shw"; \
1499	case 0x05: return a_szPrefix ",Pnd,Shw"; \
1500	case 0x06: return a_szPrefix ",Shw,Dbg"; \
1501	case 0x07: return a_szPrefix ",Pnd,Shw,Dbg"; \
1502	default: AssertFailedReturn("WTF?"); \
1503	} \
1504	while (0)
1505
1506	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1507	/**
1508	* Translates the execution stat bitfield into a short log string, VID version.
1509	*
1510	* @returns Read-only log string.
1511	* @param pMsgHdr The header which state to summarize.
1512	*/
1513	static const char nemHCWinExecStateToLogStr(HV_X64_INTERCEPT_MESSAGE_HEADER const pMsgHdr)
1514	{
1515	unsigned u = (unsigned)pMsgHdr->ExecutionState.InterruptionPending
1516	\| ((unsigned)pMsgHdr->ExecutionState.DebugActive << 1)
1517	\| ((unsigned)pMsgHdr->ExecutionState.InterruptShadow << 2);
1518	if (pMsgHdr->ExecutionState.EferLma)
1519	SWITCH_IT("LM");
1520	else if (pMsgHdr->ExecutionState.Cr0Pe)
1521	SWITCH_IT("PM");
1522	else
1523	SWITCH_IT("RM");
1524	}
1525	#elif defined(IN_RING3)
1526	/**
1527	* Translates the execution stat bitfield into a short log string, WinHv version.
1528	*
1529	* @returns Read-only log string.
1530	* @param pExitCtx The exit context which state to summarize.
1531	*/
1532	static const char nemR3WinExecStateToLogStr(WHV_VP_EXIT_CONTEXT const pExitCtx)
1533	{
1534	unsigned u = (unsigned)pExitCtx->ExecutionState.InterruptionPending
1535	\| ((unsigned)pExitCtx->ExecutionState.DebugActive << 1)
1536	\| ((unsigned)pExitCtx->ExecutionState.InterruptShadow << 2);
1537	if (pExitCtx->ExecutionState.EferLma)
1538	SWITCH_IT("LM");
1539	else if (pExitCtx->ExecutionState.Cr0Pe)
1540	SWITCH_IT("PM");
1541	else
1542	SWITCH_IT("RM");
1543	}
1544	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1545	#undef SWITCH_IT
1546
1547
1548	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1549	/**
1550	* Advances the guest RIP and clear EFLAGS.RF, VID version.
1551	*
1552	* This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
1553	*
1554	* @param pVCpu The cross context virtual CPU structure.
1555	* @param pExitCtx The exit context.
1556	* @param cbMinInstr The minimum instruction length, or 1 if not unknown.
1557	*/
1558	DECLINLINE(void)
1559	nemHCWinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr, uint8_t cbMinInstr)
1560	{
1561	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
1562
1563	/* Advance the RIP. */
1564	Assert(pMsgHdr->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
1565	pVCpu->cpum.GstCtx.rip += pMsgHdr->InstructionLength;
1566	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
1567
1568	/* Update interrupt inhibition. */
1569	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1570	{ /* likely */ }
1571	else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
1572	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1573	}
1574	#elif defined(IN_RING3)
1575	/**
1576	* Advances the guest RIP and clear EFLAGS.RF, WinHv version.
1577	*
1578	* This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
1579	*
1580	* @param pVCpu The cross context virtual CPU structure.
1581	* @param pExitCtx The exit context.
1582	* @param cbMinInstr The minimum instruction length, or 1 if not unknown.
1583	*/
1584	DECLINLINE(void) nemR3WinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx, uint8_t cbMinInstr)
1585	{
1586	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
1587
1588	/* Advance the RIP. */
1589	Assert(pExitCtx->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
1590	pVCpu->cpum.GstCtx.rip += pExitCtx->InstructionLength;
1591	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
1592
1593	/* Update interrupt inhibition. */
1594	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1595	{ /* likely */ }
1596	else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
1597	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1598	}
1599	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1600
1601	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING3)
1602
1603	NEM_TMPL_STATIC DECLCALLBACK(int)
1604	nemHCWinUnmapOnePageCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint8_t pu2NemState, void pvUser)
1605	{
1606	RT_NOREF_PV(pvUser);
1607	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1608	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1609	AssertRC(rc);
1610	if (RT_SUCCESS(rc))
1611	# else
1612	RT_NOREF_PV(pVCpu);
1613	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
1614	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1615	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
1616	if (SUCCEEDED(hrc))
1617	# endif
1618	{
1619	Log5(("NEM GPA unmap all: %RGp (cMappedPages=%u)\n", GCPhys, pVM->nem.s.cMappedPages - 1));
1620	*pu2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1621	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
1622	}
1623	else
1624	{
1625	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1626	LogRel(("nemHCWinUnmapOnePageCallback: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
1627	# else
1628	LogRel(("nemHCWinUnmapOnePageCallback: GCPhys=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1629	GCPhys, g_apszPageStates[*pu2NemState], hrc, hrc, RTNtLastStatusValue(),
1630	RTNtLastErrorValue(), pVM->nem.s.cMappedPages));
1631	# endif
1632	*pu2NemState = NEM_WIN_PAGE_STATE_NOT_SET;
1633	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
1634	}
1635	if (pVM->nem.s.cMappedPages > 0)
1636	ASMAtomicDecU32(&pVM->nem.s.cMappedPages);
1637	return VINF_SUCCESS;
1638	}
1639
1640
1641	/**
1642	* State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
1643	* and nemHCWinHandleMemoryAccessPageCheckerCallback.
1644	*/
1645	typedef struct NEMHCWINHMACPCCSTATE
1646	{
1647	/** Input: Write access. */
1648	bool fWriteAccess;
1649	/** Output: Set if we did something. */
1650	bool fDidSomething;
1651	/** Output: Set it we should resume. */
1652	bool fCanResume;
1653	} NEMHCWINHMACPCCSTATE;
1654
1655	/**
1656	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1657	* Worker for nemR3WinHandleMemoryAccess; pvUser points to a
1658	* NEMHCWINHMACPCCSTATE structure. }
1659	*/
1660	NEM_TMPL_STATIC DECLCALLBACK(int)
1661	nemHCWinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1662	{
1663	NEMHCWINHMACPCCSTATE pState = (NEMHCWINHMACPCCSTATE )pvUser;
1664	pState->fDidSomething = false;
1665	pState->fCanResume = false;
1666
1667	/* If A20 is disabled, we may need to make another query on the masked
1668	page to get the correct protection information. */
1669	uint8_t u2State = pInfo->u2NemState;
1670	RTGCPHYS GCPhysSrc;
1671	# ifdef NEM_WIN_WITH_A20
1672	if ( pVM->nem.s.fA20Enabled
1673	\|\| !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
1674	# endif
1675	GCPhysSrc = GCPhys;
1676	# ifdef NEM_WIN_WITH_A20
1677	else
1678	{
1679	GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20);
1680	PGMPHYSNEMPAGEINFO Info2;
1681	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL);
1682	AssertRCReturn(rc, rc);
1683
1684	*pInfo = Info2;
1685	pInfo->u2NemState = u2State;
1686	}
1687	# endif
1688
1689	/*
1690	* Consolidate current page state with actual page protection and access type.
1691	* We don't really consider downgrades here, as they shouldn't happen.
1692	*/
1693	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1694	/** @todo Someone at microsoft please explain:
1695	* I'm not sure WTF was going on, but I ended up in a loop if I remapped a
1696	* readonly page as writable (unmap, then map again). Specifically, this was an
1697	* issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1. So, in
1698	* a hope to work around that we no longer pre-map anything, just unmap stuff
1699	* and do it lazily here. And here we will first unmap, restart, and then remap
1700	* with new protection or backing.
1701	*/
1702	# endif
1703	int rc;
1704	switch (u2State)
1705	{
1706	case NEM_WIN_PAGE_STATE_UNMAPPED:
1707	case NEM_WIN_PAGE_STATE_NOT_SET:
1708	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1709	{
1710	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1711	return VINF_SUCCESS;
1712	}
1713
1714	/* Don't bother remapping it if it's a write request to a non-writable page. */
1715	if ( pState->fWriteAccess
1716	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1717	{
1718	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1719	return VINF_SUCCESS;
1720	}
1721
1722	/* Map the page. */
1723	rc = nemHCNativeSetPhysPage(pVM,
1724	pVCpu,
1725	GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1726	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1727	pInfo->fNemProt,
1728	&u2State,
1729	true /fBackingState/);
1730	pInfo->u2NemState = u2State;
1731	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1732	GCPhys, g_apszPageStates[u2State], rc));
1733	pState->fDidSomething = true;
1734	pState->fCanResume = true;
1735	return rc;
1736
1737	case NEM_WIN_PAGE_STATE_READABLE:
1738	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1739	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1740	{
1741	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1742	return VINF_SUCCESS;
1743	}
1744
1745	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1746	/* Upgrade page to writable. */
1747	/** @todo test this*/
1748	if ( (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1749	&& pState->fWriteAccess)
1750	{
1751	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhys,
1752	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
1753	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
1754	AssertRC(rc);
1755	if (RT_SUCCESS(rc))
1756	{
1757	STAM_REL_COUNTER_INC(&pVM->nem.s.StatRemapPage);
1758	pInfo->u2NemState = NEM_WIN_PAGE_STATE_WRITABLE;
1759	pState->fDidSomething = true;
1760	pState->fCanResume = true;
1761	Log5(("NEM GPA write-upgrade/exit: %RGp (was %s, cMappedPages=%u)\n",
1762	GCPhys, g_apszPageStates[u2State], pVM->nem.s.cMappedPages));
1763	}
1764	else
1765	STAM_REL_COUNTER_INC(&pVM->nem.s.StatRemapPageFailed);
1766	}
1767	else
1768	{
1769	/* Need to emulate the acces. */
1770	AssertBreak(pInfo->fNemProt != NEM_PAGE_PROT_NONE); /* There should be no downgrades. */
1771	rc = VINF_SUCCESS;
1772	}
1773	return rc;
1774	# else
1775	break;
1776	# endif
1777
1778	case NEM_WIN_PAGE_STATE_WRITABLE:
1779	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1780	{
1781	if (pInfo->u2OldNemState == NEM_WIN_PAGE_STATE_WRITABLE)
1782	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
1783	else
1784	{
1785	pState->fCanResume = true;
1786	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
1787	GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
1788	}
1789	return VINF_SUCCESS;
1790	}
1791	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1792	AssertFailed(); /* There should be no downgrades. */
1793	# endif
1794	break;
1795
1796	default:
1797	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1798	}
1799
1800	/*
1801	* Unmap and restart the instruction.
1802	* If this fails, which it does every so often, just unmap everything for now.
1803	*/
1804	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1805	rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1806	AssertRC(rc);
1807	if (RT_SUCCESS(rc))
1808	# else
1809	/** @todo figure out whether we mess up the state or if it's WHv. */
1810	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
1811	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1812	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
1813	if (SUCCEEDED(hrc))
1814	# endif
1815	{
1816	pState->fDidSomething = true;
1817	pState->fCanResume = true;
1818	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1819	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
1820	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
1821	Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
1822	return VINF_SUCCESS;
1823	}
1824	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
1825	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1826	LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhys, rc));
1827	return rc;
1828	# elif defined(VBOX_WITH_PGM_NEM_MODE)
1829	LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x)\n",
1830	GCPhys, g_apszPageStates[u2State], hrc, hrc));
1831	return VERR_NEM_UNMAP_PAGES_FAILED;
1832	# else
1833	LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1834	GCPhys, g_apszPageStates[u2State], hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue(),
1835	pVM->nem.s.cMappedPages));
1836
1837	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemHCWinUnmapOnePageCallback, NULL);
1838	Log(("nemHCWinHandleMemoryAccessPageCheckerCallback: Unmapped all (cMappedPages=%u)\n", pVM->nem.s.cMappedPages));
1839	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapAllPages);
1840
1841	pState->fDidSomething = true;
1842	pState->fCanResume = true;
1843	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1844	return VINF_SUCCESS;
1845	# endif
1846	}
1847
1848	#endif /* defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING3) */
1849
1850
1851	#if defined(IN_RING0) && defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API)
1852	/**
1853	* Wrapper around nemR0WinImportState that converts VERR_NEM_FLUSH_TLB
1854	* into informational status codes and logs+asserts statuses.
1855	*
1856	* @returns VBox strict status code.
1857	* @param pGVM The global (ring-0) VM structure.
1858	* @param pGVCpu The global (ring-0) per CPU structure.
1859	* @param fWhat What to import.
1860	* @param pszCaller Who is doing the importing.
1861	*/
1862	DECLINLINE(VBOXSTRICTRC) nemR0WinImportStateStrict(PGVM pGVM, PGVMCPU pGVCpu, uint64_t fWhat, const char *pszCaller)
1863	{
1864	int rc = nemR0WinImportState(pGVM, pGVCpu, &pGVCpu->cpum.GstCtx, fWhat, true /fCanUpdateCr3/);
1865	if (RT_SUCCESS(rc))
1866	{
1867	Assert(rc == VINF_SUCCESS);
1868	return VINF_SUCCESS;
1869	}
1870
1871	if (rc == VERR_NEM_FLUSH_TLB)
1872	{
1873	Log4(("%s/%u: nemR0WinImportState -> %Rrc\n", pszCaller, pGVCpu->idCpu, -rc));
1874	return -rc;
1875	}
1876	RT_NOREF(pszCaller);
1877	AssertMsgFailedReturn(("%s/%u: nemR0WinImportState failed: %Rrc\n", pszCaller, pGVCpu->idCpu, rc), rc);
1878	}
1879	#endif /* IN_RING0 && NEM_WIN_TEMPLATE_MODE_OWN_RUN_API*/
1880
1881	#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3)
1882	/**
1883	* Wrapper around nemR0WinImportStateStrict and nemHCWinCopyStateFromHyperV.
1884	*
1885	* Unlike the wrapped APIs, this checks whether it's necessary.
1886	*
1887	* @returns VBox strict status code.
1888	* @param pVCpu The cross context per CPU structure.
1889	* @param fWhat What to import.
1890	* @param pszCaller Who is doing the importing.
1891	*/
1892	DECLINLINE(VBOXSTRICTRC) nemHCWinImportStateIfNeededStrict(PVMCPUCC pVCpu, uint64_t fWhat, const char *pszCaller)
1893	{
1894	if (pVCpu->cpum.GstCtx.fExtrn & fWhat)
1895	{
1896	# ifdef IN_RING0
1897	return nemR0WinImportStateStrict(pVCpu->pGVM, pVCpu, fWhat, pszCaller);
1898	# else
1899	RT_NOREF(pszCaller);
1900	int rc = nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
1901	AssertRCReturn(rc, rc);
1902	# endif
1903	}
1904	return VINF_SUCCESS;
1905	}
1906	#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API \|\| IN_RING3 */
1907
1908	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1909	/**
1910	* Copies register state from the X64 intercept message header.
1911	*
1912	* ASSUMES no state copied yet.
1913	*
1914	* @param pVCpu The cross context per CPU structure.
1915	* @param pHdr The X64 intercept message header.
1916	* @sa nemR3WinCopyStateFromX64Header
1917	*/
1918	DECLINLINE(void) nemHCWinCopyStateFromX64Header(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pHdr)
1919	{
1920	Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_INHIBIT_INT))
1921	== (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_INHIBIT_INT));
1922	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pHdr->CsSegment);
1923	pVCpu->cpum.GstCtx.rip = pHdr->Rip;
1924	pVCpu->cpum.GstCtx.rflags.u = pHdr->Rflags;
1925
1926	pVCpu->nem.s.fLastInterruptShadow = pHdr->ExecutionState.InterruptShadow;
1927	if (!pHdr->ExecutionState.InterruptShadow)
1928	{
1929	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1930	{ /* likely */ }
1931	else
1932	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1933	}
1934	else
1935	EMSetInhibitInterruptsPC(pVCpu, pHdr->Rip);
1936
1937	APICSetTpr(pVCpu, pHdr->Cr8 << 4);
1938
1939	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_APIC_TPR);
1940	}
1941	#elif defined(IN_RING3)
1942	/**
1943	* Copies register state from the (common) exit context.
1944	*
1945	* ASSUMES no state copied yet.
1946	*
1947	* @param pVCpu The cross context per CPU structure.
1948	* @param pExitCtx The common exit context.
1949	* @sa nemHCWinCopyStateFromX64Header
1950	*/
1951	DECLINLINE(void) nemR3WinCopyStateFromX64Header(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx)
1952	{
1953	Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_INHIBIT_INT))
1954	== (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_INHIBIT_INT));
1955	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pExitCtx->Cs);
1956	pVCpu->cpum.GstCtx.rip = pExitCtx->Rip;
1957	pVCpu->cpum.GstCtx.rflags.u = pExitCtx->Rflags;
1958
1959	pVCpu->nem.s.fLastInterruptShadow = pExitCtx->ExecutionState.InterruptShadow;
1960	if (!pExitCtx->ExecutionState.InterruptShadow)
1961	{
1962	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1963	{ /* likely */ }
1964	else
1965	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1966	}
1967	else
1968	EMSetInhibitInterruptsPC(pVCpu, pExitCtx->Rip);
1969
1970	APICSetTpr(pVCpu, pExitCtx->Cr8 << 4);
1971
1972	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_APIC_TPR);
1973	}
1974	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1975
1976
1977	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1978	/**
1979	* Deals with memory intercept message.
1980	*
1981	* @returns Strict VBox status code.
1982	* @param pVM The cross context VM structure.
1983	* @param pVCpu The cross context per CPU structure.
1984	* @param pMsg The message.
1985	* @sa nemR3WinHandleExitMemory
1986	*/
1987	NEM_TMPL_STATIC VBOXSTRICTRC
1988	nemHCWinHandleMessageMemory(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_MEMORY_INTERCEPT_MESSAGE const *pMsg)
1989	{
1990	uint64_t const uHostTsc = ASMReadTSC();
1991	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
1992	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
1993	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
1994
1995	/*
1996	* Whatever we do, we must clear pending event injection upon resume.
1997	*/
1998	if (pMsg->Header.ExecutionState.InterruptionPending)
1999	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2000
2001	# if 0 /* Experiment: 20K -> 34K exit/s. */
2002	if ( pMsg->Header.ExecutionState.EferLma
2003	&& pMsg->Header.CsSegment.Long
2004	&& pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2005	{
2006	if ( pMsg->Header.Rip - (uint64_t)0xf65a < (uint64_t)(0xf662 - 0xf65a)
2007	&& pMsg->InstructionBytes[0] == 0x89
2008	&& pMsg->InstructionBytes[1] == 0x03)
2009	{
2010	pVCpu->cpum.GstCtx.rip = pMsg->Header.Rip + 2;
2011	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
2012	AssertMsg(pMsg->Header.InstructionLength == 2, ("%#x\n", pMsg->Header.InstructionLength));
2013	//Log(("%RX64 msg:\n%.80Rhxd\n", pVCpu->cpum.GstCtx.rip, pMsg));
2014	return VINF_SUCCESS;
2015	}
2016	}
2017	# endif
2018
2019	/*
2020	* Ask PGM for information about the given GCPhys. We need to check if we're
2021	* out of sync first.
2022	*/
2023	NEMHCWINHMACPCCSTATE State = { pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE, false, false };
2024	PGMPHYSNEMPAGEINFO Info;
2025	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pMsg->GuestPhysicalAddress, State.fWriteAccess, &Info,
2026	nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
2027	if (RT_SUCCESS(rc))
2028	{
2029	if (Info.fNemProt & ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2030	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
2031	{
2032	if (State.fCanResume)
2033	{
2034	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
2035	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2036	pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2037	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2038	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2039	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
2040	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
2041	return VINF_SUCCESS;
2042	}
2043	}
2044	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
2045	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2046	pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2047	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2048	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2049	}
2050	else
2051	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
2052	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2053	pMsg->GuestPhysicalAddress, rc, State.fDidSomething ? " modified-backing" : "",
2054	g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2055
2056	/*
2057	* Emulate the memory access, either access handler or special memory.
2058	*/
2059	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2060	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2061	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2062	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2063	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
2064	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2065	VBOXSTRICTRC rcStrict;
2066	# ifdef IN_RING0
2067	rcStrict = nemR0WinImportStateStrict(pVM, pVCpu,
2068	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES, "MemExit");
2069	if (rcStrict != VINF_SUCCESS)
2070	return rcStrict;
2071	# else
2072	rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2073	AssertRCReturn(rc, rc);
2074	# endif
2075
2076	if (pMsg->Reserved1)
2077	Log(("MemExit/Reserved1=%#x\n", pMsg->Reserved1));
2078	if (pMsg->Header.ExecutionState.Reserved0 \|\| pMsg->Header.ExecutionState.Reserved1)
2079	Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pMsg->Header.ExecutionState.Reserved0, pMsg->Header.ExecutionState.Reserved1));
2080
2081	if (!pExitRec)
2082	{
2083	//if (pMsg->InstructionByteCount > 0)
2084	// Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
2085	if (pMsg->InstructionByteCount > 0)
2086	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
2087	pMsg->InstructionBytes, pMsg->InstructionByteCount);
2088	else
2089	rcStrict = IEMExecOne(pVCpu);
2090	/** @todo do we need to do anything wrt debugging here? */
2091	}
2092	else
2093	{
2094	/* Frequent access or probing. */
2095	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2096	Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2097	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2098	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2099	}
2100	return rcStrict;
2101	}
2102	#elif defined(IN_RING3)
2103	/**
2104	* Deals with memory access exits (WHvRunVpExitReasonMemoryAccess).
2105	*
2106	* @returns Strict VBox status code.
2107	* @param pVM The cross context VM structure.
2108	* @param pVCpu The cross context per CPU structure.
2109	* @param pExit The VM exit information to handle.
2110	* @sa nemHCWinHandleMessageMemory
2111	*/
2112	NEM_TMPL_STATIC VBOXSTRICTRC
2113	nemR3WinHandleExitMemory(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2114	{
2115	uint64_t const uHostTsc = ASMReadTSC();
2116	Assert(pExit->MemoryAccess.AccessInfo.AccessType != 3);
2117
2118	/*
2119	* Whatever we do, we must clear pending event injection upon resume.
2120	*/
2121	if (pExit->VpContext.ExecutionState.InterruptionPending)
2122	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2123
2124	/*
2125	* Ask PGM for information about the given GCPhys. We need to check if we're
2126	* out of sync first.
2127	*/
2128	NEMHCWINHMACPCCSTATE State = { pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite, false, false };
2129	PGMPHYSNEMPAGEINFO Info;
2130	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pExit->MemoryAccess.Gpa, State.fWriteAccess, &Info,
2131	nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
2132	if (RT_SUCCESS(rc))
2133	{
2134	if (Info.fNemProt & ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2135	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
2136	{
2137	if (State.fCanResume)
2138	{
2139	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
2140	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2141	pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2142	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2143	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2144	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
2145	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
2146	return VINF_SUCCESS;
2147	}
2148	}
2149	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
2150	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2151	pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2152	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2153	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2154	}
2155	else
2156	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
2157	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2158	pExit->MemoryAccess.Gpa, rc, State.fDidSomething ? " modified-backing" : "",
2159	g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2160
2161	/*
2162	* Emulate the memory access, either access handler or special memory.
2163	*/
2164	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2165	pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2166	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2167	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2168	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
2169	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2170	rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2171	AssertRCReturn(rc, rc);
2172	if (pExit->VpContext.ExecutionState.Reserved0 \|\| pExit->VpContext.ExecutionState.Reserved1)
2173	Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pExit->VpContext.ExecutionState.Reserved0, pExit->VpContext.ExecutionState.Reserved1));
2174
2175	VBOXSTRICTRC rcStrict;
2176	if (!pExitRec)
2177	{
2178	//if (pMsg->InstructionByteCount > 0)
2179	// Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
2180	if (pExit->MemoryAccess.InstructionByteCount > 0)
2181	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
2182	pExit->MemoryAccess.InstructionBytes, pExit->MemoryAccess.InstructionByteCount);
2183	else
2184	rcStrict = IEMExecOne(pVCpu);
2185	/** @todo do we need to do anything wrt debugging here? */
2186	}
2187	else
2188	{
2189	/* Frequent access or probing. */
2190	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2191	Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2192	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2193	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2194	}
2195	return rcStrict;
2196	}
2197	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2198
2199
2200	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2201	/**
2202	* Deals with I/O port intercept message.
2203	*
2204	* @returns Strict VBox status code.
2205	* @param pVM The cross context VM structure.
2206	* @param pVCpu The cross context per CPU structure.
2207	* @param pMsg The message.
2208	*/
2209	NEM_TMPL_STATIC VBOXSTRICTRC
2210	nemHCWinHandleMessageIoPort(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_IO_PORT_INTERCEPT_MESSAGE const *pMsg)
2211	{
2212	/*
2213	* Assert message sanity.
2214	*/
2215	Assert( pMsg->AccessInfo.AccessSize == 1
2216	\|\| pMsg->AccessInfo.AccessSize == 2
2217	\|\| pMsg->AccessInfo.AccessSize == 4);
2218	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
2219	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2220	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2221	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2222	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2223	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2224	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
2225	if (pMsg->AccessInfo.StringOp)
2226	{
2227	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterDs, pMsg->DsSegment);
2228	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterEs, pMsg->EsSegment);
2229	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
2230	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsi, pMsg->Rsi);
2231	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdi, pMsg->Rdi);
2232	}
2233
2234	/*
2235	* Whatever we do, we must clear pending event injection upon resume.
2236	*/
2237	if (pMsg->Header.ExecutionState.InterruptionPending)
2238	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2239
2240	/*
2241	* Add history first to avoid two paths doing EMHistoryExec calls.
2242	*/
2243	VBOXSTRICTRC rcStrict;
2244	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2245	!pMsg->AccessInfo.StringOp
2246	? ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2247	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
2248	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
2249	: ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2250	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
2251	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
2252	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2253	if (!pExitRec)
2254	{
2255	if (!pMsg->AccessInfo.StringOp)
2256	{
2257	/*
2258	* Simple port I/O.
2259	*/
2260	static uint32_t const s_fAndMask[8] =
2261	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
2262	uint32_t const fAndMask = s_fAndMask[pMsg->AccessInfo.AccessSize];
2263
2264	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2265	if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2266	{
2267	rcStrict = IOMIOPortWrite(pVM, pVCpu, pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize);
2268	Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2269	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2270	pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
2271	if (IOM_SUCCESS(rcStrict))
2272	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
2273	# ifdef IN_RING0
2274	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
2275	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
2276	/** @todo check for debug breakpoints */ )
2277	return EMRZSetPendingIoPortWrite(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
2278	pMsg->AccessInfo.AccessSize, (uint32_t)pMsg->Rax & fAndMask);
2279	# endif
2280	else
2281	{
2282	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2283	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2284	}
2285	}
2286	else
2287	{
2288	uint32_t uValue = 0;
2289	rcStrict = IOMIOPortRead(pVM, pVCpu, pMsg->PortNumber, &uValue, pMsg->AccessInfo.AccessSize);
2290	Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2291	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2292	pMsg->PortNumber, pMsg->AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2293	if (IOM_SUCCESS(rcStrict))
2294	{
2295	if (pMsg->AccessInfo.AccessSize != 4)
2296	pVCpu->cpum.GstCtx.rax = (pMsg->Rax & ~(uint64_t)fAndMask) \| (uValue & fAndMask);
2297	else
2298	pVCpu->cpum.GstCtx.rax = uValue;
2299	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2300	Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pMsg->Rax, pVCpu->cpum.GstCtx.rax));
2301	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
2302	}
2303	else
2304	{
2305	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2306	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2307	# ifdef IN_RING0
2308	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
2309	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
2310	/** @todo check for debug breakpoints */ )
2311	return EMRZSetPendingIoPortRead(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
2312	pMsg->AccessInfo.AccessSize);
2313	# endif
2314	}
2315	}
2316	}
2317	else
2318	{
2319	/*
2320	* String port I/O.
2321	*/
2322	/** @todo Someone at Microsoft please explain how we can get the address mode
2323	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
2324	* getting the default mode, it can always be overridden by a prefix. This
2325	* forces us to interpret the instruction from opcodes, which is suboptimal.
2326	* Both AMD-V and VT-x includes the address size in the exit info, at least on
2327	* CPUs that are reasonably new.
2328	*
2329	* Of course, it's possible this is an undocumented and we just need to do some
2330	* experiments to figure out how it's communicated. Alternatively, we can scan
2331	* the opcode bytes for possible evil prefixes.
2332	*/
2333	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2334	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2335	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2336	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
2337	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
2338	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2339	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2340	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
2341	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
2342	# ifdef IN_RING0
2343	rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
2344	if (rcStrict != VINF_SUCCESS)
2345	return rcStrict;
2346	# else
2347	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2348	AssertRCReturn(rc, rc);
2349	# endif
2350
2351	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
2352	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2353	pMsg->AccessInfo.RepPrefix ? "REP " : "",
2354	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUTS" : "INS",
2355	pMsg->PortNumber, pMsg->AccessInfo.AccessSize ));
2356	rcStrict = IEMExecOne(pVCpu);
2357	}
2358	if (IOM_SUCCESS(rcStrict))
2359	{
2360	/*
2361	* Do debug checks.
2362	*/
2363	if ( pMsg->Header.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
2364	\|\| (pMsg->Header.Rflags & X86_EFL_TF)
2365	\|\| DBGFBpIsHwIoArmed(pVM) )
2366	{
2367	/** @todo Debugging. */
2368	}
2369	}
2370	return rcStrict;
2371	}
2372
2373	/*
2374	* Frequent exit or something needing probing.
2375	* Get state and call EMHistoryExec.
2376	*/
2377	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2378	if (!pMsg->AccessInfo.StringOp)
2379	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2380	else
2381	{
2382	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2383	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2384	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
2385	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
2386	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2387	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
2388	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
2389	}
2390	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2391
2392	# ifdef IN_RING0
2393	rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
2394	if (rcStrict != VINF_SUCCESS)
2395	return rcStrict;
2396	# else
2397	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2398	AssertRCReturn(rc, rc);
2399	# endif
2400
2401	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
2402	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2403	pMsg->AccessInfo.RepPrefix ? "REP " : "",
2404	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUT" : "IN",
2405	pMsg->AccessInfo.StringOp ? "S" : "",
2406	pMsg->PortNumber, pMsg->AccessInfo.AccessSize));
2407	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2408	Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2409	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2410	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2411	return rcStrict;
2412	}
2413	#elif defined(IN_RING3)
2414	/**
2415	* Deals with I/O port access exits (WHvRunVpExitReasonX64IoPortAccess).
2416	*
2417	* @returns Strict VBox status code.
2418	* @param pVM The cross context VM structure.
2419	* @param pVCpu The cross context per CPU structure.
2420	* @param pExit The VM exit information to handle.
2421	* @sa nemHCWinHandleMessageIoPort
2422	*/
2423	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitIoPort(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2424	{
2425	Assert( pExit->IoPortAccess.AccessInfo.AccessSize == 1
2426	\|\| pExit->IoPortAccess.AccessInfo.AccessSize == 2
2427	\|\| pExit->IoPortAccess.AccessInfo.AccessSize == 4);
2428
2429	/*
2430	* Whatever we do, we must clear pending event injection upon resume.
2431	*/
2432	if (pExit->VpContext.ExecutionState.InterruptionPending)
2433	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2434
2435	/*
2436	* Add history first to avoid two paths doing EMHistoryExec calls.
2437	*/
2438	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2439	!pExit->IoPortAccess.AccessInfo.StringOp
2440	? ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2441	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
2442	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
2443	: ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2444	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
2445	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
2446	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2447	if (!pExitRec)
2448	{
2449	VBOXSTRICTRC rcStrict;
2450	if (!pExit->IoPortAccess.AccessInfo.StringOp)
2451	{
2452	/*
2453	* Simple port I/O.
2454	*/
2455	static uint32_t const s_fAndMask[8] =
2456	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
2457	uint32_t const fAndMask = s_fAndMask[pExit->IoPortAccess.AccessInfo.AccessSize];
2458	if (pExit->IoPortAccess.AccessInfo.IsWrite)
2459	{
2460	rcStrict = IOMIOPortWrite(pVM, pVCpu, pExit->IoPortAccess.PortNumber,
2461	(uint32_t)pExit->IoPortAccess.Rax & fAndMask,
2462	pExit->IoPortAccess.AccessInfo.AccessSize);
2463	Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2464	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2465	pExit->IoPortAccess.PortNumber, (uint32_t)pExit->IoPortAccess.Rax & fAndMask,
2466	pExit->IoPortAccess.AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
2467	if (IOM_SUCCESS(rcStrict))
2468	{
2469	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2470	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
2471	}
2472	}
2473	else
2474	{
2475	uint32_t uValue = 0;
2476	rcStrict = IOMIOPortRead(pVM, pVCpu, pExit->IoPortAccess.PortNumber, &uValue,
2477	pExit->IoPortAccess.AccessInfo.AccessSize);
2478	Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2479	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2480	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2481	if (IOM_SUCCESS(rcStrict))
2482	{
2483	if (pExit->IoPortAccess.AccessInfo.AccessSize != 4)
2484	pVCpu->cpum.GstCtx.rax = (pExit->IoPortAccess.Rax & ~(uint64_t)fAndMask) \| (uValue & fAndMask);
2485	else
2486	pVCpu->cpum.GstCtx.rax = uValue;
2487	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2488	Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pExit->IoPortAccess.Rax, pVCpu->cpum.GstCtx.rax));
2489	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2490	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
2491	}
2492	}
2493	}
2494	else
2495	{
2496	/*
2497	* String port I/O.
2498	*/
2499	/** @todo Someone at Microsoft please explain how we can get the address mode
2500	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
2501	* getting the default mode, it can always be overridden by a prefix. This
2502	* forces us to interpret the instruction from opcodes, which is suboptimal.
2503	* Both AMD-V and VT-x includes the address size in the exit info, at least on
2504	* CPUs that are reasonably new.
2505	*
2506	* Of course, it's possible this is an undocumented and we just need to do some
2507	* experiments to figure out how it's communicated. Alternatively, we can scan
2508	* the opcode bytes for possible evil prefixes.
2509	*/
2510	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2511	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2512	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2513	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
2514	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
2515	pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
2516	pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
2517	pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
2518	pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
2519	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2520	AssertRCReturn(rc, rc);
2521
2522	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
2523	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2524	pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
2525	pExit->IoPortAccess.AccessInfo.IsWrite ? "OUTS" : "INS",
2526	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize ));
2527	rcStrict = IEMExecOne(pVCpu);
2528	}
2529	if (IOM_SUCCESS(rcStrict))
2530	{
2531	/*
2532	* Do debug checks.
2533	*/
2534	if ( pExit->VpContext.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
2535	\|\| (pExit->VpContext.Rflags & X86_EFL_TF)
2536	\|\| DBGFBpIsHwIoArmed(pVM) )
2537	{
2538	/** @todo Debugging. */
2539	}
2540	}
2541	return rcStrict;
2542	}
2543
2544	/*
2545	* Frequent exit or something needing probing.
2546	* Get state and call EMHistoryExec.
2547	*/
2548	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2549	if (!pExit->IoPortAccess.AccessInfo.StringOp)
2550	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2551	else
2552	{
2553	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2554	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2555	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
2556	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
2557	pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
2558	pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
2559	pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
2560	}
2561	pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
2562	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2563	AssertRCReturn(rc, rc);
2564	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
2565	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2566	pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
2567	pExit->IoPortAccess.AccessInfo.IsWrite ? "OUT" : "IN",
2568	pExit->IoPortAccess.AccessInfo.StringOp ? "S" : "",
2569	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize));
2570	VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2571	Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2572	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2573	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2574	return rcStrict;
2575	}
2576	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2577
2578
2579	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2580	/**
2581	* Deals with interrupt window message.
2582	*
2583	* @returns Strict VBox status code.
2584	* @param pVM The cross context VM structure.
2585	* @param pVCpu The cross context per CPU structure.
2586	* @param pMsg The message.
2587	* @sa nemR3WinHandleExitInterruptWindow
2588	*/
2589	NEM_TMPL_STATIC VBOXSTRICTRC
2590	nemHCWinHandleMessageInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_INTERRUPT_WINDOW_MESSAGE const *pMsg)
2591	{
2592	/*
2593	* Assert message sanity.
2594	*/
2595	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE
2596	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ // READ & WRITE are probably not used here
2597	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2598	AssertMsg(pMsg->Type == HvX64PendingInterrupt \|\| pMsg->Type == HvX64PendingNmi, ("%#x\n", pMsg->Type));
2599
2600	/*
2601	* Just copy the state we've got and handle it in the loop for now.
2602	*/
2603	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2604	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2605
2606	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2607	Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d\n",
2608	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2609	pMsg->Type, RT_BOOL(pMsg->Header.Rflags & X86_EFL_IF), pMsg->Header.ExecutionState.InterruptShadow));
2610
2611	/** @todo call nemHCWinHandleInterruptFF */
2612	RT_NOREF(pVM);
2613	return VINF_SUCCESS;
2614	}
2615	#elif defined(IN_RING3)
2616	/**
2617	* Deals with interrupt window exits (WHvRunVpExitReasonX64InterruptWindow).
2618	*
2619	* @returns Strict VBox status code.
2620	* @param pVM The cross context VM structure.
2621	* @param pVCpu The cross context per CPU structure.
2622	* @param pExit The VM exit information to handle.
2623	* @sa nemHCWinHandleMessageInterruptWindow
2624	*/
2625	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2626	{
2627	/*
2628	* Assert message sanity.
2629	*/
2630	AssertMsg( pExit->InterruptWindow.DeliverableType == WHvX64PendingInterrupt
2631	\|\| pExit->InterruptWindow.DeliverableType == WHvX64PendingNmi,
2632	("%#x\n", pExit->InterruptWindow.DeliverableType));
2633
2634	/*
2635	* Just copy the state we've got and handle it in the loop for now.
2636	*/
2637	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2638	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2639
2640	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2641	Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d CR8=%#x\n",
2642	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2643	pExit->InterruptWindow.DeliverableType, RT_BOOL(pExit->VpContext.Rflags & X86_EFL_IF),
2644	pExit->VpContext.ExecutionState.InterruptShadow, pExit->VpContext.Cr8));
2645
2646	/** @todo call nemHCWinHandleInterruptFF */
2647	RT_NOREF(pVM);
2648	return VINF_SUCCESS;
2649	}
2650	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2651
2652
2653	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2654	/**
2655	* Deals with CPUID intercept message.
2656	*
2657	* @returns Strict VBox status code.
2658	* @param pVM The cross context VM structure.
2659	* @param pVCpu The cross context per CPU structure.
2660	* @param pMsg The message.
2661	* @sa nemR3WinHandleExitCpuId
2662	*/
2663	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageCpuId(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_CPUID_INTERCEPT_MESSAGE const *pMsg)
2664	{
2665	/* Check message register value sanity. */
2666	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2667	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2668	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2669	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2670	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
2671	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
2672	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
2673	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbx, pMsg->Rbx);
2674
2675	/* Do exit history. */
2676	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
2677	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2678	if (!pExitRec)
2679	{
2680	/*
2681	* Soak up state and execute the instruction.
2682	*
2683	* Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
2684	* function and make everyone use it.
2685	*/
2686	/** @todo Combine implementations into IEMExecDecodedCpuId as this will
2687	* only get weirder with nested VT-x and AMD-V support. */
2688	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2689
2690	/* Copy in the low register values (top is always cleared). */
2691	pVCpu->cpum.GstCtx.rax = (uint32_t)pMsg->Rax;
2692	pVCpu->cpum.GstCtx.rcx = (uint32_t)pMsg->Rcx;
2693	pVCpu->cpum.GstCtx.rdx = (uint32_t)pMsg->Rdx;
2694	pVCpu->cpum.GstCtx.rbx = (uint32_t)pMsg->Rbx;
2695	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2696
2697	/* Get the correct values. */
2698	CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
2699	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
2700
2701	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
2702	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2703	pMsg->Rax, pMsg->Rcx, pMsg->Rdx, pMsg->Rbx,
2704	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx,
2705	pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
2706
2707	/* Move RIP and we're done. */
2708	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2709
2710	return VINF_SUCCESS;
2711	}
2712
2713	/*
2714	* Frequent exit or something needing probing.
2715	* Get state and call EMHistoryExec.
2716	*/
2717	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2718	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2719	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2720	pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
2721	pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
2722	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2723	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
2724	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2725	pMsg->Rax, pMsg->Rcx, pMsg->Rdx, pMsg->Rbx,
2726	pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
2727	# ifdef IN_RING0
2728	VBOXSTRICTRC rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "CpuIdExit");
2729	if (rcStrict != VINF_SUCCESS)
2730	return rcStrict;
2731	RT_NOREF(pVM);
2732	# else
2733	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2734	AssertRCReturn(rc, rc);
2735	# endif
2736	VBOXSTRICTRC rcStrictExec = EMHistoryExec(pVCpu, pExitRec, 0);
2737	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2738	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2739	VBOXSTRICTRC_VAL(rcStrictExec), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2740	return rcStrictExec;
2741	}
2742	#elif defined(IN_RING3)
2743	/**
2744	* Deals with CPUID exits (WHvRunVpExitReasonX64Cpuid).
2745	*
2746	* @returns Strict VBox status code.
2747	* @param pVM The cross context VM structure.
2748	* @param pVCpu The cross context per CPU structure.
2749	* @param pExit The VM exit information to handle.
2750	* @sa nemHCWinHandleMessageCpuId
2751	*/
2752	NEM_TMPL_STATIC VBOXSTRICTRC
2753	nemR3WinHandleExitCpuId(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2754	{
2755	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
2756	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2757	if (!pExitRec)
2758	{
2759	/*
2760	* Soak up state and execute the instruction.
2761	*
2762	* Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
2763	* function and make everyone use it.
2764	*/
2765	/** @todo Combine implementations into IEMExecDecodedCpuId as this will
2766	* only get weirder with nested VT-x and AMD-V support. */
2767	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2768
2769	/* Copy in the low register values (top is always cleared). */
2770	pVCpu->cpum.GstCtx.rax = (uint32_t)pExit->CpuidAccess.Rax;
2771	pVCpu->cpum.GstCtx.rcx = (uint32_t)pExit->CpuidAccess.Rcx;
2772	pVCpu->cpum.GstCtx.rdx = (uint32_t)pExit->CpuidAccess.Rdx;
2773	pVCpu->cpum.GstCtx.rbx = (uint32_t)pExit->CpuidAccess.Rbx;
2774	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2775
2776	/* Get the correct values. */
2777	CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
2778	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
2779
2780	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
2781	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2782	pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx,
2783	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx,
2784	pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
2785
2786	/* Move RIP and we're done. */
2787	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
2788
2789	RT_NOREF_PV(pVM);
2790	return VINF_SUCCESS;
2791	}
2792
2793	/*
2794	* Frequent exit or something needing probing.
2795	* Get state and call EMHistoryExec.
2796	*/
2797	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2798	pVCpu->cpum.GstCtx.rax = pExit->CpuidAccess.Rax;
2799	pVCpu->cpum.GstCtx.rcx = pExit->CpuidAccess.Rcx;
2800	pVCpu->cpum.GstCtx.rdx = pExit->CpuidAccess.Rdx;
2801	pVCpu->cpum.GstCtx.rbx = pExit->CpuidAccess.Rbx;
2802	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2803	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
2804	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2805	pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx,
2806	pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
2807	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2808	AssertRCReturn(rc, rc);
2809	VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2810	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2811	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2812	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2813	return rcStrict;
2814	}
2815	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2816
2817
2818	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2819	/**
2820	* Deals with MSR intercept message.
2821	*
2822	* @returns Strict VBox status code.
2823	* @param pVCpu The cross context per CPU structure.
2824	* @param pMsg The message.
2825	* @sa nemR3WinHandleExitMsr
2826	*/
2827	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageMsr(PVMCPUCC pVCpu, HV_X64_MSR_INTERCEPT_MESSAGE const *pMsg)
2828	{
2829	/*
2830	* A wee bit of sanity first.
2831	*/
2832	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
2833	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2834	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2835	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2836	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2837	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2838	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
2839	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
2840
2841	/*
2842	* Check CPL as that's common to both RDMSR and WRMSR.
2843	*/
2844	VBOXSTRICTRC rcStrict;
2845	if (pMsg->Header.ExecutionState.Cpl == 0)
2846	{
2847	/*
2848	* Get all the MSR state. Since we're getting EFER, we also need to
2849	* get CR0, CR4 and CR3.
2850	*/
2851	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2852	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2853	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
2854	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2855	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2856
2857	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2858	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
2859	(!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
2860	\| CPUMCTX_EXTRN_ALL_MSRS \| CPUMCTX_EXTRN_CR0
2861	\| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4,
2862	"MSRs");
2863	if (rcStrict == VINF_SUCCESS)
2864	{
2865	if (!pExitRec)
2866	{
2867	/*
2868	* Handle writes.
2869	*/
2870	if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2871	{
2872	rcStrict = CPUMSetGuestMsr(pVCpu, pMsg->MsrNumber, RT_MAKE_U64((uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx));
2873	Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n",
2874	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2875	pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
2876	if (rcStrict == VINF_SUCCESS)
2877	{
2878	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2879	return VINF_SUCCESS;
2880	}
2881	# ifndef IN_RING3
2882	/* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
2883	if (rcStrict == VERR_CPUM_RAISE_GP_0)
2884	rcStrict = VINF_CPUM_R3_MSR_WRITE;
2885	return rcStrict;
2886	# else
2887	LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n",
2888	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2889	pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
2890	# endif
2891	}
2892	/*
2893	* Handle reads.
2894	*/
2895	else
2896	{
2897	uint64_t uValue = 0;
2898	rcStrict = CPUMQueryGuestMsr(pVCpu, pMsg->MsrNumber, &uValue);
2899	Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
2900	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2901	pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2902	if (rcStrict == VINF_SUCCESS)
2903	{
2904	pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
2905	pVCpu->cpum.GstCtx.rdx = uValue >> 32;
2906	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
2907	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2908	return VINF_SUCCESS;
2909	}
2910	# ifndef IN_RING3
2911	/* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
2912	if (rcStrict == VERR_CPUM_RAISE_GP_0)
2913	rcStrict = VINF_CPUM_R3_MSR_READ;
2914	return rcStrict;
2915	# else
2916	LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
2917	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2918	pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2919	# endif
2920	}
2921	}
2922	else
2923	{
2924	/*
2925	* Handle frequent exit or something needing probing.
2926	*/
2927	Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
2928	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2929	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD", pMsg->MsrNumber));
2930	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2931	Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2932	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2933	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2934	return rcStrict;
2935	}
2936	}
2937	else
2938	{
2939	LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
2940	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2941	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD",
2942	pMsg->MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
2943	return rcStrict;
2944	}
2945	}
2946	else if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2947	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n",
2948	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2949	pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx ));
2950	else
2951	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n",
2952	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2953	pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber));
2954
2955	/*
2956	* If we get down here, we're supposed to #GP(0).
2957	*/
2958	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL_MSRS, "MSR");
2959	if (rcStrict == VINF_SUCCESS)
2960	{
2961	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
2962	if (rcStrict == VINF_IEM_RAISED_XCPT)
2963	rcStrict = VINF_SUCCESS;
2964	else if (rcStrict != VINF_SUCCESS)
2965	Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
2966	}
2967	return rcStrict;
2968	}
2969	#elif defined(IN_RING3)
2970	/**
2971	* Deals with MSR access exits (WHvRunVpExitReasonX64MsrAccess).
2972	*
2973	* @returns Strict VBox status code.
2974	* @param pVM The cross context VM structure.
2975	* @param pVCpu The cross context per CPU structure.
2976	* @param pExit The VM exit information to handle.
2977	* @sa nemHCWinHandleMessageMsr
2978	*/
2979	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitMsr(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2980	{
2981	/*
2982	* Check CPL as that's common to both RDMSR and WRMSR.
2983	*/
2984	VBOXSTRICTRC rcStrict;
2985	if (pExit->VpContext.ExecutionState.Cpl == 0)
2986	{
2987	/*
2988	* Get all the MSR state. Since we're getting EFER, we also need to
2989	* get CR0, CR4 and CR3.
2990	*/
2991	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2992	pExit->MsrAccess.AccessInfo.IsWrite
2993	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
2994	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2995	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2996	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2997	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
2998	(!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
2999	\| CPUMCTX_EXTRN_ALL_MSRS \| CPUMCTX_EXTRN_CR0
3000	\| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4,
3001	"MSRs");
3002	if (rcStrict == VINF_SUCCESS)
3003	{
3004	if (!pExitRec)
3005	{
3006	/*
3007	* Handle writes.
3008	*/
3009	if (pExit->MsrAccess.AccessInfo.IsWrite)
3010	{
3011	rcStrict = CPUMSetGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber,
3012	RT_MAKE_U64((uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx));
3013	Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3014	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
3015	(uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
3016	if (rcStrict == VINF_SUCCESS)
3017	{
3018	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
3019	return VINF_SUCCESS;
3020	}
3021	LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n", pVCpu->idCpu,
3022	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3023	pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx,
3024	VBOXSTRICTRC_VAL(rcStrict) ));
3025	}
3026	/*
3027	* Handle reads.
3028	*/
3029	else
3030	{
3031	uint64_t uValue = 0;
3032	rcStrict = CPUMQueryGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber, &uValue);
3033	Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu,
3034	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3035	pExit->MsrAccess.MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
3036	if (rcStrict == VINF_SUCCESS)
3037	{
3038	pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
3039	pVCpu->cpum.GstCtx.rdx = uValue >> 32;
3040	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
3041	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
3042	return VINF_SUCCESS;
3043	}
3044	LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3045	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
3046	uValue, VBOXSTRICTRC_VAL(rcStrict) ));
3047	}
3048	}
3049	else
3050	{
3051	/*
3052	* Handle frequent exit or something needing probing.
3053	*/
3054	Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
3055	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3056	pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber));
3057	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
3058	Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
3059	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3060	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
3061	return rcStrict;
3062	}
3063	}
3064	else
3065	{
3066	LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
3067	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3068	pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
3069	return rcStrict;
3070	}
3071	}
3072	else if (pExit->MsrAccess.AccessInfo.IsWrite)
3073	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3074	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
3075	pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx ));
3076	else
3077	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3078	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
3079	pExit->MsrAccess.MsrNumber));
3080
3081	/*
3082	* If we get down here, we're supposed to #GP(0).
3083	*/
3084	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL_MSRS, "MSR");
3085	if (rcStrict == VINF_SUCCESS)
3086	{
3087	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
3088	if (rcStrict == VINF_IEM_RAISED_XCPT)
3089	rcStrict = VINF_SUCCESS;
3090	else if (rcStrict != VINF_SUCCESS)
3091	Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
3092	}
3093
3094	RT_NOREF_PV(pVM);
3095	return rcStrict;
3096	}
3097	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3098
3099
3100	/**
3101	* Worker for nemHCWinHandleMessageException & nemR3WinHandleExitException that
3102	* checks if the given opcodes are of interest at all.
3103	*
3104	* @returns true if interesting, false if not.
3105	* @param cbOpcodes Number of opcode bytes available.
3106	* @param pbOpcodes The opcode bytes.
3107	* @param f64BitMode Whether we're in 64-bit mode.
3108	*/
3109	DECLINLINE(bool) nemHcWinIsInterestingUndefinedOpcode(uint8_t cbOpcodes, uint8_t const *pbOpcodes, bool f64BitMode)
3110	{
3111	/*
3112	* Currently only interested in VMCALL and VMMCALL.
3113	*/
3114	while (cbOpcodes >= 3)
3115	{
3116	switch (pbOpcodes[0])
3117	{
3118	case 0x0f:
3119	switch (pbOpcodes[1])
3120	{
3121	case 0x01:
3122	switch (pbOpcodes[2])
3123	{
3124	case 0xc1: /* 0f 01 c1 VMCALL */
3125	return true;
3126	case 0xd9: /* 0f 01 d9 VMMCALL */
3127	return true;
3128	default:
3129	break;
3130	}
3131	break;
3132	}
3133	break;
3134
3135	default:
3136	return false;
3137
3138	/* prefixes */
3139	case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47:
3140	case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
3141	if (!f64BitMode)
3142	return false;
3143	RT_FALL_THRU();
3144	case X86_OP_PRF_CS:
3145	case X86_OP_PRF_SS:
3146	case X86_OP_PRF_DS:
3147	case X86_OP_PRF_ES:
3148	case X86_OP_PRF_FS:
3149	case X86_OP_PRF_GS:
3150	case X86_OP_PRF_SIZE_OP:
3151	case X86_OP_PRF_SIZE_ADDR:
3152	case X86_OP_PRF_LOCK:
3153	case X86_OP_PRF_REPZ:
3154	case X86_OP_PRF_REPNZ:
3155	cbOpcodes--;
3156	pbOpcodes++;
3157	continue;
3158	}
3159	break;
3160	}
3161	return false;
3162	}
3163
3164
3165	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3166	/**
3167	* Copies state included in a exception intercept message.
3168	*
3169	* @param pVCpu The cross context per CPU structure.
3170	* @param pMsg The message.
3171	* @param fClearXcpt Clear pending exception.
3172	*/
3173	DECLINLINE(void)
3174	nemHCWinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg, bool fClearXcpt)
3175	{
3176	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
3177	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_DS
3178	\| (fClearXcpt ? CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT : 0) );
3179	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
3180	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
3181	pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
3182	pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
3183	pVCpu->cpum.GstCtx.rsp = pMsg->Rsp;
3184	pVCpu->cpum.GstCtx.rbp = pMsg->Rbp;
3185	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
3186	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
3187	pVCpu->cpum.GstCtx.r8 = pMsg->R8;
3188	pVCpu->cpum.GstCtx.r9 = pMsg->R9;
3189	pVCpu->cpum.GstCtx.r10 = pMsg->R10;
3190	pVCpu->cpum.GstCtx.r11 = pMsg->R11;
3191	pVCpu->cpum.GstCtx.r12 = pMsg->R12;
3192	pVCpu->cpum.GstCtx.r13 = pMsg->R13;
3193	pVCpu->cpum.GstCtx.r14 = pMsg->R14;
3194	pVCpu->cpum.GstCtx.r15 = pMsg->R15;
3195	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
3196	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ss, pMsg->SsSegment);
3197	}
3198	#elif defined(IN_RING3)
3199	/**
3200	* Copies state included in a exception intercept exit.
3201	*
3202	* @param pVCpu The cross context per CPU structure.
3203	* @param pExit The VM exit information.
3204	* @param fClearXcpt Clear pending exception.
3205	*/
3206	DECLINLINE(void) nemR3WinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit, bool fClearXcpt)
3207	{
3208	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3209	if (fClearXcpt)
3210	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
3211	}
3212	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3213
3214
3215	/**
3216	* Advances the guest RIP by the number of bytes specified in @a cb.
3217	*
3218	* @param pVCpu The cross context virtual CPU structure.
3219	* @param cb RIP increment value in bytes.
3220	*/
3221	DECLINLINE(void) nemHcWinAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
3222	{
3223	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3224	pCtx->rip += cb;
3225
3226	/* Update interrupt shadow. */
3227	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
3228	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3229	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3230	}
3231
3232
3233	/**
3234	* Hacks its way around the lovely mesa driver's backdoor accesses.
3235	*
3236	* @sa hmR0VmxHandleMesaDrvGp
3237	* @sa hmR0SvmHandleMesaDrvGp
3238	*/
3239	static int nemHcWinHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx)
3240	{
3241	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK)));
3242	RT_NOREF(pCtx);
3243
3244	/* For now we'll just skip the instruction. */
3245	nemHcWinAdvanceRip(pVCpu, 1);
3246	return VINF_SUCCESS;
3247	}
3248
3249
3250	/**
3251	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
3252	* backdoor logging w/o checking what it is running inside.
3253	*
3254	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
3255	* backdoor port and magic numbers loaded in registers.
3256	*
3257	* @returns true if it is, false if it isn't.
3258	* @sa hmR0VmxIsMesaDrvGp
3259	* @sa hmR0SvmIsMesaDrvGp
3260	*/
3261	DECLINLINE(bool) nemHcWinIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, const uint8_t *pbInsn, uint32_t cbInsn)
3262	{
3263	/* #GP(0) is already checked by caller. */
3264
3265	/* Check magic and port. */
3266	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RAX)));
3267	if (pCtx->dx != UINT32_C(0x5658))
3268	return false;
3269	if (pCtx->rax != UINT32_C(0x564d5868))
3270	return false;
3271
3272	/* Flat ring-3 CS. */
3273	if (CPUMGetGuestCPL(pVCpu) != 3)
3274	return false;
3275	if (pCtx->cs.u64Base != 0)
3276	return false;
3277
3278	/* 0xed: IN eAX,dx */
3279	if (cbInsn < 1) /* Play safe (shouldn't happen). */
3280	{
3281	uint8_t abInstr[1];
3282	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
3283	if (RT_FAILURE(rc))
3284	return false;
3285	if (abInstr[0] != 0xed)
3286	return false;
3287	}
3288	else
3289	{
3290	if (pbInsn[0] != 0xed)
3291	return false;
3292	}
3293
3294	return true;
3295	}
3296
3297
3298	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3299	/**
3300	* Deals with exception intercept message (HvMessageTypeX64ExceptionIntercept).
3301	*
3302	* @returns Strict VBox status code.
3303	* @param pVCpu The cross context per CPU structure.
3304	* @param pMsg The message.
3305	* @sa nemR3WinHandleExitMsr
3306	*/
3307	NEM_TMPL_STATIC VBOXSTRICTRC
3308	nemHCWinHandleMessageException(PVMCPUCC pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg)
3309	{
3310	/*
3311	* Assert sanity.
3312	*/
3313	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
3314	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
3315	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
3316	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
3317	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
3318	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
3319	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
3320	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterDs, pMsg->DsSegment);
3321	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterSs, pMsg->SsSegment);
3322	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
3323	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
3324	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
3325	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbx, pMsg->Rbx);
3326	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsp, pMsg->Rsp);
3327	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbp, pMsg->Rbp);
3328	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsi, pMsg->Rsi);
3329	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdi, pMsg->Rdi);
3330	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR8, pMsg->R8);
3331	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR9, pMsg->R9);
3332	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR10, pMsg->R10);
3333	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR11, pMsg->R11);
3334	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR12, pMsg->R12);
3335	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR13, pMsg->R13);
3336	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR14, pMsg->R14);
3337	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR15, pMsg->R15);
3338
3339	/*
3340	* Get most of the register state since we'll end up making IEM inject the
3341	* event. The exception isn't normally flaged as a pending event, so duh.
3342	*
3343	* Note! We can optimize this later with event injection.
3344	*/
3345	Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n",
3346	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
3347	pMsg->ExceptionVector, pMsg->ErrorCode, pMsg->ExceptionParameter));
3348	nemHCWinCopyStateFromExceptionMessage(pVCpu, pMsg, true /fClearXcpt/);
3349	uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
3350	if (pMsg->ExceptionVector == X86_XCPT_DB)
3351	fWhat \|= CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_DR6;
3352	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt");
3353	if (rcStrict != VINF_SUCCESS)
3354	return rcStrict;
3355
3356	/*
3357	* Handle the intercept.
3358	*/
3359	TRPMEVENT enmEvtType = TRPM_TRAP;
3360	switch (pMsg->ExceptionVector)
3361	{
3362	/*
3363	* We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
3364	* and need to turn them over to GIM.
3365	*
3366	* Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
3367	* #UD for handling non-native hypercall instructions. (IEM will
3368	* decode both and let the GIM provider decide whether to accept it.)
3369	*/
3370	case X86_XCPT_UD:
3371	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
3372	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
3373	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3374
3375	if (nemHcWinIsInterestingUndefinedOpcode(pMsg->InstructionByteCount, pMsg->InstructionBytes,
3376	pMsg->Header.ExecutionState.EferLma && pMsg->Header.CsSegment.Long ))
3377	{
3378	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
3379	pMsg->InstructionBytes, pMsg->InstructionByteCount);
3380	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
3381	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
3382	nemHCWinExecStateToLogStr(&pMsg->Header), VBOXSTRICTRC_VAL(rcStrict) ));
3383	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
3384	return rcStrict;
3385	}
3386	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
3387	pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->InstructionByteCount, pMsg->InstructionBytes ));
3388	break;
3389
3390	/*
3391	* Workaround the lovely mesa driver assuming that vmsvga means vmware
3392	* hypervisor and tries to log stuff to the host.
3393	*/
3394	case X86_XCPT_GP:
3395	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGp);
3396	/** @todo r=bird: Need workaround in IEM for this, right?
3397	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_GP),
3398	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC()); */
3399	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
3400	\|\| !nemHcWinIsMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx, pMsg->InstructionBytes, pMsg->InstructionByteCount))
3401	{
3402	# if 1 /** @todo Need to emulate instruction or we get a triple fault when trying to inject the #GP... */
3403	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
3404	pMsg->InstructionBytes, pMsg->InstructionByteCount);
3405	Log4(("XcptExit/%u: %04x:%08RX64/%s: #GP -> emulated -> %Rrc\n",
3406	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
3407	nemHCWinExecStateToLogStr(&pMsg->Header), VBOXSTRICTRC_VAL(rcStrict) ));
3408	return rcStrict;
3409	# else
3410	break;
3411	# endif
3412	}
3413	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGpMesa);
3414	return nemHcWinHandleMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx);
3415
3416	/*
3417	* Filter debug exceptions.
3418	*/
3419	case X86_XCPT_DB:
3420	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
3421	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
3422	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3423	Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
3424	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header) ));
3425	break;
3426
3427	case X86_XCPT_BP:
3428	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
3429	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
3430	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3431	Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
3432	pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->Header.InstructionLength));
3433	enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
3434	break;
3435
3436	/* This shouldn't happen. */
3437	default:
3438	AssertLogRelMsgFailedReturn(("ExceptionVector=%#x\n", pMsg->ExceptionVector), VERR_IEM_IPE_6);
3439	}
3440
3441	/*
3442	* Inject it.
3443	*/
3444	rcStrict = IEMInjectTrap(pVCpu, pMsg->ExceptionVector, enmEvtType, pMsg->ErrorCode,
3445	pMsg->ExceptionParameter /??/, pMsg->Header.InstructionLength);
3446	Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
3447	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
3448	nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->ExceptionVector, VBOXSTRICTRC_VAL(rcStrict) ));
3449	return rcStrict;
3450	}
3451	#elif defined(IN_RING3)
3452	/**
3453	* Deals with MSR access exits (WHvRunVpExitReasonException).
3454	*
3455	* @returns Strict VBox status code.
3456	* @param pVM The cross context VM structure.
3457	* @param pVCpu The cross context per CPU structure.
3458	* @param pExit The VM exit information to handle.
3459	* @sa nemR3WinHandleExitException
3460	*/
3461	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3462	{
3463	/*
3464	* Get most of the register state since we'll end up making IEM inject the
3465	* event. The exception isn't normally flaged as a pending event, so duh.
3466	*
3467	* Note! We can optimize this later with event injection.
3468	*/
3469	Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3470	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType,
3471	pExit->VpException.ErrorCode, pExit->VpException.ExceptionParameter ));
3472	nemR3WinCopyStateFromExceptionMessage(pVCpu, pExit, true /fClearXcpt/);
3473	uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
3474	if (pExit->VpException.ExceptionType == X86_XCPT_DB)
3475	fWhat \|= CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_DR6;
3476	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt");
3477	if (rcStrict != VINF_SUCCESS)
3478	return rcStrict;
3479
3480	/*
3481	* Handle the intercept.
3482	*/
3483	TRPMEVENT enmEvtType = TRPM_TRAP;
3484	switch (pExit->VpException.ExceptionType)
3485	{
3486	/*
3487	* We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
3488	* and need to turn them over to GIM.
3489	*
3490	* Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
3491	* #UD for handling non-native hypercall instructions. (IEM will
3492	* decode both and let the GIM provider decide whether to accept it.)
3493	*/
3494	case X86_XCPT_UD:
3495	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
3496	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
3497	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3498	if (nemHcWinIsInterestingUndefinedOpcode(pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes,
3499	pExit->VpContext.ExecutionState.EferLma && pExit->VpContext.Cs.Long ))
3500	{
3501	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
3502	pExit->VpException.InstructionBytes,
3503	pExit->VpException.InstructionByteCount);
3504	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
3505	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
3506	nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
3507	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
3508	return rcStrict;
3509	}
3510
3511	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu,
3512	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3513	pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes ));
3514	break;
3515
3516	/*
3517	* Workaround the lovely mesa driver assuming that vmsvga means vmware
3518	* hypervisor and tries to log stuff to the host.
3519	*/
3520	case X86_XCPT_GP:
3521	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGp);
3522	/** @todo r=bird: Need workaround in IEM for this, right?
3523	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_GP),
3524	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); */
3525	if ( !pVCpu->nem.s.fTrapXcptGpForLovelyMesaDrv
3526	\|\| !nemHcWinIsMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx, pExit->VpException.InstructionBytes,
3527	pExit->VpException.InstructionByteCount))
3528	{
3529	# if 1 /** @todo Need to emulate instruction or we get a triple fault when trying to inject the #GP... */
3530	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
3531	pExit->VpException.InstructionBytes,
3532	pExit->VpException.InstructionByteCount);
3533	Log4(("XcptExit/%u: %04x:%08RX64/%s: #GP -> emulated -> %Rrc\n",
3534	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
3535	nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
3536	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
3537	return rcStrict;
3538	# else
3539	break;
3540	# endif
3541	}
3542	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGpMesa);
3543	return nemHcWinHandleMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx);
3544
3545	/*
3546	* Filter debug exceptions.
3547	*/
3548	case X86_XCPT_DB:
3549	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
3550	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
3551	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3552	Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
3553	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext) ));
3554	break;
3555
3556	case X86_XCPT_BP:
3557	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
3558	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
3559	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3560	Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3561	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.InstructionLength));
3562	enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
3563	break;
3564
3565	/* This shouldn't happen. */
3566	default:
3567	AssertLogRelMsgFailedReturn(("ExceptionType=%#x\n", pExit->VpException.ExceptionType), VERR_IEM_IPE_6);
3568	}
3569
3570	/*
3571	* Inject it.
3572	*/
3573	rcStrict = IEMInjectTrap(pVCpu, pExit->VpException.ExceptionType, enmEvtType, pExit->VpException.ErrorCode,
3574	pExit->VpException.ExceptionParameter /??/, pExit->VpContext.InstructionLength);
3575	Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
3576	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
3577	nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType, VBOXSTRICTRC_VAL(rcStrict) ));
3578
3579	RT_NOREF_PV(pVM);
3580	return rcStrict;
3581	}
3582	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3583
3584
3585	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3586	/**
3587	* Deals with unrecoverable exception (triple fault).
3588	*
3589	* Seen WRMSR 0x201 (IA32_MTRR_PHYSMASK0) writes from grub / debian9 ending up
3590	* here too. So we'll leave it to IEM to decide.
3591	*
3592	* @returns Strict VBox status code.
3593	* @param pVCpu The cross context per CPU structure.
3594	* @param pMsgHdr The message header.
3595	* @sa nemR3WinHandleExitUnrecoverableException
3596	*/
3597	NEM_TMPL_STATIC VBOXSTRICTRC
3598	nemHCWinHandleMessageUnrecoverableException(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr)
3599	{
3600	/* Check message register value sanity. */
3601	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsgHdr->CsSegment);
3602	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsgHdr->Rip);
3603	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsgHdr->Rflags);
3604	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsgHdr->Cr8);
3605
3606	# if 0
3607	/*
3608	* Just copy the state we've got and handle it in the loop for now.
3609	*/
3610	nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
3611	Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n",
3612	pVCpu->idCpu, pMsgHdr->CsSegment.Selector, pMsgHdr->Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsgHdr->Rflags));
3613	return VINF_EM_TRIPLE_FAULT;
3614	# else
3615	/*
3616	* Let IEM decide whether this is really it.
3617	*/
3618	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
3619	pMsgHdr->Rip + pMsgHdr->CsSegment.Base, ASMReadTSC());
3620	nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
3621	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL, "TripleExit");
3622	if (rcStrict == VINF_SUCCESS)
3623	{
3624	rcStrict = IEMExecOne(pVCpu);
3625	if (rcStrict == VINF_SUCCESS)
3626	{
3627	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3628	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags ));
3629	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
3630	return VINF_SUCCESS;
3631	}
3632	if (rcStrict == VINF_EM_TRIPLE_FAULT)
3633	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3634	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3635	else
3636	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3637	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3638	}
3639	else
3640	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3641	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3642	return rcStrict;
3643	# endif
3644	}
3645	#elif defined(IN_RING3)
3646	/**
3647	* Deals with MSR access exits (WHvRunVpExitReasonUnrecoverableException).
3648	*
3649	* @returns Strict VBox status code.
3650	* @param pVM The cross context VM structure.
3651	* @param pVCpu The cross context per CPU structure.
3652	* @param pExit The VM exit information to handle.
3653	* @sa nemHCWinHandleMessageUnrecoverableException
3654	*/
3655	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitUnrecoverableException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3656	{
3657	# if 0
3658	/*
3659	* Just copy the state we've got and handle it in the loop for now.
3660	*/
3661	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3662	Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3663	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
3664	RT_NOREF_PV(pVM);
3665	return VINF_EM_TRIPLE_FAULT;
3666	# else
3667	/*
3668	* Let IEM decide whether this is really it.
3669	*/
3670	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
3671	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3672	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3673	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL, "TripleExit");
3674	if (rcStrict == VINF_SUCCESS)
3675	{
3676	rcStrict = IEMExecOne(pVCpu);
3677	if (rcStrict == VINF_SUCCESS)
3678	{
3679	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3680	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
3681	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
3682	return VINF_SUCCESS;
3683	}
3684	if (rcStrict == VINF_EM_TRIPLE_FAULT)
3685	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3686	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3687	else
3688	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3689	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3690	}
3691	else
3692	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3693	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3694	RT_NOREF_PV(pVM);
3695	return rcStrict;
3696	# endif
3697
3698	}
3699	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3700
3701
3702	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3703	/**
3704	* Handles messages (VM exits).
3705	*
3706	* @returns Strict VBox status code.
3707	* @param pVM The cross context VM structure.
3708	* @param pVCpu The cross context per CPU structure.
3709	* @param pMappingHeader The message slot mapping.
3710	* @sa nemR3WinHandleExit
3711	*/
3712	NEM_TMPL_STATIC VBOXSTRICTRC
3713	nemHCWinHandleMessage(PVMCC pVM, PVMCPUCC pVCpu, VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader)
3714	{
3715	if (pMappingHeader->enmVidMsgType == VidMessageHypervisorMessage)
3716	{
3717	AssertMsg(pMappingHeader->cbMessage == HV_MESSAGE_SIZE, ("%#x\n", pMappingHeader->cbMessage));
3718	HV_MESSAGE const pMsg = (HV_MESSAGE const )(pMappingHeader + 1);
3719	switch (pMsg->Header.MessageType)
3720	{
3721	case HvMessageTypeUnmappedGpa:
3722	Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
3723	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
3724	return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept);
3725
3726	case HvMessageTypeGpaIntercept:
3727	Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
3728	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemIntercept);
3729	return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept);
3730
3731	case HvMessageTypeX64IoPortIntercept:
3732	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64IoPortIntercept));
3733	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
3734	return nemHCWinHandleMessageIoPort(pVM, pVCpu, &pMsg->X64IoPortIntercept);
3735
3736	case HvMessageTypeX64Halt:
3737	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
3738	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
3739	pMsg->X64InterceptHeader.Rip + pMsg->X64InterceptHeader.CsSegment.Base, ASMReadTSC());
3740	Log4(("HaltExit\n"));
3741	return VINF_EM_HALT;
3742
3743	case HvMessageTypeX64InterruptWindow:
3744	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterruptWindow));
3745	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
3746	return nemHCWinHandleMessageInterruptWindow(pVM, pVCpu, &pMsg->X64InterruptWindow);
3747
3748	case HvMessageTypeX64CpuidIntercept:
3749	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64CpuIdIntercept));
3750	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
3751	return nemHCWinHandleMessageCpuId(pVM, pVCpu, &pMsg->X64CpuIdIntercept);
3752
3753	case HvMessageTypeX64MsrIntercept:
3754	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64MsrIntercept));
3755	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
3756	return nemHCWinHandleMessageMsr(pVCpu, &pMsg->X64MsrIntercept);
3757
3758	case HvMessageTypeX64ExceptionIntercept:
3759	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64ExceptionIntercept));
3760	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
3761	return nemHCWinHandleMessageException(pVCpu, &pMsg->X64ExceptionIntercept);
3762
3763	case HvMessageTypeUnrecoverableException:
3764	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterceptHeader));
3765	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
3766	return nemHCWinHandleMessageUnrecoverableException(pVCpu, &pMsg->X64InterceptHeader);
3767
3768	case HvMessageTypeInvalidVpRegisterValue:
3769	case HvMessageTypeUnsupportedFeature:
3770	case HvMessageTypeTlbPageSizeMismatch:
3771	LogRel(("Unimplemented msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3772	AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n%.32Rhxd\n", pMsg->Header.MessageType, pMsg),
3773	VERR_NEM_IPE_3);
3774
3775	case HvMessageTypeX64ApicEoi:
3776	case HvMessageTypeX64LegacyFpError:
3777	case HvMessageTypeX64RegisterIntercept:
3778	case HvMessageTypeApicEoi:
3779	case HvMessageTypeFerrAsserted:
3780	case HvMessageTypeEventLogBufferComplete:
3781	case HvMessageTimerExpired:
3782	LogRel(("Unexpected msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3783	AssertLogRelMsgFailedReturn(("Unexpected message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
3784	VERR_NEM_IPE_3);
3785
3786	default:
3787	LogRel(("Unknown msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3788	AssertLogRelMsgFailedReturn(("Unknown message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
3789	VERR_NEM_IPE_3);
3790	}
3791	}
3792	else
3793	AssertLogRelMsgFailedReturn(("Unexpected VID message type on CPU #%u: %#x LB %u\n",
3794	pVCpu->idCpu, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage),
3795	VERR_NEM_IPE_4);
3796	}
3797	#elif defined(IN_RING3)
3798	/**
3799	* Handles VM exits.
3800	*
3801	* @returns Strict VBox status code.
3802	* @param pVM The cross context VM structure.
3803	* @param pVCpu The cross context per CPU structure.
3804	* @param pExit The VM exit information to handle.
3805	* @sa nemHCWinHandleMessage
3806	*/
3807	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExit(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3808	{
3809	switch (pExit->ExitReason)
3810	{
3811	case WHvRunVpExitReasonMemoryAccess:
3812	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
3813	return nemR3WinHandleExitMemory(pVM, pVCpu, pExit);
3814
3815	case WHvRunVpExitReasonX64IoPortAccess:
3816	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
3817	return nemR3WinHandleExitIoPort(pVM, pVCpu, pExit);
3818
3819	case WHvRunVpExitReasonX64Halt:
3820	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
3821	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
3822	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3823	Log4(("HaltExit/%u\n", pVCpu->idCpu));
3824	return VINF_EM_HALT;
3825
3826	case WHvRunVpExitReasonCanceled:
3827	Log4(("CanceledExit/%u\n", pVCpu->idCpu));
3828	return VINF_SUCCESS;
3829
3830	case WHvRunVpExitReasonX64InterruptWindow:
3831	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
3832	return nemR3WinHandleExitInterruptWindow(pVM, pVCpu, pExit);
3833
3834	case WHvRunVpExitReasonX64Cpuid:
3835	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
3836	return nemR3WinHandleExitCpuId(pVM, pVCpu, pExit);
3837
3838	case WHvRunVpExitReasonX64MsrAccess:
3839	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
3840	return nemR3WinHandleExitMsr(pVM, pVCpu, pExit);
3841
3842	case WHvRunVpExitReasonException:
3843	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
3844	return nemR3WinHandleExitException(pVM, pVCpu, pExit);
3845
3846	case WHvRunVpExitReasonUnrecoverableException:
3847	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
3848	return nemR3WinHandleExitUnrecoverableException(pVM, pVCpu, pExit);
3849
3850	case WHvRunVpExitReasonUnsupportedFeature:
3851	case WHvRunVpExitReasonInvalidVpRegisterValue:
3852	LogRel(("Unimplemented exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
3853	AssertLogRelMsgFailedReturn(("Unexpected exit on CPU #%u: %#x\n%.32Rhxd\n",
3854	pVCpu->idCpu, pExit->ExitReason, pExit), VERR_NEM_IPE_3);
3855
3856	/* Undesired exits: */
3857	case WHvRunVpExitReasonNone:
3858	default:
3859	LogRel(("Unknown exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
3860	AssertLogRelMsgFailedReturn(("Unknown exit on CPU #%u: %#x!\n", pVCpu->idCpu, pExit->ExitReason), VERR_NEM_IPE_3);
3861	}
3862	}
3863	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3864
3865
3866	#if defined(IN_RING0) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
3867	/**
3868	* Perform an I/O control operation on the partition handle (VID.SYS),
3869	* restarting on alert-like behaviour.
3870	*
3871	* @returns NT status code.
3872	* @param pGVM The ring-0 VM structure.
3873	* @param pGVCpu The global (ring-0) per CPU structure.
3874	* @param fFlags The wait flags.
3875	* @param cMillies The timeout in milliseconds
3876	*/
3877	static NTSTATUS nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(PGVM pGVM, PGVMCPU pGVCpu, uint32_t fFlags, uint32_t cMillies)
3878	{
3879	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
3880	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = fFlags;
3881	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
3882	NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVCpu, pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
3883	&pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
3884	pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
3885	NULL, 0);
3886	if (rcNt == STATUS_SUCCESS)
3887	{ /* likely */ }
3888	/*
3889	* Generally, if we get down here, we have been interrupted between ACK'ing
3890	* a message and waiting for the next due to a NtAlertThread call. So, we
3891	* should stop ACK'ing the previous message and get on waiting on the next.
3892	* See similar stuff in nemHCWinRunGC().
3893	*/
3894	else if ( rcNt == STATUS_TIMEOUT
3895	\|\| rcNt == STATUS_ALERTED /* just in case */
3896	\|\| rcNt == STATUS_KERNEL_APC /* just in case */
3897	\|\| rcNt == STATUS_USER_APC /* just in case */)
3898	{
3899	DBGFTRACE_CUSTOM(pGVCpu->CTX_SUFF(pVM), "IoCtlMessageSlotHandleAndGetNextRestart/1 %#x (f=%#x)", rcNt, fFlags);
3900	STAM_REL_COUNTER_INC(&pGVCpu->nem.s.StatStopCpuPendingAlerts);
3901	Assert(fFlags & VID_MSHAGN_F_GET_NEXT_MESSAGE);
3902
3903	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
3904	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = fFlags & ~VID_MSHAGN_F_HANDLE_MESSAGE;
3905	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
3906	rcNt = nemR0NtPerformIoControl(pGVM, pGVCpu, pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
3907	&pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
3908	pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
3909	NULL, 0);
3910	DBGFTRACE_CUSTOM(pGVM, "IoCtlMessageSlotHandleAndGetNextRestart/2 %#x", rcNt);
3911	}
3912	return rcNt;
3913	}
3914	#endif /* IN_RING0 */
3915
3916
3917	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3918	/**
3919	* Worker for nemHCWinRunGC that stops the execution on the way out.
3920	*
3921	* The CPU was running the last time we checked, no there are no messages that
3922	* needs being marked handled/whatever. Caller checks this.
3923	*
3924	* @returns rcStrict on success, error status on failure.
3925	* @param pVM The cross context VM structure.
3926	* @param pVCpu The cross context per CPU structure.
3927	* @param rcStrict The nemHCWinRunGC return status. This is a little
3928	* bit unnecessary, except in internal error cases,
3929	* since we won't need to stop the CPU if we took an
3930	* exit.
3931	* @param pMappingHeader The message slot mapping.
3932	*/
3933	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinStopCpu(PVMCC pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict,
3934	VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader)
3935	{
3936	# ifdef DBGFTRACE_ENABLED
3937	HV_MESSAGE const volatile pMsgForTrace = (HV_MESSAGE const volatile )(pMappingHeader + 1);
3938	# endif
3939
3940	/*
3941	* Try stopping the processor. If we're lucky we manage to do this before it
3942	* does another VM exit.
3943	*/
3944	DBGFTRACE_CUSTOM(pVM, "nemStop#0");
3945	# ifdef IN_RING0
3946	pVCpu->nem.s.uIoCtlBuf.idCpu = pVCpu->idCpu;
3947	NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlStopVirtualProcessor.uFunction,
3948	&pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
3949	NULL, 0);
3950	if (NT_SUCCESS(rcNt))
3951	{
3952	DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay (%#x)", rcNt);
3953	Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
3954	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
3955	return rcStrict;
3956	}
3957	# else
3958	BOOL fRet = VidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu);
3959	if (fRet)
3960	{
3961	DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay");
3962	Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
3963	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
3964	return rcStrict;
3965	}
3966	# endif
3967
3968	/*
3969	* Dang. The CPU stopped by itself and we got a couple of message to deal with.
3970	*/
3971	# ifdef IN_RING0
3972	DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", rcNt);
3973	AssertLogRelMsgReturn(rcNt == ERROR_VID_STOP_PENDING, ("rcNt=%#x\n", rcNt),
3974	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3975	# else
3976	DWORD dwErr = RTNtLastErrorValue();
3977	DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", dwErr);
3978	AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u (%#x)\n", dwErr, dwErr),
3979	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3980	# endif
3981	Log8(("nemHCWinStopCpu: Stopping CPU #%u pending...\n", pVCpu->idCpu));
3982	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPending);
3983
3984	/*
3985	* First message: Exit or similar, sometimes VidMessageStopRequestComplete.
3986	* Note! We can safely ASSUME that rcStrict isn't an important information one.
3987	*/
3988	# ifdef IN_RING0
3989	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu, VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3990	DBGFTRACE_CUSTOM(pVM, "nemStop#1: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
3991	pMsgForTrace->Header.MessageType);
3992	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
3993	("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
3994	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3995	# else
3996	BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
3997	VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3998	DBGFTRACE_CUSTOM(pVM, "nemStop#1: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
3999	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
4000	AssertLogRelMsgReturn(fWait, ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
4001	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
4002	# endif
4003
4004	VID_MESSAGE_TYPE enmVidMsgType = pMappingHeader->enmVidMsgType;
4005	if (enmVidMsgType != VidMessageStopRequestComplete)
4006	{
4007	VBOXSTRICTRC rcStrict2 = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader);
4008	if (rcStrict2 != VINF_SUCCESS && RT_SUCCESS(rcStrict))
4009	rcStrict = rcStrict2;
4010	DBGFTRACE_CUSTOM(pVM, "nemStop#1: handled %#x -> %d", pMsgForTrace->Header.MessageType, VBOXSTRICTRC_VAL(rcStrict));
4011
4012	/*
4013	* Mark it as handled and get the stop request completed message, then mark
4014	* that as handled too. CPU is back into fully stopped stated then.
4015	*/
4016	# ifdef IN_RING0
4017	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu,
4018	VID_MSHAGN_F_HANDLE_MESSAGE \| VID_MSHAGN_F_GET_NEXT_MESSAGE,
4019	30000 /ms/);
4020	DBGFTRACE_CUSTOM(pVM, "nemStop#2: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
4021	pMsgForTrace->Header.MessageType);
4022	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
4023	("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
4024	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
4025	# else
4026	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
4027	VID_MSHAGN_F_HANDLE_MESSAGE \| VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
4028	DBGFTRACE_CUSTOM(pVM, "nemStop#2: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
4029	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
4030	AssertLogRelMsgReturn(fWait, ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
4031	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
4032	# endif
4033
4034	/* It should be a stop request completed message. */
4035	enmVidMsgType = pMappingHeader->enmVidMsgType;
4036	AssertLogRelMsgReturn(enmVidMsgType == VidMessageStopRequestComplete,
4037	("Unexpected 2nd message following ERROR_VID_STOP_PENDING: %#x LB %#x\n",
4038	enmVidMsgType, pMappingHeader->cbMessage),
4039	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
4040
4041	/*
4042	* Mark the VidMessageStopRequestComplete message as handled.
4043	*/
4044	# ifdef IN_RING0
4045	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /ms/);
4046	DBGFTRACE_CUSTOM(pVM, "nemStop#3: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType,
4047	pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
4048	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
4049	("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
4050	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
4051	# else
4052	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /ms/);
4053	DBGFTRACE_CUSTOM(pVM, "nemStop#3: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
4054	pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
4055	AssertLogRelMsgReturn(fWait, ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
4056	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
4057	# endif
4058	Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict) ));
4059	}
4060	else
4061	{
4062	/** @todo I'm not so sure about this now... */
4063	DBGFTRACE_CUSTOM(pVM, "nemStop#9: %#x %#x %#x", pMappingHeader->enmVidMsgType,
4064	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
4065	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPendingOdd);
4066	Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc) - 1st VidMessageSlotHandleAndGetNext got VidMessageStopRequestComplete.\n",
4067	VBOXSTRICTRC_VAL(rcStrict) ));
4068	}
4069	return rcStrict;
4070	}
4071	#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
4072
4073	#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3)
4074
4075	/**
4076	* Deals with pending interrupt related force flags, may inject interrupt.
4077	*
4078	* @returns VBox strict status code.
4079	* @param pVM The cross context VM structure.
4080	* @param pVCpu The cross context per CPU structure.
4081	* @param pfInterruptWindows Where to return interrupt window flags.
4082	*/
4083	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleInterruptFF(PVMCC pVM, PVMCPUCC pVCpu, uint8_t *pfInterruptWindows)
4084	{
4085	Assert(!TRPMHasTrap(pVCpu));
4086	RT_NOREF_PV(pVM);
4087
4088	/*
4089	* First update APIC. We ASSUME this won't need TPR/CR8.
4090	*/
4091	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
4092	{
4093	APICUpdatePendingInterrupts(pVCpu);
4094	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
4095	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4096	return VINF_SUCCESS;
4097	}
4098
4099	/*
4100	* We don't currently implement SMIs.
4101	*/
4102	AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
4103
4104	/*
4105	* Check if we've got the minimum of state required for deciding whether we
4106	* can inject interrupts and NMIs. If we don't have it, get all we might require
4107	* for injection via IEM.
4108	*/
4109	bool const fPendingNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
4110	uint64_t fNeedExtrn = CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
4111	\| (fPendingNmi ? CPUMCTX_EXTRN_INHIBIT_NMI : 0);
4112	if (pVCpu->cpum.GstCtx.fExtrn & fNeedExtrn)
4113	{
4114	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "IntFF");
4115	if (rcStrict != VINF_SUCCESS)
4116	return rcStrict;
4117	}
4118	bool const fInhibitInterrupts = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
4119	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip;
4120
4121	/*
4122	* NMI? Try deliver it first.
4123	*/
4124	if (fPendingNmi)
4125	{
4126	if ( !fInhibitInterrupts
4127	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
4128	{
4129	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
4130	if (rcStrict == VINF_SUCCESS)
4131	{
4132	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
4133	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_NMI, TRPM_HARDWARE_INT, 0, 0, 0);
4134	Log8(("Injected NMI on %u (%d)\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4135	}
4136	return rcStrict;
4137	}
4138	*pfInterruptWindows \|= NEM_WIN_INTW_F_NMI;
4139	Log8(("NMI window pending on %u\n", pVCpu->idCpu));
4140	}
4141
4142	/*
4143	* APIC or PIC interrupt?
4144	*/
4145	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
4146	{
4147	if ( !fInhibitInterrupts
4148	&& pVCpu->cpum.GstCtx.rflags.Bits.u1IF)
4149	{
4150	AssertCompile(NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT & CPUMCTX_EXTRN_APIC_TPR);
4151	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
4152	if (rcStrict == VINF_SUCCESS)
4153	{
4154	uint8_t bInterrupt;
4155	int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
4156	if (RT_SUCCESS(rc))
4157	{
4158	Log8(("Injecting interrupt %#x on %u: %04x:%08RX64 efl=%#x\n", bInterrupt, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags));
4159	rcStrict = IEMInjectTrap(pVCpu, bInterrupt, TRPM_HARDWARE_INT, 0, 0, 0);
4160	Log8(("Injected interrupt %#x on %u (%d)\n", bInterrupt, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4161	}
4162	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
4163	{
4164	*pfInterruptWindows \|= ((bInterrupt >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT) \| NEM_WIN_INTW_F_REGULAR;
4165	Log8(("VERR_APIC_INTR_MASKED_BY_TPR: pfInterruptWindows=%#x\n", pfInterruptWindows));
4166	}
4167	else
4168	Log8(("PDMGetInterrupt failed -> %Rrc\n", rc));
4169	}
4170	return rcStrict;
4171	}
4172
4173	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC) && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC))
4174	{
4175	/* If only an APIC interrupt is pending, we need to know its priority. Otherwise we'll
4176	* likely get pointless deliverability notifications with IF=1 but TPR still too high.
4177	*/
4178	bool fPendingIntr = false;
4179	uint8_t bTpr = 0;
4180	uint8_t bPendingIntr = 0;
4181	int rc = APICGetTpr(pVCpu, &bTpr, &fPendingIntr, &bPendingIntr);
4182	AssertRC(rc);
4183	*pfInterruptWindows \|= (bPendingIntr >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT;
4184	Log8(("Interrupt window pending on %u: %#x (bTpr=%#x fPendingIntr=%d bPendingIntr=%#x)\n",
4185	pVCpu->idCpu, *pfInterruptWindows, bTpr, fPendingIntr, bPendingIntr));
4186	}
4187	else
4188	{
4189	*pfInterruptWindows \|= NEM_WIN_INTW_F_REGULAR;
4190	Log8(("Interrupt window pending on %u: %#x\n", pVCpu->idCpu, *pfInterruptWindows));
4191	}
4192	}
4193
4194	return VINF_SUCCESS;
4195	}
4196
4197
4198	/**
4199	* Inner NEM runloop for windows.
4200	*
4201	* @returns Strict VBox status code.
4202	* @param pVM The cross context VM structure.
4203	* @param pVCpu The cross context per CPU structure.
4204	*/
4205	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVMCC pVM, PVMCPUCC pVCpu)
4206	{
4207	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags));
4208	# ifdef LOG_ENABLED
4209	if (LogIs3Enabled())
4210	nemHCWinLogState(pVM, pVCpu);
4211	# endif
4212
4213	/*
4214	* Try switch to NEM runloop state.
4215	*/
4216	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
4217	{ /* likely */ }
4218	else
4219	{
4220	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4221	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
4222	return VINF_SUCCESS;
4223	}
4224
4225	/*
4226	* The run loop.
4227	*
4228	* Current approach to state updating to use the sledgehammer and sync
4229	* everything every time. This will be optimized later.
4230	*/
4231	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4232	VID_MESSAGE_MAPPING_HEADER volatile pMappingHeader = (VID_MESSAGE_MAPPING_HEADER volatile )pVCpu->nem.s.pvMsgSlotMapping;
4233	# endif
4234	const bool fSingleStepping = DBGFIsStepping(pVCpu);
4235	// const uint32_t fCheckVmFFs = !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK
4236	// : VM_FF_HP_R0_PRE_HM_STEP_MASK;
4237	// const uint32_t fCheckCpuFFs = !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK;
4238	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4239	for (unsigned iLoop = 0;; iLoop++)
4240	{
4241	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) && !defined(VBOX_WITH_PGM_NEM_MODE)
4242	/*
4243	* Hack alert!
4244	*/
4245	uint32_t const cMappedPages = pVM->nem.s.cMappedPages;
4246	if (cMappedPages < pVM->nem.s.cMaxMappedPages)
4247	{ /* likely*/ }
4248	else
4249	{
4250	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemHCWinUnmapOnePageCallback, NULL);
4251	Log(("nemHCWinRunGC: Unmapped all; cMappedPages=%u -> %u\n", cMappedPages, pVM->nem.s.cMappedPages));
4252	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapAllPages);
4253	}
4254	# endif
4255
4256	/*
4257	* Pending interrupts or such? Need to check and deal with this prior
4258	* to the state syncing.
4259	*/
4260	pVCpu->nem.s.fDesiredInterruptWindows = 0;
4261	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_UPDATE_APIC \| VMCPU_FF_INTERRUPT_PIC
4262	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4263	{
4264	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4265	/* Make sure the CPU isn't executing. */
4266	if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
4267	{
4268	pVCpu->nem.s.fHandleAndGetFlags = 0;
4269	rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader);
4270	if (rcStrict == VINF_SUCCESS)
4271	{ /* likely */ }
4272	else
4273	{
4274	LogFlow(("NEM/%u: breaking: nemHCWinStopCpu -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4275	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4276	break;
4277	}
4278	}
4279	# endif
4280
4281	/* Try inject interrupt. */
4282	rcStrict = nemHCWinHandleInterruptFF(pVM, pVCpu, &pVCpu->nem.s.fDesiredInterruptWindows);
4283	if (rcStrict == VINF_SUCCESS)
4284	{ /* likely */ }
4285	else
4286	{
4287	LogFlow(("NEM/%u: breaking: nemHCWinHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4288	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4289	break;
4290	}
4291	}
4292
4293	# ifndef NEM_WIN_WITH_A20
4294	/*
4295	* Do not execute in hyper-V if the A20 isn't enabled.
4296	*/
4297	if (PGMPhysIsA20Enabled(pVCpu))
4298	{ /* likely */ }
4299	else
4300	{
4301	rcStrict = VINF_EM_RESCHEDULE_REM;
4302	LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
4303	break;
4304	}
4305	# endif
4306
4307	/*
4308	* Ensure that hyper-V has the whole state.
4309	* (We always update the interrupt windows settings when active as hyper-V seems
4310	* to forget about it after an exit.)
4311	*/
4312	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK))
4313	!= (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK)
4314	\|\| ( ( pVCpu->nem.s.fDesiredInterruptWindows
4315	\|\| pVCpu->nem.s.fCurrentInterruptWindows != pVCpu->nem.s.fDesiredInterruptWindows)
4316	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4317	&& pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */
4318	# endif
4319	)
4320	)
4321	{
4322	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4323	AssertMsg(pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */,
4324	("%#x fExtrn=%#RX64 (%#RX64) fDesiredInterruptWindows=%d fCurrentInterruptWindows=%#x vs %#x\n",
4325	pVCpu->nem.s.fHandleAndGetFlags, pVCpu->cpum.GstCtx.fExtrn, ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK),
4326	pVCpu->nem.s.fDesiredInterruptWindows, pVCpu->nem.s.fCurrentInterruptWindows, pVCpu->nem.s.fDesiredInterruptWindows));
4327	# endif
4328	# ifdef IN_RING0
4329	int rc2 = nemR0WinExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx);
4330	# else
4331	int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu);
4332	# endif
4333	AssertRCReturn(rc2, rc2);
4334	}
4335
4336	/*
4337	* Poll timers and run for a bit.
4338	*
4339	* With the VID approach (ring-0 or ring-3) we can specify a timeout here,
4340	* so we take the time of the next timer event and uses that as a deadline.
4341	* The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
4342	*/
4343	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
4344	* the whole polling job when timers have changed... */
4345	uint64_t offDeltaIgnored;
4346	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
4347	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4348	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4349	{
4350	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4351	if (pVCpu->nem.s.fHandleAndGetFlags)
4352	{ /* Very likely that the CPU does NOT need starting (pending msg, running). */ }
4353	else
4354	{
4355	# ifdef IN_RING0
4356	pVCpu->nem.s.uIoCtlBuf.idCpu = pVCpu->idCpu;
4357	NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlStartVirtualProcessor.uFunction,
4358	&pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
4359	NULL, 0);
4360	LogFlow(("NEM/%u: IoCtlStartVirtualProcessor -> %#x\n", pVCpu->idCpu, rcNt));
4361	AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("VidStartVirtualProcessor failed for CPU #%u: %#x\n", pVCpu->idCpu, rcNt),
4362	VERR_NEM_IPE_5);
4363	# else
4364	AssertLogRelMsgReturn(g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
4365	("VidStartVirtualProcessor failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
4366	pVCpu->idCpu, RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()),
4367	VERR_NEM_IPE_5);
4368	# endif
4369	pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
4370	}
4371	# endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
4372
4373	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
4374	{
4375	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4376	uint64_t const nsNow = RTTimeNanoTS();
4377	int64_t const cNsNextTimerEvt = nsNow - nsNextTimerEvt;
4378	uint32_t cMsWait;
4379	if (cNsNextTimerEvt < 100000 /* ns */)
4380	cMsWait = 0;
4381	else if ((uint64_t)cNsNextTimerEvt < RT_NS_1SEC)
4382	{
4383	if ((uint32_t)cNsNextTimerEvt < 2*RT_NS_1MS)
4384	cMsWait = 1;
4385	else
4386	cMsWait = ((uint32_t)cNsNextTimerEvt - 100000 /ns/) / RT_NS_1MS;
4387	}
4388	else
4389	cMsWait = RT_MS_1SEC;
4390	# ifdef IN_RING0
4391	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pVCpu->idCpu;
4392	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = pVCpu->nem.s.fHandleAndGetFlags;
4393	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMsWait;
4394	NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
4395	&pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
4396	pVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
4397	NULL, 0);
4398	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4399	if (rcNt == STATUS_SUCCESS)
4400	# else
4401	BOOL fRet = VidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
4402	pVCpu->nem.s.fHandleAndGetFlags, cMsWait);
4403	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4404	if (fRet)
4405	# endif
4406	# else
4407	# ifdef LOG_ENABLED
4408	if (LogIsFlowEnabled())
4409	{
4410	static const WHV_REGISTER_NAME s_aNames[6] = { WHvX64RegisterCs, WHvX64RegisterRip, WHvX64RegisterRflags,
4411	WHvX64RegisterSs, WHvX64RegisterRsp, WHvX64RegisterCr0 };
4412	WHV_REGISTER_VALUE aRegs[RT_ELEMENTS(s_aNames)] = {0};
4413	WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, s_aNames, RT_ELEMENTS(s_aNames), aRegs);
4414	LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
4415	pVCpu->idCpu, aRegs[0].Segment.Selector, aRegs[1].Reg64, RT_BOOL(aRegs[2].Reg64 & X86_EFL_IF),
4416	aRegs[2].Reg64, aRegs[3].Segment.Selector, aRegs[4].Reg64, aRegs[5].Reg64));
4417	}
4418	# endif
4419	WHV_RUN_VP_EXIT_CONTEXT ExitReason = {0};
4420	TMNotifyStartOfExecution(pVM, pVCpu);
4421
4422	HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
4423
4424	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4425	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
4426	# ifdef LOG_ENABLED
4427	LogFlow(("NEM/%u: Exit @ %04X:%08RX64 IF=%d CR8=%#x Reason=%#x\n", pVCpu->idCpu, ExitReason.VpContext.Cs.Selector,
4428	ExitReason.VpContext.Rip, RT_BOOL(ExitReason.VpContext.Rflags & X86_EFL_IF), ExitReason.VpContext.Cr8,
4429	ExitReason.ExitReason));
4430	# endif
4431	if (SUCCEEDED(hrc))
4432	# endif
4433	{
4434	/*
4435	* Deal with the message.
4436	*/
4437	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4438	rcStrict = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader);
4439	pVCpu->nem.s.fHandleAndGetFlags \|= VID_MSHAGN_F_HANDLE_MESSAGE;
4440	# else
4441	rcStrict = nemR3WinHandleExit(pVM, pVCpu, &ExitReason);
4442	# endif
4443	if (rcStrict == VINF_SUCCESS)
4444	{ /* hopefully likely */ }
4445	else
4446	{
4447	LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4448	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4449	break;
4450	}
4451	}
4452	else
4453	{
4454	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4455
4456	/* VID.SYS merges STATUS_ALERTED and STATUS_USER_APC into STATUS_TIMEOUT,
4457	so after NtAlertThread we end up here with a STATUS_TIMEOUT. And yeah,
4458	the error code conversion is into WAIT_XXX, i.e. NT status codes. */
4459	# ifndef IN_RING0
4460	DWORD rcNt = GetLastError();
4461	# endif
4462	LogFlow(("NEM/%u: VidMessageSlotHandleAndGetNext -> %#x\n", pVCpu->idCpu, rcNt));
4463	AssertLogRelMsgReturn( rcNt == STATUS_TIMEOUT
4464	\|\| rcNt == STATUS_ALERTED /* just in case */
4465	\|\| rcNt == STATUS_USER_APC /* ditto */
4466	\|\| rcNt == STATUS_KERNEL_APC /* ditto */
4467	, ("VidMessageSlotHandleAndGetNext failed for CPU #%u: %#x (%u)\n",
4468	pVCpu->idCpu, rcNt, rcNt),
4469	VERR_NEM_IPE_0);
4470	pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
4471	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatGetMsgTimeout);
4472	# else
4473	AssertLogRelMsgFailedReturn(("WHvRunVirtualProcessor failed for CPU #%u: %#x (%u)\n",
4474	pVCpu->idCpu, hrc, GetLastError()),
4475	VERR_NEM_IPE_0);
4476	# endif
4477	}
4478
4479	/*
4480	* If no relevant FFs are pending, loop.
4481	*/
4482	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
4483	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4484	continue;
4485
4486	/** @todo Try handle pending flags, not just return to EM loops. Take care
4487	* not to set important RCs here unless we've handled a message. */
4488	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
4489	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
4490	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
4491	}
4492	else
4493	{
4494	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
4495	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
4496	}
4497	}
4498	else
4499	{
4500	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
4501	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
4502	}
4503	break;
4504	} /* the run loop */
4505
4506
4507	/*
4508	* If the CPU is running, make sure to stop it before we try sync back the
4509	* state and return to EM. We don't sync back the whole state if we can help it.
4510	*/
4511	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4512	if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
4513	{
4514	pVCpu->nem.s.fHandleAndGetFlags = 0;
4515	rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader);
4516	}
4517	# endif
4518
4519	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
4520	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4521
4522	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
4523	{
4524	/* Try anticipate what we might need. */
4525	uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI;
4526	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
4527	\|\| RT_FAILURE(rcStrict))
4528	fImport = CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
4529	# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
4530	else if ( rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
4531	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
4532	fImport = CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_INHIBIT_INT;
4533	else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
4534	fImport = CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_INHIBIT_INT;
4535	# endif
4536	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
4537	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4538	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
4539
4540	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
4541	{
4542	# ifdef IN_RING0
4543	int rc2 = nemR0WinImportState(pVM, pVCpu, &pVCpu->cpum.GstCtx, fImport \| CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT,
4544	true /fCanUpdateCr3/);
4545	if (RT_SUCCESS(rc2))
4546	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4547	else if (rc2 == VERR_NEM_FLUSH_TLB)
4548	{
4549	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4550	if (rcStrict == VINF_SUCCESS \|\| rcStrict == -rc2)
4551	rcStrict = -rc2;
4552	else
4553	{
4554	pVCpu->nem.s.rcPending = -rc2;
4555	LogFlow(("NEM/%u: rcPending=%Rrc (rcStrict=%Rrc)\n", pVCpu->idCpu, rc2, VBOXSTRICTRC_VAL(rcStrict) ));
4556	}
4557	}
4558	# else
4559	int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, fImport \| CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
4560	if (RT_SUCCESS(rc2))
4561	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4562	# endif
4563	else if (RT_SUCCESS(rcStrict))
4564	rcStrict = rc2;
4565	if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
4566	pVCpu->cpum.GstCtx.fExtrn = 0;
4567	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
4568	}
4569	else
4570	{
4571	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4572	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
4573	}
4574	}
4575	else
4576	{
4577	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4578	pVCpu->cpum.GstCtx.fExtrn = 0;
4579	}
4580
4581	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
4582	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
4583	return rcStrict;
4584	}
4585
4586	#endif /* defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3) */
4587	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING3)
4588
4589	/**
4590	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
4591	*/
4592	NEM_TMPL_STATIC DECLCALLBACK(int) nemHCWinUnsetForA20CheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys,
4593	PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
4594	{
4595	/* We'll just unmap the memory. */
4596	if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
4597	{
4598	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4599	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
4600	AssertRC(rc);
4601	if (RT_SUCCESS(rc))
4602	# else
4603	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
4604	if (SUCCEEDED(hrc))
4605	# endif
4606	{
4607	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
4608	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4609	Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
4610	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
4611	}
4612	else
4613	{
4614	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
4615	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4616	LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
4617	return rc;
4618	# else
4619	LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4620	GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4621	return VERR_NEM_IPE_2;
4622	# endif
4623	}
4624	}
4625	RT_NOREF(pVCpu, pvUser);
4626	return VINF_SUCCESS;
4627	}
4628
4629
4630	/**
4631	* Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
4632	*
4633	* @returns The PGMPhysNemQueryPageInfo result.
4634	* @param pVM The cross context VM structure.
4635	* @param pVCpu The cross context virtual CPU structure.
4636	* @param GCPhys The page to unmap.
4637	*/
4638	NEM_TMPL_STATIC int nemHCWinUnmapPageForA20Gate(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys)
4639	{
4640	PGMPHYSNEMPAGEINFO Info;
4641	return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /fMakeWritable/, &Info,
4642	nemHCWinUnsetForA20CheckerCallback, NULL);
4643	}
4644
4645	#endif /* defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING3) */
4646
4647	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
4648	{
4649	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
4650	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
4651	}
4652
4653
4654	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
4655	RTR3PTR pvMemR3, uint8_t *pu2State)
4656	{
4657	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
4658	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
4659
4660	*pu2State = UINT8_MAX;
4661	#if !defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) && defined(VBOX_WITH_PGM_NEM_MODE) && defined(IN_RING3)
4662	if (pvMemR3)
4663	{
4664	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a);
4665	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvMemR3, GCPhys, cb,
4666	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute \| WHvMapGpaRangeFlagWrite);
4667	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a);
4668	if (SUCCEEDED(hrc))
4669	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4670	else
4671	AssertLogRelMsgFailed(("NEMHCNotifyHandlerPhysicalDeregister: WHvMapGpaRange(,%p,%RGp,%RGp,) -> %Rhrc\n",
4672	pvMemR3, GCPhys, cb, hrc));
4673	}
4674	RT_NOREF(enmKind);
4675	#else
4676	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
4677	#endif
4678	}
4679
4680
4681	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
4682	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
4683	{
4684	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
4685	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
4686	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
4687	}
4688
4689
4690	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING3)
4691	/**
4692	* Worker that maps pages into Hyper-V.
4693	*
4694	* This is used by the PGM physical page notifications as well as the memory
4695	* access VMEXIT handlers.
4696	*
4697	* @returns VBox status code.
4698	* @param pVM The cross context VM structure.
4699	* @param pVCpu The cross context virtual CPU structure of the
4700	* calling EMT.
4701	* @param GCPhysSrc The source page address.
4702	* @param GCPhysDst The hyper-V destination page. This may differ from
4703	* GCPhysSrc when A20 is disabled.
4704	* @param fPageProt NEM_PAGE_PROT_XXX.
4705	* @param pu2State Our page state (input/output).
4706	* @param fBackingChanged Set if the page backing is being changed.
4707	* @thread EMT(pVCpu)
4708	*/
4709	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
4710	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
4711	{
4712	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4713	/*
4714	* When using the hypercalls instead of the ring-3 APIs, we don't need to
4715	* unmap memory before modifying it. We still want to track the state though,
4716	* since unmap will fail when called an unmapped page and we don't want to redo
4717	* upgrades/downgrades.
4718	*/
4719	uint8_t const u2OldState = *pu2State;
4720	int rc;
4721	if (fPageProt == NEM_PAGE_PROT_NONE)
4722	{
4723	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
4724	{
4725	rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4726	if (RT_SUCCESS(rc))
4727	{
4728	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4729	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
4730	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4731	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4732	}
4733	else
4734	{
4735	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
4736	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4737	}
4738	}
4739	else
4740	rc = VINF_SUCCESS;
4741	}
4742	else if (fPageProt & NEM_PAGE_PROT_WRITE)
4743	{
4744	if (u2OldState != NEM_WIN_PAGE_STATE_WRITABLE \|\| fBackingChanged)
4745	{
4746	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4747	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
4748	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4749	if (RT_SUCCESS(rc))
4750	{
4751	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4752	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
4753	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
4754	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
4755	Log5(("NEM GPA writable/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4756	NOREF(cMappedPages);
4757	}
4758	else
4759	{
4760	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
4761	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4762	}
4763	}
4764	else
4765	rc = VINF_SUCCESS;
4766	}
4767	else
4768	{
4769	if (u2OldState != NEM_WIN_PAGE_STATE_READABLE \|\| fBackingChanged)
4770	{
4771	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4772	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4773	if (RT_SUCCESS(rc))
4774	{
4775	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4776	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
4777	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
4778	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
4779	Log5(("NEM GPA read+exec/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4780	NOREF(cMappedPages);
4781	}
4782	else
4783	{
4784	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
4785	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4786	}
4787	}
4788	else
4789	rc = VINF_SUCCESS;
4790	}
4791
4792	return VINF_SUCCESS;
4793
4794	# else /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
4795	/*
4796	* Looks like we need to unmap a page before we can change the backing
4797	* or even modify the protection. This is going to be REALLY efficient.
4798	* PGM lends us two bits to keep track of the state here.
4799	*/
4800	RT_NOREF(pVCpu);
4801	uint8_t const u2OldState = *pu2State;
4802	uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
4803	: fPageProt & NEM_PAGE_PROT_READ ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED;
4804	if ( fBackingChanged
4805	\|\| u2NewState != u2OldState)
4806	{
4807	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
4808	{
4809	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4810	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4811	AssertRC(rc);
4812	if (RT_SUCCESS(rc))
4813	{
4814	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4815	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
4816	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4817	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
4818	{
4819	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
4820	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4821	return VINF_SUCCESS;
4822	}
4823	}
4824	else
4825	{
4826	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
4827	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4828	return rc;
4829	}
4830	# else
4831	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
4832	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE);
4833	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
4834	if (SUCCEEDED(hrc))
4835	{
4836	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4837	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
4838	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4839	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
4840	{
4841	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
4842	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4843	return VINF_SUCCESS;
4844	}
4845	}
4846	else
4847	{
4848	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
4849	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4850	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4851	return VERR_NEM_INIT_FAILED;
4852	}
4853	# endif
4854	}
4855	}
4856
4857	/*
4858	* Writeable mapping?
4859	*/
4860	if (fPageProt & NEM_PAGE_PROT_WRITE)
4861	{
4862	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4863	int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4864	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
4865	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4866	AssertRC(rc);
4867	if (RT_SUCCESS(rc))
4868	{
4869	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4870	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
4871	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4872	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4873	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4874	return VINF_SUCCESS;
4875	}
4876	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
4877	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4878	return rc;
4879	# else
4880	void *pvPage;
4881	int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
4882	if (RT_SUCCESS(rc))
4883	{
4884	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE,
4885	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute \| WHvMapGpaRangeFlagWrite);
4886	if (SUCCEEDED(hrc))
4887	{
4888	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4889	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
4890	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4891	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4892	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4893	return VINF_SUCCESS;
4894	}
4895	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
4896	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4897	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4898	return VERR_NEM_INIT_FAILED;
4899	}
4900	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
4901	return rc;
4902	# endif
4903	}
4904
4905	if (fPageProt & NEM_PAGE_PROT_READ)
4906	{
4907	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4908	int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4909	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4910	AssertRC(rc);
4911	if (RT_SUCCESS(rc))
4912	{
4913	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4914	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
4915	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4916	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4917	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4918	return VINF_SUCCESS;
4919	}
4920	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
4921	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4922	return rc;
4923	# else
4924	const void *pvPage;
4925	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
4926	if (RT_SUCCESS(rc))
4927	{
4928	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRangePage, a);
4929	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE,
4930	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute);
4931	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRangePage, a);
4932	if (SUCCEEDED(hrc))
4933	{
4934	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4935	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
4936	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4937	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4938	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4939	return VINF_SUCCESS;
4940	}
4941	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
4942	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4943	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4944	return VERR_NEM_INIT_FAILED;
4945	}
4946	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
4947	return rc;
4948	# endif
4949	}
4950
4951	/* We already unmapped it above. */
4952	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4953	return VINF_SUCCESS;
4954	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
4955	}
4956	#endif /* defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING3) */
4957
4958
4959	NEM_TMPL_STATIC int nemHCJustUnmapPageFromHyperV(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
4960	{
4961	if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED)
4962	{
4963	Log5(("nemHCJustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst));
4964	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4965	return VINF_SUCCESS;
4966	}
4967
4968	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES)
4969	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4970	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4971	AssertRC(rc);
4972	if (RT_SUCCESS(rc))
4973	{
4974	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
4975	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4976	Log5(("NEM GPA unmapped/just: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[*pu2State], cMappedPages));
4977	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4978	return VINF_SUCCESS;
4979	}
4980	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
4981	LogRel(("nemHCJustUnmapPageFromHyperV/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4982	return rc;
4983
4984	#elif defined(IN_RING3)
4985	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
4986	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
4987	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
4988	if (SUCCEEDED(hrc))
4989	{
4990	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
4991	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4992	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4993	Log5(("nemHCJustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
4994	return VINF_SUCCESS;
4995	}
4996	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
4997	LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n",
4998	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4999	return VERR_NEM_IPE_6;
5000	#else
5001	RT_NOREF(pVM, GCPhysDst, pu2State);
5002	LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): Why are we here?!?\n", GCPhysDst));
5003	return VERR_NEM_IPE_6;
5004	#endif
5005	}
5006
5007
5008	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
5009	PGMPAGETYPE enmType, uint8_t *pu2State)
5010	{
5011	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
5012	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
5013	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
5014
5015	int rc;
5016	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
5017	PVMCPUCC pVCpu = VMMGetCpu(pVM);
5018	# ifdef NEM_WIN_WITH_A20
5019	if ( pVM->nem.s.fA20Enabled
5020	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
5021	# endif
5022	rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
5023	# ifdef NEM_WIN_WITH_A20
5024	else
5025	{
5026	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
5027	rc = nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
5028	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys) && RT_SUCCESS(rc))
5029	rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
5030
5031	}
5032	# endif
5033	#else
5034	RT_NOREF_PV(fPageProt);
5035	# ifdef NEM_WIN_WITH_A20
5036	if ( pVM->nem.s.fA20Enabled
5037	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
5038	# endif
5039	rc = nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
5040	# ifdef NEM_WIN_WITH_A20
5041	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
5042	rc = nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
5043	else
5044	rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */
5045	# endif
5046	#endif
5047	return rc;
5048	}
5049
5050
5051	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
5052	PGMPAGETYPE enmType, uint8_t *pu2State)
5053	{
5054	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
5055	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
5056	Assert(VM_IS_NEM_ENABLED(pVM));
5057	RT_NOREF(HCPhys, enmType, pvR3);
5058
5059	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
5060	PVMCPUCC pVCpu = VMMGetCpu(pVM);
5061	# ifdef NEM_WIN_WITH_A20
5062	if ( pVM->nem.s.fA20Enabled
5063	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
5064	# endif
5065	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
5066	# ifdef NEM_WIN_WITH_A20
5067	else
5068	{
5069	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
5070	nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
5071	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
5072	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
5073	}
5074	# endif
5075	#else
5076	RT_NOREF_PV(fPageProt);
5077	# ifdef NEM_WIN_WITH_A20
5078	if ( pVM->nem.s.fA20Enabled
5079	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
5080	# endif
5081	nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
5082	# ifdef NEM_WIN_WITH_A20
5083	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
5084	nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
5085	/* else: ignore since we've got the alias page at this address. */
5086	# endif
5087	#endif
5088	}
5089
5090
5091	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
5092	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
5093	{
5094	Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp pvNewR3=%p fPageProt=%#x enmType=%d *pu2State=%d\n",
5095	GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, *pu2State));
5096	Assert(VM_IS_NEM_ENABLED(pVM));
5097	RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, enmType);
5098
5099	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
5100	PVMCPUCC pVCpu = VMMGetCpu(pVM);
5101	# ifdef NEM_WIN_WITH_A20
5102	if ( pVM->nem.s.fA20Enabled
5103	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
5104	# endif
5105	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
5106	# ifdef NEM_WIN_WITH_A20
5107	else
5108	{
5109	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
5110	nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
5111	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
5112	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
5113	}
5114	# endif
5115	#else
5116	RT_NOREF_PV(fPageProt);
5117	# ifdef NEM_WIN_WITH_A20
5118	if ( pVM->nem.s.fA20Enabled
5119	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
5120	# endif
5121	nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
5122	# ifdef NEM_WIN_WITH_A20
5123	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
5124	nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
5125	/* else: ignore since we've got the alias page at this address. */
5126	# endif
5127	#endif
5128	}
5129
5130
5131	/**
5132	* Returns features supported by the NEM backend.
5133	*
5134	* @returns Flags of features supported by the native NEM backend.
5135	* @param pVM The cross context VM structure.
5136	*/
5137	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
5138	{
5139	RT_NOREF(pVM);
5140	/** @todo Make use of the WHvGetVirtualProcessorXsaveState/WHvSetVirtualProcessorXsaveState
5141	* interface added in 2019 to enable passthrough of xsave/xrstor (and depending) features to the guest. */
5142	/** @todo Is NEM_FEAT_F_FULL_GST_EXEC always true? */
5143	return NEM_FEAT_F_NESTED_PAGING \| NEM_FEAT_F_FULL_GST_EXEC;
5144	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h@ 92561

Download in other formats: