NEMR3Native-win.cpp@ 71136

Last change on this file since 71136 was 71136, checked in by vboxsync, 7 years ago
SUPDrv,VMM/NEM/win: Abstracted kernel I/O control work in SUPDrv, currently only implemented & needed on windows. Made first I/O control call to VID from ring-0. bugref:9044
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 172.8 KB

Line
1	/* $Id: NEMR3Native-win.cpp 71136 2018-02-27 13:17:36Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 Windows backend.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	* Log group 6: Ring-0 memory management
9	* Log group 12: API intercepts.
10	*/
11
12	/*
13	* Copyright (C) 2018 Oracle Corporation
14	*
15	* This file is part of VirtualBox Open Source Edition (OSE), as
16	* available from http://www.virtualbox.org. This file is free software;
17	* you can redistribute it and/or modify it under the terms of the GNU
18	* General Public License (GPL) as published by the Free Software
19	* Foundation, in version 2 as it comes in the "COPYING" file of the
20	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22	*/
23
24
25	/*********************************************************************************************************************************
26	* Header Files *
27	*********************************************************************************************************************************/
28	#define LOG_GROUP LOG_GROUP_NEM
29	#include <iprt/nt/nt-and-windows.h>
30	#include <iprt/nt/hyperv.h>
31	#include <iprt/nt/vid.h>
32	#include <WinHvPlatform.h>
33
34	#ifndef _WIN32_WINNT_WIN10
35	# error "Missing _WIN32_WINNT_WIN10"
36	#endif
37	#ifndef _WIN32_WINNT_WIN10_RS1 /* Missing define, causing trouble for us. */
38	# define _WIN32_WINNT_WIN10_RS1 (_WIN32_WINNT_WIN10 + 1)
39	#endif
40	#include <sysinfoapi.h>
41	#include <debugapi.h>
42	#include <errhandlingapi.h>
43	#include <fileapi.h>
44	#include <winerror.h> /* no api header for this. */
45
46	#include <VBox/vmm/nem.h>
47	#include <VBox/vmm/iem.h>
48	#include <VBox/vmm/em.h>
49	#include <VBox/vmm/apic.h>
50	#include "NEMInternal.h"
51	#include <VBox/vmm/vm.h>
52
53	#include <iprt/ldr.h>
54	#include <iprt/path.h>
55	#include <iprt/string.h>
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	#ifdef LOG_ENABLED
62	# define NEM_WIN_INTERCEPT_NT_IO_CTLS
63	#endif
64
65	/** @name Our two-bit physical page state for PGMPAGE
66	* @{ */
67	#define NEM_WIN_PAGE_STATE_NOT_SET 0
68	#define NEM_WIN_PAGE_STATE_UNMAPPED 1
69	#define NEM_WIN_PAGE_STATE_READABLE 2
70	#define NEM_WIN_PAGE_STATE_WRITABLE 3
71	/** @} */
72
73	/** Checks if a_GCPhys is subject to the limited A20 gate emulation. */
74	#define NEM_WIN_IS_SUBJECT_TO_A20(a_GCPhys) ((RTGCPHYS)((a_GCPhys) - _1M) < (RTGCPHYS)_64K)
75
76	/** Checks if a_GCPhys is relevant to the limited A20 gate emulation. */
77	#define NEM_WIN_IS_RELEVANT_TO_A20(a_GCPhys) \
78	( ((RTGCPHYS)((a_GCPhys) - _1M) < (RTGCPHYS)_64K) \|\| ((RTGCPHYS)(a_GCPhys) < (RTGCPHYS)_64K) )
79
80	/** VID I/O control detection: Fake partition handle input. */
81	#define NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE ((HANDLE)(uintptr_t)38479125)
82	/** VID I/O control detection: Fake partition ID return. */
83	#define NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID UINT64_C(0xfa1e000042424242)
84	/** VID I/O control detection: Fake CPU index input. */
85	#define NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX UINT32_C(42)
86	/** VID I/O control detection: Fake timeout input. */
87	#define NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT UINT32_C(0x00080286)
88
89
90	/*********************************************************************************************************************************
91	* Global Variables *
92	*********************************************************************************************************************************/
93	/** @name APIs imported from WinHvPlatform.dll
94	* @{ */
95	static decltype(WHvGetCapability) * g_pfnWHvGetCapability;
96	static decltype(WHvCreatePartition) * g_pfnWHvCreatePartition;
97	static decltype(WHvSetupPartition) * g_pfnWHvSetupPartition;
98	static decltype(WHvDeletePartition) * g_pfnWHvDeletePartition;
99	static decltype(WHvGetPartitionProperty) * g_pfnWHvGetPartitionProperty;
100	static decltype(WHvSetPartitionProperty) * g_pfnWHvSetPartitionProperty;
101	static decltype(WHvMapGpaRange) * g_pfnWHvMapGpaRange;
102	static decltype(WHvUnmapGpaRange) * g_pfnWHvUnmapGpaRange;
103	static decltype(WHvTranslateGva) * g_pfnWHvTranslateGva;
104	#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
105	static decltype(WHvCreateVirtualProcessor) * g_pfnWHvCreateVirtualProcessor;
106	static decltype(WHvDeleteVirtualProcessor) * g_pfnWHvDeleteVirtualProcessor;
107	static decltype(WHvRunVirtualProcessor) * g_pfnWHvRunVirtualProcessor;
108	static decltype(WHvGetRunExitContextSize) * g_pfnWHvGetRunExitContextSize;
109	static decltype(WHvCancelRunVirtualProcessor) * g_pfnWHvCancelRunVirtualProcessor;
110	static decltype(WHvGetVirtualProcessorRegisters) * g_pfnWHvGetVirtualProcessorRegisters;
111	static decltype(WHvSetVirtualProcessorRegisters) * g_pfnWHvSetVirtualProcessorRegisters;
112	#endif
113	/** @} */
114
115	/** @name APIs imported from Vid.dll
116	* @{ */
117	static decltype(VidGetHvPartitionId) *g_pfnVidGetHvPartitionId;
118	static decltype(VidStartVirtualProcessor) *g_pfnVidStartVirtualProcessor;
119	static decltype(VidStopVirtualProcessor) *g_pfnVidStopVirtualProcessor;
120	static decltype(VidMessageSlotMap) *g_pfnVidMessageSlotMap;
121	static decltype(VidMessageSlotHandleAndGetNext) *g_pfnVidMessageSlotHandleAndGetNext;
122	#ifdef LOG_ENABLED
123	static decltype(VidGetVirtualProcessorState) *g_pfnVidGetVirtualProcessorState;
124	static decltype(VidSetVirtualProcessorState) *g_pfnVidSetVirtualProcessorState;
125	static decltype(VidGetVirtualProcessorRunningStatus) *g_pfnVidGetVirtualProcessorRunningStatus;
126	#endif
127	/** @} */
128
129
130	/**
131	* Import instructions.
132	*/
133	static const struct
134	{
135	uint8_t idxDll; /*< 0 for WinHvPlatform.dll, 1 for vid.dll. /
136	bool fOptional; /*< Set if import is optional. /
137	PFNRT ppfn; /< The function pointer variable. /
138	const char pszName; /< The function name. /
139	} g_aImports[] =
140	{
141	#define NEM_WIN_IMPORT(a_idxDll, a_fOptional, a_Name) { (a_idxDll), (a_fOptional), (PFNRT *)&RT_CONCAT(g_pfn,a_Name), #a_Name }
142	NEM_WIN_IMPORT(0, false, WHvGetCapability),
143	NEM_WIN_IMPORT(0, false, WHvCreatePartition),
144	NEM_WIN_IMPORT(0, false, WHvSetupPartition),
145	NEM_WIN_IMPORT(0, false, WHvDeletePartition),
146	NEM_WIN_IMPORT(0, false, WHvGetPartitionProperty),
147	NEM_WIN_IMPORT(0, false, WHvSetPartitionProperty),
148	NEM_WIN_IMPORT(0, false, WHvMapGpaRange),
149	NEM_WIN_IMPORT(0, false, WHvUnmapGpaRange),
150	NEM_WIN_IMPORT(0, false, WHvTranslateGva),
151	#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
152	NEM_WIN_IMPORT(0, false, WHvCreateVirtualProcessor),
153	NEM_WIN_IMPORT(0, false, WHvDeleteVirtualProcessor),
154	NEM_WIN_IMPORT(0, false, WHvRunVirtualProcessor),
155	NEM_WIN_IMPORT(0, false, WHvCancelRunVirtualProcessor),
156	NEM_WIN_IMPORT(0, false, WHvGetRunExitContextSize),
157	NEM_WIN_IMPORT(0, false, WHvGetVirtualProcessorRegisters),
158	NEM_WIN_IMPORT(0, false, WHvSetVirtualProcessorRegisters),
159	#endif
160	NEM_WIN_IMPORT(1, false, VidGetHvPartitionId),
161	NEM_WIN_IMPORT(1, false, VidMessageSlotMap),
162	NEM_WIN_IMPORT(1, false, VidMessageSlotHandleAndGetNext),
163	NEM_WIN_IMPORT(1, false, VidStartVirtualProcessor),
164	NEM_WIN_IMPORT(1, false, VidStopVirtualProcessor),
165	#ifdef LOG_ENABLED
166	NEM_WIN_IMPORT(1, false, VidGetVirtualProcessorState),
167	NEM_WIN_IMPORT(1, false, VidSetVirtualProcessorState),
168	NEM_WIN_IMPORT(1, false, VidGetVirtualProcessorRunningStatus),
169	#endif
170	#undef NEM_WIN_IMPORT
171	};
172
173
174	/** The real NtDeviceIoControlFile API in NTDLL. */
175	static decltype(NtDeviceIoControlFile) *g_pfnNtDeviceIoControlFile;
176	/** Pointer to the NtDeviceIoControlFile import table entry. */
177	static decltype(NtDeviceIoControlFile) **g_ppfnVidNtDeviceIoControlFile;
178	/** Info about the VidGetHvPartitionId I/O control interface. */
179	static NEMWINIOCTL g_IoCtlGetHvPartitionId;
180	/** Info about the VidStartVirtualProcessor I/O control interface. */
181	static NEMWINIOCTL g_IoCtlStartVirtualProcessor;
182	/** Info about the VidStopVirtualProcessor I/O control interface. */
183	static NEMWINIOCTL g_IoCtlStopVirtualProcessor;
184	/** Info about the VidMessageSlotHandleAndGetNext I/O control interface. */
185	static NEMWINIOCTL g_IoCtlMessageSlotHandleAndGetNext;
186	#ifdef LOG_ENABLED
187	/** Info about the VidMessageSlotMap I/O control interface - for logging. */
188	static NEMWINIOCTL g_IoCtlMessageSlotMap;
189	/* Info about the VidGetVirtualProcessorState I/O control interface - for logging. */
190	static NEMWINIOCTL g_IoCtlGetVirtualProcessorState;
191	/* Info about the VidSetVirtualProcessorState I/O control interface - for logging. */
192	static NEMWINIOCTL g_IoCtlSetVirtualProcessorState;
193	/** Pointer to what nemR3WinIoctlDetector_ForLogging should fill in. */
194	static NEMWINIOCTL *g_pIoCtlDetectForLogging;
195	#endif
196
197	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
198	/** Mapping slot for CPU #0.
199	* @{ */
200	static VID_MESSAGE_MAPPING_HEADER *g_pMsgSlotMapping = NULL;
201	static const HV_MESSAGE_HEADER *g_pHvMsgHdr;
202	static const HV_X64_INTERCEPT_MESSAGE_HEADER *g_pX64MsgHdr;
203	/** @} */
204	#endif
205
206
207	/*
208	* Let the preprocessor alias the APIs to import variables for better autocompletion.
209	*/
210	#ifndef IN_SLICKEDIT
211	# define WHvGetCapability g_pfnWHvGetCapability
212	# define WHvCreatePartition g_pfnWHvCreatePartition
213	# define WHvSetupPartition g_pfnWHvSetupPartition
214	# define WHvDeletePartition g_pfnWHvDeletePartition
215	# define WHvGetPartitionProperty g_pfnWHvGetPartitionProperty
216	# define WHvSetPartitionProperty g_pfnWHvSetPartitionProperty
217	# define WHvMapGpaRange g_pfnWHvMapGpaRange
218	# define WHvUnmapGpaRange g_pfnWHvUnmapGpaRange
219	# define WHvTranslateGva g_pfnWHvTranslateGva
220	# define WHvCreateVirtualProcessor g_pfnWHvCreateVirtualProcessor
221	# define WHvDeleteVirtualProcessor g_pfnWHvDeleteVirtualProcessor
222	# define WHvRunVirtualProcessor g_pfnWHvRunVirtualProcessor
223	# define WHvGetRunExitContextSize g_pfnWHvGetRunExitContextSize
224	# define WHvCancelRunVirtualProcessor g_pfnWHvCancelRunVirtualProcessor
225	# define WHvGetVirtualProcessorRegisters g_pfnWHvGetVirtualProcessorRegisters
226	# define WHvSetVirtualProcessorRegisters g_pfnWHvSetVirtualProcessorRegisters
227	#endif
228
229	/** NEM_WIN_PAGE_STATE_XXX names. */
230	static const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
231	/** WHV_MEMORY_ACCESS_TYPE names */
232	static const char * const g_apszWHvMemAccesstypes[4] = { "read", "write", "exec", "!undefined!" };
233
234
235	/*********************************************************************************************************************************
236	* Internal Functions *
237	*********************************************************************************************************************************/
238	static int nemR3NativeSetPhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fPageProt,
239	uint8_t *pu2State, bool fBackingChanged);
240
241
242
243	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
244	/**
245	* Wrapper that logs the call from VID.DLL.
246	*
247	* This is very handy for figuring out why an API call fails.
248	*/
249	static NTSTATUS WINAPI
250	nemR3WinLogWrapper_NtDeviceIoControlFile(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
251	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
252	PVOID pvOutput, ULONG cbOutput)
253	{
254
255	char szFunction[32];
256	const char *pszFunction;
257	if (uFunction == g_IoCtlMessageSlotHandleAndGetNext.uFunction)
258	pszFunction = "VidMessageSlotHandleAndGetNext";
259	else if (uFunction == g_IoCtlStartVirtualProcessor.uFunction)
260	pszFunction = "VidStartVirtualProcessor";
261	else if (uFunction == g_IoCtlStopVirtualProcessor.uFunction)
262	pszFunction = "VidStopVirtualProcessor";
263	else if (uFunction == g_IoCtlMessageSlotMap.uFunction)
264	pszFunction = "VidMessageSlotMap";
265	else if (uFunction == g_IoCtlGetVirtualProcessorState.uFunction)
266	pszFunction = "VidGetVirtualProcessorState";
267	else if (uFunction == g_IoCtlSetVirtualProcessorState.uFunction)
268	pszFunction = "VidSetVirtualProcessorState";
269	else
270	{
271	RTStrPrintf(szFunction, sizeof(szFunction), "%#x", uFunction);
272	pszFunction = szFunction;
273	}
274
275	if (cbInput > 0 && pvInput)
276	Log12(("VID!NtDeviceIoControlFile: %s/input: %.*Rhxs\n", pszFunction, RT_MIN(cbInput, 32), pvInput));
277	NTSTATUS rcNt = g_pfnNtDeviceIoControlFile(hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, uFunction,
278	pvInput, cbInput, pvOutput, cbOutput);
279	if (!hEvt && !pfnApcCallback && !pvApcCtx)
280	Log12(("VID!NtDeviceIoControlFile: hFile=%#zx pIos=%p->{s:%#x, i:%#zx} uFunction=%s Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
281	hFile, pIos, pIos->Status, pIos->Information, pszFunction, pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
282	else
283	Log12(("VID!NtDeviceIoControlFile: hFile=%#zx hEvt=%#zx Apc=%p/%p pIos=%p->{s:%#x, i:%#zx} uFunction=%s Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
284	hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, pIos->Status, pIos->Information, pszFunction,
285	pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
286	if (cbOutput > 0 && pvOutput)
287	{
288	Log12(("VID!NtDeviceIoControlFile: %s/output: %.*Rhxs\n", pszFunction, RT_MIN(cbOutput, 32), pvOutput));
289	if (uFunction == 0x2210cc && g_pMsgSlotMapping == NULL && cbOutput >= sizeof(void *))
290	{
291	g_pMsgSlotMapping = (VID_MESSAGE_MAPPING_HEADER *)pvOutput;
292	g_pHvMsgHdr = (const HV_MESSAGE_HEADER *)(g_pMsgSlotMapping + 1);
293	g_pX64MsgHdr = (const HV_X64_INTERCEPT_MESSAGE_HEADER *)(g_pHvMsgHdr + 1);
294	Log12(("VID!NtDeviceIoControlFile: Message slot mapping: %p\n", g_pMsgSlotMapping));
295	}
296	}
297	if ( g_pMsgSlotMapping
298	&& ( uFunction == g_IoCtlMessageSlotHandleAndGetNext.uFunction
299	\|\| uFunction == g_IoCtlStopVirtualProcessor.uFunction
300	\|\| uFunction == g_IoCtlMessageSlotMap.uFunction
301	))
302	Log12(("VID!NtDeviceIoControlFile: enmVidMsgType=%#x cb=%#x msg=%#x payload=%u cs:rip=%04x:%08RX64 (%s)\n",
303	g_pMsgSlotMapping->enmVidMsgType, g_pMsgSlotMapping->cbMessage,
304	g_pHvMsgHdr->MessageType, g_pHvMsgHdr->PayloadSize,
305	g_pX64MsgHdr->CsSegment.Selector, g_pX64MsgHdr->Rip, pszFunction));
306
307	return rcNt;
308	}
309	#endif /* NEM_WIN_INTERCEPT_NT_IO_CTLS */
310
311
312	/**
313	* Patches the call table of VID.DLL so we can intercept NtDeviceIoControlFile.
314	*
315	* This is for used to figure out the I/O control codes and in logging builds
316	* for logging API calls that WinHvPlatform.dll does.
317	*
318	* @returns VBox status code.
319	* @param hLdrModVid The VID module handle.
320	* @param pErrInfo Where to return additional error information.
321	*/
322	static int nemR3WinInitVidIntercepts(RTLDRMOD hLdrModVid, PRTERRINFO pErrInfo)
323	{
324	/*
325	* Locate the real API.
326	*/
327	g_pfnNtDeviceIoControlFile = (decltype(NtDeviceIoControlFile) *)RTLdrGetSystemSymbol("NTDLL.DLL", "NtDeviceIoControlFile");
328	AssertReturn(g_pfnNtDeviceIoControlFile != NULL,
329	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Failed to resolve NtDeviceIoControlFile from NTDLL.DLL"));
330
331	/*
332	* Locate the PE header and get what we need from it.
333	*/
334	uint8_t const pbImage = (uint8_t const )RTLdrGetNativeHandle(hLdrModVid);
335	IMAGE_DOS_HEADER const pMzHdr = (IMAGE_DOS_HEADER const )pbImage;
336	AssertReturn(pMzHdr->e_magic == IMAGE_DOS_SIGNATURE,
337	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL mapping doesn't start with MZ signature: %#x", pMzHdr->e_magic));
338	IMAGE_NT_HEADERS const pNtHdrs = (IMAGE_NT_HEADERS const )&pbImage[pMzHdr->e_lfanew];
339	AssertReturn(pNtHdrs->Signature == IMAGE_NT_SIGNATURE,
340	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL has invalid PE signaturre: %#x @%#x",
341	pNtHdrs->Signature, pMzHdr->e_lfanew));
342
343	uint32_t const cbImage = pNtHdrs->OptionalHeader.SizeOfImage;
344	IMAGE_DATA_DIRECTORY const ImportDir = pNtHdrs->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
345
346	/*
347	* Walk the import descriptor table looking for NTDLL.DLL.
348	*/
349	AssertReturn( ImportDir.Size > 0
350	&& ImportDir.Size < cbImage,
351	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory size: %#x", ImportDir.Size));
352	AssertReturn( ImportDir.VirtualAddress > 0
353	&& ImportDir.VirtualAddress <= cbImage - ImportDir.Size,
354	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory RVA: %#x", ImportDir.VirtualAddress));
355
356	for (PIMAGE_IMPORT_DESCRIPTOR pImps = (PIMAGE_IMPORT_DESCRIPTOR)&pbImage[ImportDir.VirtualAddress];
357	pImps->Name != 0 && pImps->FirstThunk != 0;
358	pImps++)
359	{
360	AssertReturn(pImps->Name < cbImage,
361	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory entry name: %#x", pImps->Name));
362	const char pszModName = (const char )&pbImage[pImps->Name];
363	if (RTStrICmpAscii(pszModName, "ntdll.dll"))
364	continue;
365	AssertReturn(pImps->FirstThunk < cbImage,
366	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
367	AssertReturn(pImps->OriginalFirstThunk < cbImage,
368	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
369
370	/*
371	* Walk the thunks table(s) looking for NtDeviceIoControlFile.
372	*/
373	PIMAGE_THUNK_DATA pFirstThunk = (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]; /* update this. */
374	PIMAGE_THUNK_DATA pThunk = pImps->OriginalFirstThunk == 0 /* read from this. */
375	? (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]
376	: (PIMAGE_THUNK_DATA)&pbImage[pImps->OriginalFirstThunk];
377	while (pThunk->u1.Ordinal != 0)
378	{
379	if (!(pThunk->u1.Ordinal & IMAGE_ORDINAL_FLAG32))
380	{
381	AssertReturn(pThunk->u1.Ordinal > 0 && pThunk->u1.Ordinal < cbImage,
382	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
383
384	const char pszSymbol = (const char )&pbImage[(uintptr_t)pThunk->u1.AddressOfData + 2];
385	if (strcmp(pszSymbol, "NtDeviceIoControlFile") == 0)
386	{
387	DWORD fOldProt = PAGE_READONLY;
388	VirtualProtect(&pFirstThunk->u1.Function, sizeof(uintptr_t), PAGE_EXECUTE_READWRITE, &fOldProt);
389	g_ppfnVidNtDeviceIoControlFile = (decltype(NtDeviceIoControlFile) **)&pFirstThunk->u1.Function;
390	/* Don't restore the protection here, so we modify the NtDeviceIoControlFile pointer later. */
391	}
392	}
393
394	pThunk++;
395	pFirstThunk++;
396	}
397	}
398
399	if (*g_ppfnVidNtDeviceIoControlFile)
400	{
401	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
402	*g_ppfnVidNtDeviceIoControlFile = nemR3WinLogWrapper_NtDeviceIoControlFile;
403	#endif
404	return VINF_SUCCESS;
405	}
406	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Failed to patch NtDeviceIoControlFile import in VID.DLL!");
407	}
408
409
410	/**
411	* Worker for nemR3NativeInit that probes and load the native API.
412	*
413	* @returns VBox status code.
414	* @param fForced Whether the HMForced flag is set and we should
415	* fail if we cannot initialize.
416	* @param pErrInfo Where to always return error info.
417	*/
418	static int nemR3WinInitProbeAndLoad(bool fForced, PRTERRINFO pErrInfo)
419	{
420	/*
421	* Check that the DLL files we need are present, but without loading them.
422	* We'd like to avoid loading them unnecessarily.
423	*/
424	WCHAR wszPath[MAX_PATH + 64];
425	UINT cwcPath = GetSystemDirectoryW(wszPath, MAX_PATH);
426	if (cwcPath >= MAX_PATH \|\| cwcPath < 2)
427	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "GetSystemDirectoryW failed (%#x / %u)", cwcPath, GetLastError());
428
429	if (wszPath[cwcPath - 1] != '\\' \|\| wszPath[cwcPath - 1] != '/')
430	wszPath[cwcPath++] = '\\';
431	RTUtf16CopyAscii(&wszPath[cwcPath], RT_ELEMENTS(wszPath) - cwcPath, "WinHvPlatform.dll");
432	if (GetFileAttributesW(wszPath) == INVALID_FILE_ATTRIBUTES)
433	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "The native API dll was not found (%ls)", wszPath);
434
435	/*
436	* Check that we're in a VM and that the hypervisor identifies itself as Hyper-V.
437	*/
438	if (!ASMHasCpuId())
439	return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "No CPUID support");
440	if (!ASMIsValidStdRange(ASMCpuId_EAX(0)))
441	return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "No CPUID leaf #1");
442	if (!(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_HVP))
443	return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Not in a hypervisor partition (HVP=0)");
444
445	uint32_t cMaxHyperLeaf = 0;
446	uint32_t uEbx = 0;
447	uint32_t uEcx = 0;
448	uint32_t uEdx = 0;
449	ASMCpuIdExSlow(0x40000000, 0, 0, 0, &cMaxHyperLeaf, &uEbx, &uEcx, &uEdx);
450	if (!ASMIsValidHypervisorRange(cMaxHyperLeaf))
451	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Invalid hypervisor CPUID range (%#x %#x %#x %#x)",
452	cMaxHyperLeaf, uEbx, uEcx, uEdx);
453	if ( uEbx != UINT32_C(0x7263694d) /* Micr */
454	\|\| uEcx != UINT32_C(0x666f736f) /* osof */
455	\|\| uEdx != UINT32_C(0x76482074) /* t Hv */)
456	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE,
457	"Not Hyper-V CPUID signature: %#x %#x %#x (expected %#x %#x %#x)",
458	uEbx, uEcx, uEdx, UINT32_C(0x7263694d), UINT32_C(0x666f736f), UINT32_C(0x76482074));
459	if (cMaxHyperLeaf < UINT32_C(0x40000005))
460	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Too narrow hypervisor CPUID range (%#x)", cMaxHyperLeaf);
461
462	/** @todo would be great if we could recognize a root partition from the
463	* CPUID info, but I currently don't dare do that. */
464
465	/*
466	* Now try load the DLLs and resolve the APIs.
467	*/
468	static const char * const s_apszDllNames[2] = { "WinHvPlatform.dll", "vid.dll" };
469	RTLDRMOD ahMods[2] = { NIL_RTLDRMOD, NIL_RTLDRMOD };
470	int rc = VINF_SUCCESS;
471	for (unsigned i = 0; i < RT_ELEMENTS(s_apszDllNames); i++)
472	{
473	int rc2 = RTLdrLoadSystem(s_apszDllNames[i], true /fNoUnload/, &ahMods[i]);
474	if (RT_FAILURE(rc2))
475	{
476	if (!RTErrInfoIsSet(pErrInfo))
477	RTErrInfoSetF(pErrInfo, rc2, "Failed to load API DLL: %s: %Rrc", s_apszDllNames[i], rc2);
478	else
479	RTErrInfoAddF(pErrInfo, rc2, "; %s: %Rrc", s_apszDllNames[i], rc2);
480	ahMods[i] = NIL_RTLDRMOD;
481	rc = VERR_NEM_INIT_FAILED;
482	}
483	}
484	if (RT_SUCCESS(rc))
485	rc = nemR3WinInitVidIntercepts(ahMods[1], pErrInfo);
486	if (RT_SUCCESS(rc))
487	{
488	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
489	{
490	int rc2 = RTLdrGetSymbol(ahMods[g_aImports[i].idxDll], g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
491	if (RT_FAILURE(rc2))
492	{
493	*g_aImports[i].ppfn = NULL;
494
495	LogRel(("NEM: %s: Failed to import %s!%s: %Rrc",
496	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
497	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName, rc2));
498	if (!g_aImports[i].fOptional)
499	{
500	if (RTErrInfoIsSet(pErrInfo))
501	RTErrInfoAddF(pErrInfo, rc2, ", %s!%s",
502	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
503	else
504	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: %s!%s",
505	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
506	Assert(RT_FAILURE(rc));
507	}
508	}
509	}
510	if (RT_SUCCESS(rc))
511	{
512	Assert(!RTErrInfoIsSet(pErrInfo));
513	}
514	}
515
516	for (unsigned i = 0; i < RT_ELEMENTS(ahMods); i++)
517	RTLdrClose(ahMods[i]);
518	return rc;
519	}
520
521
522	/**
523	* Worker for nemR3NativeInit that gets the hypervisor capabilities.
524	*
525	* @returns VBox status code.
526	* @param pVM The cross context VM structure.
527	* @param pErrInfo Where to always return error info.
528	*/
529	static int nemR3WinInitCheckCapabilities(PVM pVM, PRTERRINFO pErrInfo)
530	{
531	#define NEM_LOG_REL_CAP_EX(a_szField, a_szFmt, a_Value) LogRel(("NEM: %-38s= " a_szFmt "\n", a_szField, a_Value))
532	#define NEM_LOG_REL_CAP_SUB_EX(a_szField, a_szFmt, a_Value) LogRel(("NEM: %36s: " a_szFmt "\n", a_szField, a_Value))
533	#define NEM_LOG_REL_CAP_SUB(a_szField, a_Value) NEM_LOG_REL_CAP_SUB_EX(a_szField, "%d", a_Value)
534
535	/*
536	* Is the hypervisor present with the desired capability?
537	*
538	* In build 17083 this translates into:
539	* - CPUID[0x00000001].HVP is set
540	* - CPUID[0x40000000] == "Microsoft Hv"
541	* - CPUID[0x40000001].eax == "Hv#1"
542	* - CPUID[0x40000003].ebx[12] is set.
543	* - VidGetExoPartitionProperty(INVALID_HANDLE_VALUE, 0x60000, &Ignored) returns
544	* a non-zero value.
545	*/
546	/**
547	* @todo Someone at Microsoft please explain weird API design:
548	* 1. Pointless CapabilityCode duplication int the output;
549	* 2. No output size.
550	*/
551	WHV_CAPABILITY Caps;
552	RT_ZERO(Caps);
553	SetLastError(0);
554	HRESULT hrc = WHvGetCapability(WHvCapabilityCodeHypervisorPresent, &Caps, sizeof(Caps));
555	DWORD rcWin = GetLastError();
556	if (FAILED(hrc))
557	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
558	"WHvGetCapability/WHvCapabilityCodeHypervisorPresent failed: %Rhrc (Last=%#x/%u)",
559	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
560	if (!Caps.HypervisorPresent)
561	{
562	if (!RTPathExists(RTPATH_NT_PASSTHRU_PREFIX "Device\\VidExo"))
563	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE,
564	"WHvCapabilityCodeHypervisorPresent is FALSE! Make sure you have enabled the 'Windows Hypervisor Platform' feature.");
565	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "WHvCapabilityCodeHypervisorPresent is FALSE! (%u)", rcWin);
566	}
567	LogRel(("NEM: WHvCapabilityCodeHypervisorPresent is TRUE, so this might work...\n"));
568
569
570	/*
571	* Check what extended VM exits are supported.
572	*/
573	RT_ZERO(Caps);
574	hrc = WHvGetCapability(WHvCapabilityCodeExtendedVmExits, &Caps, sizeof(Caps));
575	if (FAILED(hrc))
576	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
577	"WHvGetCapability/WHvCapabilityCodeExtendedVmExits failed: %Rhrc (Last=%#x/%u)",
578	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
579	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeExtendedVmExits", "%'#018RX64", Caps.ExtendedVmExits.AsUINT64);
580	pVM->nem.s.fExtendedMsrExit = RT_BOOL(Caps.ExtendedVmExits.X64MsrExit);
581	pVM->nem.s.fExtendedCpuIdExit = RT_BOOL(Caps.ExtendedVmExits.X64CpuidExit);
582	pVM->nem.s.fExtendedXcptExit = RT_BOOL(Caps.ExtendedVmExits.ExceptionExit);
583	NEM_LOG_REL_CAP_SUB("fExtendedMsrExit", pVM->nem.s.fExtendedMsrExit);
584	NEM_LOG_REL_CAP_SUB("fExtendedCpuIdExit", pVM->nem.s.fExtendedCpuIdExit);
585	NEM_LOG_REL_CAP_SUB("fExtendedXcptExit", pVM->nem.s.fExtendedXcptExit);
586	if (Caps.ExtendedVmExits.AsUINT64 & ~(uint64_t)7)
587	LogRel(("NEM: Warning! Unknown VM exit definitions: %#RX64\n", Caps.ExtendedVmExits.AsUINT64));
588	/** @todo RECHECK: WHV_EXTENDED_VM_EXITS typedef. */
589
590	/*
591	* Check features in case they end up defining any.
592	*/
593	RT_ZERO(Caps);
594	hrc = WHvGetCapability(WHvCapabilityCodeFeatures, &Caps, sizeof(Caps));
595	if (FAILED(hrc))
596	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
597	"WHvGetCapability/WHvCapabilityCodeFeatures failed: %Rhrc (Last=%#x/%u)",
598	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
599	if (Caps.Features.AsUINT64 & ~(uint64_t)0)
600	LogRel(("NEM: Warning! Unknown feature definitions: %#RX64\n", Caps.Features.AsUINT64));
601	/** @todo RECHECK: WHV_CAPABILITY_FEATURES typedef. */
602
603	/*
604	* Check that the CPU vendor is supported.
605	*/
606	RT_ZERO(Caps);
607	hrc = WHvGetCapability(WHvCapabilityCodeProcessorVendor, &Caps, sizeof(Caps));
608	if (FAILED(hrc))
609	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
610	"WHvGetCapability/WHvCapabilityCodeProcessorVendor failed: %Rhrc (Last=%#x/%u)",
611	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
612	switch (Caps.ProcessorVendor)
613	{
614	/** @todo RECHECK: WHV_PROCESSOR_VENDOR typedef. */
615	case WHvProcessorVendorIntel:
616	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d - Intel", Caps.ProcessorVendor);
617	pVM->nem.s.enmCpuVendor = CPUMCPUVENDOR_INTEL;
618	break;
619	case WHvProcessorVendorAmd:
620	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d - AMD", Caps.ProcessorVendor);
621	pVM->nem.s.enmCpuVendor = CPUMCPUVENDOR_AMD;
622	break;
623	default:
624	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d", Caps.ProcessorVendor);
625	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unknown processor vendor: %d", Caps.ProcessorVendor);
626	}
627
628	/*
629	* CPU features, guessing these are virtual CPU features?
630	*/
631	RT_ZERO(Caps);
632	hrc = WHvGetCapability(WHvCapabilityCodeProcessorFeatures, &Caps, sizeof(Caps));
633	if (FAILED(hrc))
634	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
635	"WHvGetCapability/WHvCapabilityCodeProcessorFeatures failed: %Rhrc (Last=%#x/%u)",
636	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
637	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorFeatures", "%'#018RX64", Caps.ProcessorFeatures.AsUINT64);
638	#define NEM_LOG_REL_CPU_FEATURE(a_Field) NEM_LOG_REL_CAP_SUB(#a_Field, Caps.ProcessorFeatures.a_Field)
639	NEM_LOG_REL_CPU_FEATURE(Sse3Support);
640	NEM_LOG_REL_CPU_FEATURE(LahfSahfSupport);
641	NEM_LOG_REL_CPU_FEATURE(Ssse3Support);
642	NEM_LOG_REL_CPU_FEATURE(Sse4_1Support);
643	NEM_LOG_REL_CPU_FEATURE(Sse4_2Support);
644	NEM_LOG_REL_CPU_FEATURE(Sse4aSupport);
645	NEM_LOG_REL_CPU_FEATURE(XopSupport);
646	NEM_LOG_REL_CPU_FEATURE(PopCntSupport);
647	NEM_LOG_REL_CPU_FEATURE(Cmpxchg16bSupport);
648	NEM_LOG_REL_CPU_FEATURE(Altmovcr8Support);
649	NEM_LOG_REL_CPU_FEATURE(LzcntSupport);
650	NEM_LOG_REL_CPU_FEATURE(MisAlignSseSupport);
651	NEM_LOG_REL_CPU_FEATURE(MmxExtSupport);
652	NEM_LOG_REL_CPU_FEATURE(Amd3DNowSupport);
653	NEM_LOG_REL_CPU_FEATURE(ExtendedAmd3DNowSupport);
654	NEM_LOG_REL_CPU_FEATURE(Page1GbSupport);
655	NEM_LOG_REL_CPU_FEATURE(AesSupport);
656	NEM_LOG_REL_CPU_FEATURE(PclmulqdqSupport);
657	NEM_LOG_REL_CPU_FEATURE(PcidSupport);
658	NEM_LOG_REL_CPU_FEATURE(Fma4Support);
659	NEM_LOG_REL_CPU_FEATURE(F16CSupport);
660	NEM_LOG_REL_CPU_FEATURE(RdRandSupport);
661	NEM_LOG_REL_CPU_FEATURE(RdWrFsGsSupport);
662	NEM_LOG_REL_CPU_FEATURE(SmepSupport);
663	NEM_LOG_REL_CPU_FEATURE(EnhancedFastStringSupport);
664	NEM_LOG_REL_CPU_FEATURE(Bmi1Support);
665	NEM_LOG_REL_CPU_FEATURE(Bmi2Support);
666	/* two reserved bits here, see below */
667	NEM_LOG_REL_CPU_FEATURE(MovbeSupport);
668	NEM_LOG_REL_CPU_FEATURE(Npiep1Support);
669	NEM_LOG_REL_CPU_FEATURE(DepX87FPUSaveSupport);
670	NEM_LOG_REL_CPU_FEATURE(RdSeedSupport);
671	NEM_LOG_REL_CPU_FEATURE(AdxSupport);
672	NEM_LOG_REL_CPU_FEATURE(IntelPrefetchSupport);
673	NEM_LOG_REL_CPU_FEATURE(SmapSupport);
674	NEM_LOG_REL_CPU_FEATURE(HleSupport);
675	NEM_LOG_REL_CPU_FEATURE(RtmSupport);
676	NEM_LOG_REL_CPU_FEATURE(RdtscpSupport);
677	NEM_LOG_REL_CPU_FEATURE(ClflushoptSupport);
678	NEM_LOG_REL_CPU_FEATURE(ClwbSupport);
679	NEM_LOG_REL_CPU_FEATURE(ShaSupport);
680	NEM_LOG_REL_CPU_FEATURE(X87PointersSavedSupport);
681	#undef NEM_LOG_REL_CPU_FEATURE
682	if (Caps.ProcessorFeatures.AsUINT64 & (~(RT_BIT_64(43) - 1) \| RT_BIT_64(27) \| RT_BIT_64(28)))
683	LogRel(("NEM: Warning! Unknown CPU features: %#RX64\n", Caps.ProcessorFeatures.AsUINT64));
684	pVM->nem.s.uCpuFeatures.u64 = Caps.ProcessorFeatures.AsUINT64;
685	/** @todo RECHECK: WHV_PROCESSOR_FEATURES typedef. */
686
687	/*
688	* The cache line flush size.
689	*/
690	RT_ZERO(Caps);
691	hrc = WHvGetCapability(WHvCapabilityCodeProcessorClFlushSize, &Caps, sizeof(Caps));
692	if (FAILED(hrc))
693	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
694	"WHvGetCapability/WHvCapabilityCodeProcessorClFlushSize failed: %Rhrc (Last=%#x/%u)",
695	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
696	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorClFlushSize", "2^%u", Caps.ProcessorClFlushSize);
697	if (Caps.ProcessorClFlushSize < 8 && Caps.ProcessorClFlushSize > 9)
698	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unsupported cache line flush size: %u", Caps.ProcessorClFlushSize);
699	pVM->nem.s.cCacheLineFlushShift = Caps.ProcessorClFlushSize;
700
701	/*
702	* See if they've added more properties that we're not aware of.
703	*/
704	/** @todo RECHECK: WHV_CAPABILITY_CODE typedef. */
705	if (!IsDebuggerPresent()) /* Too noisy when in debugger, so skip. */
706	{
707	static const struct
708	{
709	uint32_t iMin, iMax; } s_aUnknowns[] =
710	{
711	{ 0x0003, 0x000f },
712	{ 0x1003, 0x100f },
713	{ 0x2000, 0x200f },
714	{ 0x3000, 0x300f },
715	{ 0x4000, 0x400f },
716	};
717	for (uint32_t j = 0; j < RT_ELEMENTS(s_aUnknowns); j++)
718	for (uint32_t i = s_aUnknowns[j].iMin; i <= s_aUnknowns[j].iMax; i++)
719	{
720	RT_ZERO(Caps);
721	hrc = WHvGetCapability((WHV_CAPABILITY_CODE)i, &Caps, sizeof(Caps));
722	if (SUCCEEDED(hrc))
723	LogRel(("NEM: Warning! Unknown capability %#x returning: %.*Rhxs\n", i, sizeof(Caps), &Caps));
724	}
725	}
726
727	#undef NEM_LOG_REL_CAP_EX
728	#undef NEM_LOG_REL_CAP_SUB_EX
729	#undef NEM_LOG_REL_CAP_SUB
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Used to fill in g_IoCtlGetHvPartitionId.
736	*/
737	static NTSTATUS WINAPI
738	nemR3WinIoctlDetector_GetHvPartitionId(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
739	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
740	PVOID pvOutput, ULONG cbOutput)
741	{
742	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
743	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
744	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
745	AssertLogRelMsgReturn(cbInput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
746	RT_NOREF(pvInput);
747
748	AssertLogRelMsgReturn(RT_VALID_PTR(pvOutput), ("pvOutput=%p\n", pvOutput), STATUS_INVALID_PARAMETER_9);
749	AssertLogRelMsgReturn(cbOutput == sizeof(HV_PARTITION_ID), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
750	(HV_PARTITION_ID )pvOutput = NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID;
751
752	g_IoCtlGetHvPartitionId.cbInput = cbInput;
753	g_IoCtlGetHvPartitionId.cbOutput = cbOutput;
754	g_IoCtlGetHvPartitionId.uFunction = uFunction;
755
756	return STATUS_SUCCESS;
757	}
758
759
760	/**
761	* Used to fill in g_IoCtlStartVirtualProcessor.
762	*/
763	static NTSTATUS WINAPI
764	nemR3WinIoctlDetector_StartVirtualProcessor(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
765	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
766	PVOID pvOutput, ULONG cbOutput)
767	{
768	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
769	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
770	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
771	AssertLogRelMsgReturn(cbInput == sizeof(HV_VP_INDEX), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
772	AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
773	AssertLogRelMsgReturn((HV_VP_INDEX )pvInput == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
774	("piCpu=%u\n", (HV_VP_INDEX *)pvInput), STATUS_INVALID_PARAMETER_9);
775	AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
776	RT_NOREF(pvOutput);
777
778	g_IoCtlStartVirtualProcessor.cbInput = cbInput;
779	g_IoCtlStartVirtualProcessor.cbOutput = cbOutput;
780	g_IoCtlStartVirtualProcessor.uFunction = uFunction;
781
782	return STATUS_SUCCESS;
783	}
784
785
786	/**
787	* Used to fill in g_IoCtlStartVirtualProcessor.
788	*/
789	static NTSTATUS WINAPI
790	nemR3WinIoctlDetector_StopVirtualProcessor(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
791	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
792	PVOID pvOutput, ULONG cbOutput)
793	{
794	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
795	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
796	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
797	AssertLogRelMsgReturn(cbInput == sizeof(HV_VP_INDEX), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
798	AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
799	AssertLogRelMsgReturn((HV_VP_INDEX )pvInput == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
800	("piCpu=%u\n", (HV_VP_INDEX *)pvInput), STATUS_INVALID_PARAMETER_9);
801	AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
802	RT_NOREF(pvOutput);
803
804	g_IoCtlStopVirtualProcessor.cbInput = cbInput;
805	g_IoCtlStopVirtualProcessor.cbOutput = cbOutput;
806	g_IoCtlStopVirtualProcessor.uFunction = uFunction;
807
808	return STATUS_SUCCESS;
809	}
810
811
812	/**
813	* Used to fill in g_IoCtlMessageSlotHandleAndGetNext
814	*/
815	static NTSTATUS WINAPI
816	nemR3WinIoctlDetector_MessageSlotHandleAndGetNext(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
817	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
818	PVOID pvOutput, ULONG cbOutput)
819	{
820	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
821	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
822	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
823
824	AssertLogRelMsgReturn(cbInput == sizeof(VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT), ("cbInput=%#x\n", cbInput),
825	STATUS_INVALID_PARAMETER_8);
826	AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
827	PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT pVidIn = (PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT)pvInput;
828	AssertLogRelMsgReturn( pVidIn->iCpu == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX
829	&& pVidIn->fFlags == VID_MSHAGN_F_HANDLE_MESSAGE
830	&& pVidIn->cMillies == NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT,
831	("iCpu=%u fFlags=%#x cMillies=%#x\n", pVidIn->iCpu, pVidIn->fFlags, pVidIn->cMillies),
832	STATUS_INVALID_PARAMETER_9);
833	AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
834	RT_NOREF(pvOutput);
835
836	g_IoCtlMessageSlotHandleAndGetNext.cbInput = cbInput;
837	g_IoCtlMessageSlotHandleAndGetNext.cbOutput = cbOutput;
838	g_IoCtlMessageSlotHandleAndGetNext.uFunction = uFunction;
839
840	return STATUS_SUCCESS;
841	}
842
843
844	#ifdef LOG_ENABLED
845	/**
846	* Used to fill in what g_pIoCtlDetectForLogging points to.
847	*/
848	static NTSTATUS WINAPI nemR3WinIoctlDetector_ForLogging(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
849	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
850	PVOID pvOutput, ULONG cbOutput)
851	{
852	RT_NOREF(hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, pvInput, pvOutput);
853
854	g_pIoCtlDetectForLogging->cbInput = cbInput;
855	g_pIoCtlDetectForLogging->cbOutput = cbOutput;
856	g_pIoCtlDetectForLogging->uFunction = uFunction;
857
858	return STATUS_SUCCESS;
859	}
860	#endif
861
862
863	/**
864	* Worker for nemR3NativeInit that detect I/O control function numbers for VID.
865	*
866	* We use the function numbers directly in ring-0 and to name functions when
867	* logging NtDeviceIoControlFile calls.
868	*
869	* @note We could alternatively do this by disassembling the respective
870	* functions, but hooking NtDeviceIoControlFile and making fake calls
871	* more easily provides the desired information.
872	*
873	* @returns VBox status code.
874	* @param pVM The cross context VM structure. Will set I/O
875	* control info members.
876	* @param pErrInfo Where to always return error info.
877	*/
878	static int nemR3WinInitDiscoverIoControlProperties(PVM pVM, PRTERRINFO pErrInfo)
879	{
880	/*
881	* Probe the I/O control information for select VID APIs so we can use
882	* them directly from ring-0 and better log them.
883	*
884	*/
885	decltype(NtDeviceIoControlFile) * const pfnOrg = *g_ppfnVidNtDeviceIoControlFile;
886
887	/* VidGetHvPartitionId */
888	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_GetHvPartitionId;
889	HV_PARTITION_ID idHvPartition = HV_PARTITION_ID_INVALID;
890	BOOL fRet = g_pfnVidGetHvPartitionId(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, &idHvPartition);
891	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
892	AssertReturn(fRet && idHvPartition == NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID && g_IoCtlGetHvPartitionId.uFunction != 0,
893	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
894	"Problem figuring out VidGetHvPartitionId: fRet=%u idHvPartition=%#x dwErr=%u",
895	fRet, idHvPartition, GetLastError()) );
896	LogRel(("NEM: VidGetHvPartitionId -> fun:%#x in:%#x out:%#x\n",
897	g_IoCtlGetHvPartitionId.uFunction, g_IoCtlGetHvPartitionId.cbInput, g_IoCtlGetHvPartitionId.cbOutput));
898
899	/* VidStartVirtualProcessor */
900	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_StartVirtualProcessor;
901	fRet = g_pfnVidStartVirtualProcessor(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
902	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
903	AssertReturn(fRet && g_IoCtlStartVirtualProcessor.uFunction != 0,
904	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
905	"Problem figuring out VidStartVirtualProcessor: fRet=%u dwErr=%u",
906	fRet, GetLastError()) );
907	LogRel(("NEM: VidStartVirtualProcessor -> fun:%#x in:%#x out:%#x\n", g_IoCtlStartVirtualProcessor.uFunction,
908	g_IoCtlStartVirtualProcessor.cbInput, g_IoCtlStartVirtualProcessor.cbOutput));
909
910	/* VidStopVirtualProcessor */
911	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_StopVirtualProcessor;
912	fRet = g_pfnVidStopVirtualProcessor(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
913	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
914	AssertReturn(fRet && g_IoCtlStopVirtualProcessor.uFunction != 0,
915	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
916	"Problem figuring out VidStopVirtualProcessor: fRet=%u dwErr=%u",
917	fRet, GetLastError()) );
918	LogRel(("NEM: VidStopVirtualProcessor -> fun:%#x in:%#x out:%#x\n", g_IoCtlStopVirtualProcessor.uFunction,
919	g_IoCtlStopVirtualProcessor.cbInput, g_IoCtlStopVirtualProcessor.cbOutput));
920
921	/* VidMessageSlotHandleAndGetNext */
922	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_MessageSlotHandleAndGetNext;
923	fRet = g_pfnVidMessageSlotHandleAndGetNext(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE,
924	NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX, VID_MSHAGN_F_HANDLE_MESSAGE,
925	NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT);
926	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
927	AssertReturn(fRet && g_IoCtlMessageSlotHandleAndGetNext.uFunction != 0,
928	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
929	"Problem figuring out VidMessageSlotHandleAndGetNext: fRet=%u dwErr=%u",
930	fRet, GetLastError()) );
931	LogRel(("NEM: VidMessageSlotHandleAndGetNext -> fun:%#x in:%#x out:%#x\n",
932	g_IoCtlMessageSlotHandleAndGetNext.uFunction, g_IoCtlMessageSlotHandleAndGetNext.cbInput,
933	g_IoCtlMessageSlotHandleAndGetNext.cbOutput));
934
935	#ifdef LOG_ENABLED
936	/* The following are only for logging: */
937	union
938	{
939	VID_MAPPED_MESSAGE_SLOT MapSlot;
940	HV_REGISTER_NAME Name;
941	HV_REGISTER_VALUE Value;
942	} uBuf;
943
944	/* VidMessageSlotMap */
945	g_pIoCtlDetectForLogging = &g_IoCtlMessageSlotMap;
946	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
947	fRet = g_pfnVidMessageSlotMap(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, &uBuf.MapSlot, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
948	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
949	Assert(fRet);
950	LogRel(("NEM: VidMessageSlotMap -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
951	g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
952
953	/* VidGetVirtualProcessorState */
954	uBuf.Name = HvRegisterExplicitSuspend;
955	g_pIoCtlDetectForLogging = &g_IoCtlGetVirtualProcessorState;
956	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
957	fRet = g_pfnVidGetVirtualProcessorState(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
958	&uBuf.Name, 1, &uBuf.Value);
959	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
960	Assert(fRet);
961	LogRel(("NEM: VidGetVirtualProcessorState -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
962	g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
963
964	/* VidSetVirtualProcessorState */
965	uBuf.Name = HvRegisterExplicitSuspend;
966	g_pIoCtlDetectForLogging = &g_IoCtlSetVirtualProcessorState;
967	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
968	fRet = g_pfnVidSetVirtualProcessorState(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
969	&uBuf.Name, 1, &uBuf.Value);
970	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
971	Assert(fRet);
972	LogRel(("NEM: VidSetVirtualProcessorState -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
973	g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
974
975	g_pIoCtlDetectForLogging = NULL;
976	#endif
977
978	/* Done. */
979	pVM->nem.s.IoCtlGetHvPartitionId = g_IoCtlGetHvPartitionId;
980	pVM->nem.s.IoCtlStartVirtualProcessor = g_IoCtlStartVirtualProcessor;
981	pVM->nem.s.IoCtlStopVirtualProcessor = g_IoCtlStopVirtualProcessor;
982	pVM->nem.s.IoCtlMessageSlotHandleAndGetNext = g_IoCtlMessageSlotHandleAndGetNext;
983	return VINF_SUCCESS;
984	}
985
986
987	/**
988	* Creates and sets up a Hyper-V (exo) partition.
989	*
990	* @returns VBox status code.
991	* @param pVM The cross context VM structure.
992	* @param pErrInfo Where to always return error info.
993	*/
994	static int nemR3WinInitCreatePartition(PVM pVM, PRTERRINFO pErrInfo)
995	{
996	AssertReturn(!pVM->nem.s.hPartition, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
997	AssertReturn(!pVM->nem.s.hPartitionDevice, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
998
999	/*
1000	* Create the partition.
1001	*/
1002	WHV_PARTITION_HANDLE hPartition;
1003	HRESULT hrc = WHvCreatePartition(&hPartition);
1004	if (FAILED(hrc))
1005	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "WHvCreatePartition failed with %Rhrc (Last=%#x/%u)",
1006	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1007
1008	int rc;
1009
1010	/*
1011	* Set partition properties, most importantly the CPU count.
1012	*/
1013	/**
1014	* @todo Someone at Microsoft please explain another weird API:
1015	* - Why this API doesn't take the WHV_PARTITION_PROPERTY_CODE value as an
1016	* argument rather than as part of the struct. That is so weird if you've
1017	* used any other NT or windows API, including WHvGetCapability().
1018	* - Why use PVOID when WHV_PARTITION_PROPERTY is what's expected. We
1019	* technically only need 9 bytes for setting/getting
1020	* WHVPartitionPropertyCodeProcessorClFlushSize, but the API insists on 16. */
1021	WHV_PARTITION_PROPERTY Property;
1022	RT_ZERO(Property);
1023	Property.PropertyCode = WHvPartitionPropertyCodeProcessorCount;
1024	Property.ProcessorCount = pVM->cCpus;
1025	hrc = WHvSetPartitionProperty(hPartition, &Property, sizeof(Property));
1026	if (SUCCEEDED(hrc))
1027	{
1028	RT_ZERO(Property);
1029	Property.PropertyCode = WHvPartitionPropertyCodeExtendedVmExits;
1030	Property.ExtendedVmExits.X64CpuidExit = pVM->nem.s.fExtendedCpuIdExit;
1031	Property.ExtendedVmExits.X64MsrExit = pVM->nem.s.fExtendedMsrExit;
1032	Property.ExtendedVmExits.ExceptionExit = pVM->nem.s.fExtendedXcptExit;
1033	hrc = WHvSetPartitionProperty(hPartition, &Property, sizeof(Property));
1034	if (SUCCEEDED(hrc))
1035	{
1036	/*
1037	* We'll continue setup in nemR3NativeInitAfterCPUM.
1038	*/
1039	pVM->nem.s.fCreatedEmts = false;
1040	pVM->nem.s.hPartition = hPartition;
1041	LogRel(("NEM: Created partition %p.\n", hPartition));
1042	return VINF_SUCCESS;
1043	}
1044
1045	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
1046	"Failed setting WHvPartitionPropertyCodeExtendedVmExits to %'#RX64: %Rhrc",
1047	Property.ExtendedVmExits.AsUINT64, hrc);
1048	}
1049	else
1050	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
1051	"Failed setting WHvPartitionPropertyCodeProcessorCount to %u: %Rhrc (Last=%#x/%u)",
1052	pVM->cCpus, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1053	WHvDeletePartition(hPartition);
1054
1055	Assert(!pVM->nem.s.hPartitionDevice);
1056	Assert(!pVM->nem.s.hPartition);
1057	return rc;
1058	}
1059
1060
1061	/**
1062	* Try initialize the native API.
1063	*
1064	* This may only do part of the job, more can be done in
1065	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
1066	*
1067	* @returns VBox status code.
1068	* @param pVM The cross context VM structure.
1069	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
1070	* the latter we'll fail if we cannot initialize.
1071	* @param fForced Whether the HMForced flag is set and we should
1072	* fail if we cannot initialize.
1073	*/
1074	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
1075	{
1076	/*
1077	* Error state.
1078	* The error message will be non-empty on failure and 'rc' will be set too.
1079	*/
1080	RTERRINFOSTATIC ErrInfo;
1081	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
1082	int rc = nemR3WinInitProbeAndLoad(fForced, pErrInfo);
1083	if (RT_SUCCESS(rc))
1084	{
1085	/*
1086	* Check the capabilties of the hypervisor, starting with whether it's present.
1087	*/
1088	rc = nemR3WinInitCheckCapabilities(pVM, pErrInfo);
1089	if (RT_SUCCESS(rc))
1090	{
1091	/*
1092	* Discover the VID I/O control function numbers we need.
1093	*/
1094	rc = nemR3WinInitDiscoverIoControlProperties(pVM, pErrInfo);
1095	if (RT_SUCCESS(rc))
1096	{
1097	/*
1098	* Check out our ring-0 capabilities.
1099	*/
1100	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_NEM_INIT_VM, 0, NULL);
1101	if (RT_SUCCESS(rc))
1102	{
1103	/*
1104	* Create and initialize a partition.
1105	*/
1106	rc = nemR3WinInitCreatePartition(pVM, pErrInfo);
1107	if (RT_SUCCESS(rc))
1108	{
1109	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
1110	Log(("NEM: Marked active!\n"));
1111	}
1112	}
1113	}
1114	}
1115	}
1116
1117	/*
1118	* We only fail if in forced mode, otherwise just log the complaint and return.
1119	*/
1120	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
1121	if ( (fForced \|\| !fFallback)
1122	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
1123	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
1124
1125	if (RTErrInfoIsSet(pErrInfo))
1126	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* This is called after CPUMR3Init is done.
1133	*
1134	* @returns VBox status code.
1135	* @param pVM The VM handle..
1136	*/
1137	int nemR3NativeInitAfterCPUM(PVM pVM)
1138	{
1139	/*
1140	* Validate sanity.
1141	*/
1142	WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
1143	AssertReturn(hPartition != NULL, VERR_WRONG_ORDER);
1144	AssertReturn(!pVM->nem.s.hPartitionDevice, VERR_WRONG_ORDER);
1145	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
1146	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
1147
1148	/*
1149	* Continue setting up the partition now that we've got most of the CPUID feature stuff.
1150	*/
1151
1152	/* Not sure if we really need to set the vendor. */
1153	WHV_PARTITION_PROPERTY Property;
1154	RT_ZERO(Property);
1155	Property.PropertyCode = WHvPartitionPropertyCodeProcessorVendor;
1156	Property.ProcessorVendor = pVM->nem.s.enmCpuVendor == CPUMCPUVENDOR_AMD ? WHvProcessorVendorAmd
1157	: WHvProcessorVendorIntel;
1158	HRESULT hrc = WHvSetPartitionProperty(hPartition, &Property, sizeof(Property));
1159	if (FAILED(hrc))
1160	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1161	"Failed to set WHvPartitionPropertyCodeProcessorVendor to %u: %Rhrc (Last=%#x/%u)",
1162	Property.ProcessorVendor, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1163
1164	/* Not sure if we really need to set the cache line flush size. */
1165	RT_ZERO(Property);
1166	Property.PropertyCode = WHvPartitionPropertyCodeProcessorClFlushSize;
1167	Property.ProcessorClFlushSize = pVM->nem.s.cCacheLineFlushShift;
1168	hrc = WHvSetPartitionProperty(hPartition, &Property, sizeof(Property));
1169	if (FAILED(hrc))
1170	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1171	"Failed to set WHvPartitionPropertyCodeProcessorClFlushSize to %u: %Rhrc (Last=%#x/%u)",
1172	pVM->nem.s.cCacheLineFlushShift, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1173
1174	/*
1175	* Sync CPU features with CPUM.
1176	*/
1177	/** @todo sync CPU features with CPUM. */
1178
1179	/* Set the partition property. */
1180	RT_ZERO(Property);
1181	Property.PropertyCode = WHvPartitionPropertyCodeProcessorFeatures;
1182	Property.ProcessorFeatures.AsUINT64 = pVM->nem.s.uCpuFeatures.u64;
1183	hrc = WHvSetPartitionProperty(hPartition, &Property, sizeof(Property));
1184	if (FAILED(hrc))
1185	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1186	"Failed to set WHvPartitionPropertyCodeProcessorFeatures to %'#RX64: %Rhrc (Last=%#x/%u)",
1187	pVM->nem.s.uCpuFeatures.u64, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1188
1189	/*
1190	* Set up the partition and create EMTs.
1191	*
1192	* Seems like this is where the partition is actually instantiated and we get
1193	* a handle to it.
1194	*/
1195	hrc = WHvSetupPartition(hPartition);
1196	if (FAILED(hrc))
1197	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1198	"Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)",
1199	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1200
1201	/* Get the handle. */
1202	HANDLE hPartitionDevice;
1203	__try
1204	{
1205	hPartitionDevice = ((HANDLE *)hPartition)[1];
1206	}
1207	__except(EXCEPTION_EXECUTE_HANDLER)
1208	{
1209	hrc = GetExceptionCode();
1210	hPartitionDevice = NULL;
1211	}
1212	if ( hPartitionDevice == NULL
1213	\|\| hPartitionDevice == (HANDLE)(intptr_t)-1)
1214	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1215	"Failed to get device handle for partition %p: %Rhrc", hPartition, hrc);
1216
1217	HV_PARTITION_ID idHvPartition = HV_PARTITION_ID_INVALID;
1218	if (!g_pfnVidGetHvPartitionId(hPartitionDevice, &idHvPartition))
1219	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1220	"Failed to get device handle and/or partition ID for %p (hPartitionDevice=%p, Last=%#x/%u)",
1221	hPartition, hPartitionDevice, RTNtLastStatusValue(), RTNtLastErrorValue());
1222	pVM->nem.s.hPartitionDevice = hPartitionDevice;
1223	pVM->nem.s.idHvPartition = idHvPartition;
1224
1225	/*
1226	* Setup the EMTs.
1227	*/
1228	VMCPUID iCpu;
1229	for (iCpu = 0; iCpu < pVM->cCpus; iCpu++)
1230	{
1231	PVMCPU pVCpu = &pVM->aCpus[iCpu];
1232
1233	pVCpu->nem.s.hNativeThreadHandle = (RTR3PTR)RTThreadGetNativeHandle(VMR3GetThreadHandle(pVCpu->pUVCpu));
1234	Assert((HANDLE)pVCpu->nem.s.hNativeThreadHandle != INVALID_HANDLE_VALUE);
1235
1236	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
1237	VID_MAPPED_MESSAGE_SLOT MappedMsgSlot = { NULL, UINT32_MAX, UINT32_MAX };
1238	if (g_pfnVidMessageSlotMap(hPartitionDevice, &MappedMsgSlot, iCpu))
1239	{
1240	AssertLogRelMsg(MappedMsgSlot.iCpu == iCpu && MappedMsgSlot.uParentAdvisory == UINT32_MAX,
1241	("%#x %#x (iCpu=%#x)\n", MappedMsgSlot.iCpu, MappedMsgSlot.uParentAdvisory, iCpu));
1242	pVCpu->nem.s.pvMsgSlotMapping = MappedMsgSlot.pMsgBlock;
1243	}
1244	else
1245	{
1246	NTSTATUS const rcNtLast = RTNtLastStatusValue();
1247	DWORD const dwErrLast = RTNtLastErrorValue();
1248	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1249	"Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)", hrc, rcNtLast, dwErrLast);
1250	}
1251	#else
1252	hrc = WHvCreateVirtualProcessor(hPartition, iCpu, 0 /fFlags/);
1253	if (FAILED(hrc))
1254	{
1255	NTSTATUS const rcNtLast = RTNtLastStatusValue();
1256	DWORD const dwErrLast = RTNtLastErrorValue();
1257	while (iCpu-- > 0)
1258	{
1259	HRESULT hrc2 = WHvDeleteVirtualProcessor(hPartition, iCpu);
1260	AssertLogRelMsg(SUCCEEDED(hrc2), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
1261	hPartition, iCpu, hrc2, RTNtLastStatusValue(),
1262	RTNtLastErrorValue()));
1263	}
1264	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1265	"Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)", hrc, rcNtLast, dwErrLast);
1266	}
1267	#endif /* !NEM_WIN_USE_OUR_OWN_RUN_API */
1268	}
1269	pVM->nem.s.fCreatedEmts = true;
1270
1271	/*
1272	* Do some more ring-0 initialization now that we've got the partition handle.
1273	*/
1274	int rc = VMMR3CallR0Emt(pVM, &pVM->aCpus[0], VMMR0_DO_NEM_INIT_VM_PART_2, 0, NULL);
1275	if (RT_SUCCESS(rc))
1276	{
1277	LogRel(("NEM: Successfully set up partition (device handle %p, partition ID %#llx)\n", hPartitionDevice, idHvPartition));
1278	return VINF_SUCCESS;
1279	}
1280	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to NEMR0InitVMPart2 failed: %Rrc", rc);
1281	}
1282
1283
1284	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
1285	{
1286	NOREF(pVM); NOREF(enmWhat);
1287	return VINF_SUCCESS;
1288	}
1289
1290
1291	int nemR3NativeTerm(PVM pVM)
1292	{
1293	/*
1294	* Delete the partition.
1295	*/
1296	WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
1297	pVM->nem.s.hPartition = NULL;
1298	pVM->nem.s.hPartitionDevice = NULL;
1299	if (hPartition != NULL)
1300	{
1301	VMCPUID iCpu = pVM->nem.s.fCreatedEmts ? pVM->cCpus : 0;
1302	LogRel(("NEM: Destroying partition %p with its %u VCpus...\n", hPartition, iCpu));
1303	while (iCpu-- > 0)
1304	{
1305	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
1306	pVM->aCpus[iCpu].nem.s.pvMsgSlotMapping = NULL;
1307	#else
1308	HRESULT hrc = WHvDeleteVirtualProcessor(hPartition, iCpu);
1309	AssertLogRelMsg(SUCCEEDED(hrc), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
1310	hPartition, iCpu, hrc, RTNtLastStatusValue(),
1311	RTNtLastErrorValue()));
1312	#endif
1313	}
1314	WHvDeletePartition(hPartition);
1315	}
1316	pVM->nem.s.fCreatedEmts = false;
1317	return VINF_SUCCESS;
1318	}
1319
1320
1321	/**
1322	* VM reset notification.
1323	*
1324	* @param pVM The cross context VM structure.
1325	*/
1326	void nemR3NativeReset(PVM pVM)
1327	{
1328	/* Unfix the A20 gate. */
1329	pVM->nem.s.fA20Fixed = false;
1330	}
1331
1332
1333	/**
1334	* Reset CPU due to INIT IPI or hot (un)plugging.
1335	*
1336	* @param pVCpu The cross context virtual CPU structure of the CPU being
1337	* reset.
1338	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
1339	*/
1340	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
1341	{
1342	/* Lock the A20 gate if INIT IPI, make sure it's enabled. */
1343	if (fInitIpi && pVCpu->idCpu > 0)
1344	{
1345	PVM pVM = pVCpu->CTX_SUFF(pVM);
1346	if (!pVM->nem.s.fA20Enabled)
1347	nemR3NativeNotifySetA20(pVCpu, true);
1348	pVM->nem.s.fA20Enabled = true;
1349	pVM->nem.s.fA20Fixed = true;
1350	}
1351	}
1352
1353	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1354
1355	/**
1356	* Wrapper around VMMR0_DO_NEM_MAP_PAGES for a single page.
1357	*
1358	* @returns VBox status code.
1359	* @param pVM The cross context VM structure.
1360	* @param pVCpu The cross context virtual CPU structure of the caller.
1361	* @param GCPhysSrc The source page. Does not need to be page aligned.
1362	* @param GCPhysDst The destination page. Same as @a GCPhysSrc except for
1363	* when A20 is disabled.
1364	* @param fFlags HV_MAP_GPA_XXX.
1365	*/
1366	DECLINLINE(int) nemR3WinHypercallMapPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fFlags)
1367	{
1368	pVCpu->nem.s.Hypercall.MapPages.GCPhysSrc = GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
1369	pVCpu->nem.s.Hypercall.MapPages.GCPhysDst = GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
1370	pVCpu->nem.s.Hypercall.MapPages.cPages = 1;
1371	pVCpu->nem.s.Hypercall.MapPages.fFlags = fFlags;
1372	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_MAP_PAGES, 0, NULL);
1373	}
1374
1375
1376	/**
1377	* Wrapper around VMMR0_DO_NEM_UNMAP_PAGES for a single page.
1378	*
1379	* @returns VBox status code.
1380	* @param pVM The cross context VM structure.
1381	* @param pVCpu The cross context virtual CPU structure of the caller.
1382	* @param GCPhys The page to unmap. Does not need to be page aligned.
1383	*/
1384	DECLINLINE(int) nemR3WinHypercallUnmapPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1385	{
1386	pVCpu->nem.s.Hypercall.UnmapPages.GCPhys = GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
1387	pVCpu->nem.s.Hypercall.UnmapPages.cPages = 1;
1388	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_UNMAP_PAGES, 0, NULL);
1389	}
1390
1391	#endif /* NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
1392
1393	static int nemR3WinCopyStateToHyperV(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1394	{
1395	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1396	NOREF(pCtx);
1397	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPORT_STATE, UINT64_MAX, NULL);
1398	AssertLogRelRCReturn(rc, rc);
1399	return rc;
1400
1401	#else
1402	WHV_REGISTER_NAME aenmNames[128];
1403	WHV_REGISTER_VALUE aValues[128];
1404
1405	/* GPRs */
1406	aenmNames[0] = WHvX64RegisterRax;
1407	aValues[0].Reg64 = pCtx->rax;
1408	aenmNames[1] = WHvX64RegisterRcx;
1409	aValues[1].Reg64 = pCtx->rcx;
1410	aenmNames[2] = WHvX64RegisterRdx;
1411	aValues[2].Reg64 = pCtx->rdx;
1412	aenmNames[3] = WHvX64RegisterRbx;
1413	aValues[3].Reg64 = pCtx->rbx;
1414	aenmNames[4] = WHvX64RegisterRsp;
1415	aValues[4].Reg64 = pCtx->rsp;
1416	aenmNames[5] = WHvX64RegisterRbp;
1417	aValues[5].Reg64 = pCtx->rbp;
1418	aenmNames[6] = WHvX64RegisterRsi;
1419	aValues[6].Reg64 = pCtx->rsi;
1420	aenmNames[7] = WHvX64RegisterRdi;
1421	aValues[7].Reg64 = pCtx->rdi;
1422	aenmNames[8] = WHvX64RegisterR8;
1423	aValues[8].Reg64 = pCtx->r8;
1424	aenmNames[9] = WHvX64RegisterR9;
1425	aValues[9].Reg64 = pCtx->r9;
1426	aenmNames[10] = WHvX64RegisterR10;
1427	aValues[10].Reg64 = pCtx->r10;
1428	aenmNames[11] = WHvX64RegisterR11;
1429	aValues[11].Reg64 = pCtx->r11;
1430	aenmNames[12] = WHvX64RegisterR12;
1431	aValues[12].Reg64 = pCtx->r12;
1432	aenmNames[13] = WHvX64RegisterR13;
1433	aValues[13].Reg64 = pCtx->r13;
1434	aenmNames[14] = WHvX64RegisterR14;
1435	aValues[14].Reg64 = pCtx->r14;
1436	aenmNames[15] = WHvX64RegisterR15;
1437	aValues[15].Reg64 = pCtx->r15;
1438
1439	/* RIP & Flags */
1440	aenmNames[16] = WHvX64RegisterRip;
1441	aValues[16].Reg64 = pCtx->rip;
1442	aenmNames[17] = WHvX64RegisterRflags;
1443	aValues[17].Reg64 = pCtx->rflags.u;
1444
1445	/* Segments */
1446	#define COPY_OUT_SEG(a_idx, a_enmName, a_SReg) \
1447	do { \
1448	aenmNames[a_idx] = a_enmName; \
1449	aValues[a_idx].Segment.Base = (a_SReg).u64Base; \
1450	aValues[a_idx].Segment.Limit = (a_SReg).u32Limit; \
1451	aValues[a_idx].Segment.Selector = (a_SReg).Sel; \
1452	aValues[a_idx].Segment.Attributes = (a_SReg).Attr.u; \
1453	} while (0)
1454	COPY_OUT_SEG(18, WHvX64RegisterEs, pCtx->es);
1455	COPY_OUT_SEG(19, WHvX64RegisterCs, pCtx->cs);
1456	COPY_OUT_SEG(20, WHvX64RegisterSs, pCtx->ss);
1457	COPY_OUT_SEG(21, WHvX64RegisterDs, pCtx->ds);
1458	COPY_OUT_SEG(22, WHvX64RegisterFs, pCtx->fs);
1459	COPY_OUT_SEG(23, WHvX64RegisterGs, pCtx->gs);
1460	COPY_OUT_SEG(24, WHvX64RegisterLdtr, pCtx->ldtr);
1461	COPY_OUT_SEG(25, WHvX64RegisterTr, pCtx->tr);
1462
1463	uintptr_t iReg = 26;
1464	/* Descriptor tables. */
1465	aenmNames[iReg] = WHvX64RegisterIdtr;
1466	aValues[iReg].Table.Limit = pCtx->idtr.cbIdt;
1467	aValues[iReg].Table.Base = pCtx->idtr.pIdt;
1468	iReg++;
1469	aenmNames[iReg] = WHvX64RegisterGdtr;
1470	aValues[iReg].Table.Limit = pCtx->gdtr.cbGdt;
1471	aValues[iReg].Table.Base = pCtx->gdtr.pGdt;
1472	iReg++;
1473
1474	/* Control registers. */
1475	aenmNames[iReg] = WHvX64RegisterCr0;
1476	aValues[iReg].Reg64 = pCtx->cr0;
1477	iReg++;
1478	aenmNames[iReg] = WHvX64RegisterCr2;
1479	aValues[iReg].Reg64 = pCtx->cr2;
1480	iReg++;
1481	aenmNames[iReg] = WHvX64RegisterCr3;
1482	aValues[iReg].Reg64 = pCtx->cr3;
1483	iReg++;
1484	aenmNames[iReg] = WHvX64RegisterCr4;
1485	aValues[iReg].Reg64 = pCtx->cr4;
1486	iReg++;
1487	aenmNames[iReg] = WHvX64RegisterCr8;
1488	aValues[iReg].Reg64 = CPUMGetGuestCR8(pVCpu);
1489	iReg++;
1490
1491	/* Debug registers. */
1492	/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
1493	aenmNames[iReg] = WHvX64RegisterDr0;
1494	//aValues[iReg].Reg64 = CPUMGetHyperDR0(pVCpu);
1495	aValues[iReg].Reg64 = pCtx->dr[0];
1496	iReg++;
1497	aenmNames[iReg] = WHvX64RegisterDr1;
1498	//aValues[iReg].Reg64 = CPUMGetHyperDR1(pVCpu);
1499	aValues[iReg].Reg64 = pCtx->dr[1];
1500	iReg++;
1501	aenmNames[iReg] = WHvX64RegisterDr2;
1502	//aValues[iReg].Reg64 = CPUMGetHyperDR2(pVCpu);
1503	aValues[iReg].Reg64 = pCtx->dr[2];
1504	iReg++;
1505	aenmNames[iReg] = WHvX64RegisterDr3;
1506	//aValues[iReg].Reg64 = CPUMGetHyperDR3(pVCpu);
1507	aValues[iReg].Reg64 = pCtx->dr[3];
1508	iReg++;
1509	aenmNames[iReg] = WHvX64RegisterDr6;
1510	//aValues[iReg].Reg64 = CPUMGetHyperDR6(pVCpu);
1511	aValues[iReg].Reg64 = pCtx->dr[6];
1512	iReg++;
1513	aenmNames[iReg] = WHvX64RegisterDr7;
1514	//aValues[iReg].Reg64 = CPUMGetHyperDR7(pVCpu);
1515	aValues[iReg].Reg64 = pCtx->dr[7];
1516	iReg++;
1517
1518	/* Vector state. */
1519	aenmNames[iReg] = WHvX64RegisterXmm0;
1520	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Lo;
1521	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Hi;
1522	iReg++;
1523	aenmNames[iReg] = WHvX64RegisterXmm1;
1524	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Lo;
1525	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Hi;
1526	iReg++;
1527	aenmNames[iReg] = WHvX64RegisterXmm2;
1528	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Lo;
1529	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Hi;
1530	iReg++;
1531	aenmNames[iReg] = WHvX64RegisterXmm3;
1532	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Lo;
1533	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Hi;
1534	iReg++;
1535	aenmNames[iReg] = WHvX64RegisterXmm4;
1536	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Lo;
1537	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Hi;
1538	iReg++;
1539	aenmNames[iReg] = WHvX64RegisterXmm5;
1540	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Lo;
1541	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Hi;
1542	iReg++;
1543	aenmNames[iReg] = WHvX64RegisterXmm6;
1544	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Lo;
1545	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Hi;
1546	iReg++;
1547	aenmNames[iReg] = WHvX64RegisterXmm7;
1548	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Lo;
1549	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Hi;
1550	iReg++;
1551	aenmNames[iReg] = WHvX64RegisterXmm8;
1552	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Lo;
1553	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Hi;
1554	iReg++;
1555	aenmNames[iReg] = WHvX64RegisterXmm9;
1556	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Lo;
1557	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Hi;
1558	iReg++;
1559	aenmNames[iReg] = WHvX64RegisterXmm10;
1560	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Lo;
1561	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Hi;
1562	iReg++;
1563	aenmNames[iReg] = WHvX64RegisterXmm11;
1564	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Lo;
1565	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Hi;
1566	iReg++;
1567	aenmNames[iReg] = WHvX64RegisterXmm12;
1568	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Lo;
1569	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Hi;
1570	iReg++;
1571	aenmNames[iReg] = WHvX64RegisterXmm13;
1572	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Lo;
1573	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Hi;
1574	iReg++;
1575	aenmNames[iReg] = WHvX64RegisterXmm14;
1576	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Lo;
1577	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Hi;
1578	iReg++;
1579	aenmNames[iReg] = WHvX64RegisterXmm15;
1580	aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Lo;
1581	aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Hi;
1582	iReg++;
1583
1584	/* Floating point state. */
1585	aenmNames[iReg] = WHvX64RegisterFpMmx0;
1586	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[0].au64[0];
1587	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[0].au64[1];
1588	iReg++;
1589	aenmNames[iReg] = WHvX64RegisterFpMmx1;
1590	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[1].au64[0];
1591	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[1].au64[1];
1592	iReg++;
1593	aenmNames[iReg] = WHvX64RegisterFpMmx2;
1594	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[2].au64[0];
1595	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[2].au64[1];
1596	iReg++;
1597	aenmNames[iReg] = WHvX64RegisterFpMmx3;
1598	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[3].au64[0];
1599	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[3].au64[1];
1600	iReg++;
1601	aenmNames[iReg] = WHvX64RegisterFpMmx4;
1602	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[4].au64[0];
1603	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[4].au64[1];
1604	iReg++;
1605	aenmNames[iReg] = WHvX64RegisterFpMmx5;
1606	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[5].au64[0];
1607	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[5].au64[1];
1608	iReg++;
1609	aenmNames[iReg] = WHvX64RegisterFpMmx6;
1610	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[6].au64[0];
1611	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[6].au64[1];
1612	iReg++;
1613	aenmNames[iReg] = WHvX64RegisterFpMmx7;
1614	aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[7].au64[0];
1615	aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[7].au64[1];
1616	iReg++;
1617
1618	aenmNames[iReg] = WHvX64RegisterFpControlStatus;
1619	aValues[iReg].FpControlStatus.FpControl = pCtx->pXStateR3->x87.FCW;
1620	aValues[iReg].FpControlStatus.FpStatus = pCtx->pXStateR3->x87.FSW;
1621	aValues[iReg].FpControlStatus.FpTag = pCtx->pXStateR3->x87.FTW;
1622	aValues[iReg].FpControlStatus.Reserved = pCtx->pXStateR3->x87.FTW >> 8;
1623	aValues[iReg].FpControlStatus.LastFpOp = pCtx->pXStateR3->x87.FOP;
1624	aValues[iReg].FpControlStatus.LastFpRip = (pCtx->pXStateR3->x87.FPUIP)
1625	\| ((uint64_t)pCtx->pXStateR3->x87.CS << 32)
1626	\| ((uint64_t)pCtx->pXStateR3->x87.Rsrvd1 << 48);
1627	iReg++;
1628
1629	aenmNames[iReg] = WHvX64RegisterXmmControlStatus;
1630	aValues[iReg].XmmControlStatus.LastFpRdp = (pCtx->pXStateR3->x87.FPUDP)
1631	\| ((uint64_t)pCtx->pXStateR3->x87.DS << 32)
1632	\| ((uint64_t)pCtx->pXStateR3->x87.Rsrvd2 << 48);
1633	aValues[iReg].XmmControlStatus.XmmStatusControl = pCtx->pXStateR3->x87.MXCSR;
1634	aValues[iReg].XmmControlStatus.XmmStatusControlMask = pCtx->pXStateR3->x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
1635	iReg++;
1636
1637	/* MSRs */
1638	// WHvX64RegisterTsc - don't touch
1639	aenmNames[iReg] = WHvX64RegisterEfer;
1640	aValues[iReg].Reg64 = pCtx->msrEFER;
1641	iReg++;
1642	aenmNames[iReg] = WHvX64RegisterKernelGsBase;
1643	aValues[iReg].Reg64 = pCtx->msrKERNELGSBASE;
1644	iReg++;
1645	aenmNames[iReg] = WHvX64RegisterApicBase;
1646	aValues[iReg].Reg64 = APICGetBaseMsrNoCheck(pVCpu);
1647	iReg++;
1648	aenmNames[iReg] = WHvX64RegisterPat;
1649	aValues[iReg].Reg64 = pCtx->msrPAT;
1650	iReg++;
1651	/// @todo WHvX64RegisterSysenterCs
1652	/// @todo WHvX64RegisterSysenterEip
1653	/// @todo WHvX64RegisterSysenterEsp
1654	aenmNames[iReg] = WHvX64RegisterStar;
1655	aValues[iReg].Reg64 = pCtx->msrSTAR;
1656	iReg++;
1657	aenmNames[iReg] = WHvX64RegisterLstar;
1658	aValues[iReg].Reg64 = pCtx->msrLSTAR;
1659	iReg++;
1660	aenmNames[iReg] = WHvX64RegisterCstar;
1661	aValues[iReg].Reg64 = pCtx->msrCSTAR;
1662	iReg++;
1663	aenmNames[iReg] = WHvX64RegisterSfmask;
1664	aValues[iReg].Reg64 = pCtx->msrSFMASK;
1665	iReg++;
1666
1667	/* event injection (always clear it). */
1668	aenmNames[iReg] = WHvRegisterPendingInterruption;
1669	aValues[iReg].Reg64 = 0;
1670	iReg++;
1671	/// @todo WHvRegisterInterruptState
1672	/// @todo WHvRegisterPendingEvent0
1673	/// @todo WHvRegisterPendingEvent1
1674
1675	/*
1676	* Set the registers.
1677	*/
1678	Assert(iReg < RT_ELEMENTS(aValues));
1679	Assert(iReg < RT_ELEMENTS(aenmNames));
1680	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
1681	Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
1682	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
1683	#endif
1684	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
1685	if (SUCCEEDED(hrc))
1686	return VINF_SUCCESS;
1687	AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
1688	pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
1689	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
1690	return VERR_INTERNAL_ERROR;
1691	#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1692	}
1693
1694	static int nemR3WinCopyStateFromHyperV(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1695	{
1696	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1697	NOREF(pCtx);
1698	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_IMPORT_STATE, UINT64_MAX, NULL);
1699	if (RT_SUCCESS(rc))
1700	return rc;
1701	if (rc == VERR_NEM_FLUSH_TLB)
1702	return PGMFlushTLB(pVCpu, pCtx->cr3, true /fGlobal/);
1703	if (rc == VERR_NEM_CHANGE_PGM_MODE)
1704	return PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
1705	AssertLogRelRCReturn(rc, rc);
1706	return rc;
1707
1708	#else
1709	WHV_REGISTER_NAME aenmNames[128];
1710
1711	/* GPRs */
1712	aenmNames[0] = WHvX64RegisterRax;
1713	aenmNames[1] = WHvX64RegisterRcx;
1714	aenmNames[2] = WHvX64RegisterRdx;
1715	aenmNames[3] = WHvX64RegisterRbx;
1716	aenmNames[4] = WHvX64RegisterRsp;
1717	aenmNames[5] = WHvX64RegisterRbp;
1718	aenmNames[6] = WHvX64RegisterRsi;
1719	aenmNames[7] = WHvX64RegisterRdi;
1720	aenmNames[8] = WHvX64RegisterR8;
1721	aenmNames[9] = WHvX64RegisterR9;
1722	aenmNames[10] = WHvX64RegisterR10;
1723	aenmNames[11] = WHvX64RegisterR11;
1724	aenmNames[12] = WHvX64RegisterR12;
1725	aenmNames[13] = WHvX64RegisterR13;
1726	aenmNames[14] = WHvX64RegisterR14;
1727	aenmNames[15] = WHvX64RegisterR15;
1728
1729	/* RIP & Flags */
1730	aenmNames[16] = WHvX64RegisterRip;
1731	aenmNames[17] = WHvX64RegisterRflags;
1732
1733	/* Segments */
1734	aenmNames[18] = WHvX64RegisterEs;
1735	aenmNames[19] = WHvX64RegisterCs;
1736	aenmNames[20] = WHvX64RegisterSs;
1737	aenmNames[21] = WHvX64RegisterDs;
1738	aenmNames[22] = WHvX64RegisterFs;
1739	aenmNames[23] = WHvX64RegisterGs;
1740	aenmNames[24] = WHvX64RegisterLdtr;
1741	aenmNames[25] = WHvX64RegisterTr;
1742
1743	/* Descriptor tables. */
1744	aenmNames[26] = WHvX64RegisterIdtr;
1745	aenmNames[27] = WHvX64RegisterGdtr;
1746
1747	/* Control registers. */
1748	aenmNames[28] = WHvX64RegisterCr0;
1749	aenmNames[29] = WHvX64RegisterCr2;
1750	aenmNames[30] = WHvX64RegisterCr3;
1751	aenmNames[31] = WHvX64RegisterCr4;
1752	aenmNames[32] = WHvX64RegisterCr8;
1753
1754	/* Debug registers. */
1755	aenmNames[33] = WHvX64RegisterDr0;
1756	aenmNames[34] = WHvX64RegisterDr1;
1757	aenmNames[35] = WHvX64RegisterDr2;
1758	aenmNames[36] = WHvX64RegisterDr3;
1759	aenmNames[37] = WHvX64RegisterDr6;
1760	aenmNames[38] = WHvX64RegisterDr7;
1761
1762	/* Vector state. */
1763	aenmNames[39] = WHvX64RegisterXmm0;
1764	aenmNames[40] = WHvX64RegisterXmm1;
1765	aenmNames[41] = WHvX64RegisterXmm2;
1766	aenmNames[42] = WHvX64RegisterXmm3;
1767	aenmNames[43] = WHvX64RegisterXmm4;
1768	aenmNames[44] = WHvX64RegisterXmm5;
1769	aenmNames[45] = WHvX64RegisterXmm6;
1770	aenmNames[46] = WHvX64RegisterXmm7;
1771	aenmNames[47] = WHvX64RegisterXmm8;
1772	aenmNames[48] = WHvX64RegisterXmm9;
1773	aenmNames[49] = WHvX64RegisterXmm10;
1774	aenmNames[50] = WHvX64RegisterXmm11;
1775	aenmNames[51] = WHvX64RegisterXmm12;
1776	aenmNames[52] = WHvX64RegisterXmm13;
1777	aenmNames[53] = WHvX64RegisterXmm14;
1778	aenmNames[54] = WHvX64RegisterXmm15;
1779
1780	/* Floating point state. */
1781	aenmNames[55] = WHvX64RegisterFpMmx0;
1782	aenmNames[56] = WHvX64RegisterFpMmx1;
1783	aenmNames[57] = WHvX64RegisterFpMmx2;
1784	aenmNames[58] = WHvX64RegisterFpMmx3;
1785	aenmNames[59] = WHvX64RegisterFpMmx4;
1786	aenmNames[60] = WHvX64RegisterFpMmx5;
1787	aenmNames[61] = WHvX64RegisterFpMmx6;
1788	aenmNames[62] = WHvX64RegisterFpMmx7;
1789	aenmNames[63] = WHvX64RegisterFpControlStatus;
1790	aenmNames[64] = WHvX64RegisterXmmControlStatus;
1791
1792	/* MSRs */
1793	// WHvX64RegisterTsc - don't touch
1794	aenmNames[65] = WHvX64RegisterEfer;
1795	aenmNames[66] = WHvX64RegisterKernelGsBase;
1796	aenmNames[67] = WHvX64RegisterApicBase;
1797	aenmNames[68] = WHvX64RegisterPat;
1798	aenmNames[69] = WHvX64RegisterSysenterCs;
1799	aenmNames[70] = WHvX64RegisterSysenterEip;
1800	aenmNames[71] = WHvX64RegisterSysenterEsp;
1801	aenmNames[72] = WHvX64RegisterStar;
1802	aenmNames[73] = WHvX64RegisterLstar;
1803	aenmNames[74] = WHvX64RegisterCstar;
1804	aenmNames[75] = WHvX64RegisterSfmask;
1805
1806	/* event injection */
1807	aenmNames[76] = WHvRegisterPendingInterruption;
1808	aenmNames[77] = WHvRegisterInterruptState;
1809	aenmNames[78] = WHvRegisterInterruptState;
1810	aenmNames[79] = WHvRegisterPendingEvent0;
1811	aenmNames[80] = WHvRegisterPendingEvent1;
1812	unsigned const cRegs = 81;
1813
1814	/*
1815	* Get the registers.
1816	*/
1817	WHV_REGISTER_VALUE aValues[cRegs];
1818	RT_ZERO(aValues);
1819	Assert(RT_ELEMENTS(aValues) >= cRegs);
1820	Assert(RT_ELEMENTS(aenmNames) >= cRegs);
1821	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
1822	Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
1823	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
1824	#endif
1825	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues);
1826	if (SUCCEEDED(hrc))
1827	{
1828	/* GPRs */
1829	Assert(aenmNames[0] == WHvX64RegisterRax);
1830	Assert(aenmNames[15] == WHvX64RegisterR15);
1831	pCtx->rax = aValues[0].Reg64;
1832	pCtx->rcx = aValues[1].Reg64;
1833	pCtx->rdx = aValues[2].Reg64;
1834	pCtx->rbx = aValues[3].Reg64;
1835	pCtx->rsp = aValues[4].Reg64;
1836	pCtx->rbp = aValues[5].Reg64;
1837	pCtx->rsi = aValues[6].Reg64;
1838	pCtx->rdi = aValues[7].Reg64;
1839	pCtx->r8 = aValues[8].Reg64;
1840	pCtx->r9 = aValues[9].Reg64;
1841	pCtx->r10 = aValues[10].Reg64;
1842	pCtx->r11 = aValues[11].Reg64;
1843	pCtx->r12 = aValues[12].Reg64;
1844	pCtx->r13 = aValues[13].Reg64;
1845	pCtx->r14 = aValues[14].Reg64;
1846	pCtx->r15 = aValues[15].Reg64;
1847
1848	/* RIP & Flags */
1849	Assert(aenmNames[16] == WHvX64RegisterRip);
1850	pCtx->rip = aValues[16].Reg64;
1851	pCtx->rflags.u = aValues[17].Reg64;
1852
1853	/* Segments */
1854	#define COPY_BACK_SEG(a_idx, a_enmName, a_SReg) \
1855	do { \
1856	Assert(aenmNames[a_idx] == a_enmName); \
1857	(a_SReg).u64Base = aValues[a_idx].Segment.Base; \
1858	(a_SReg).u32Limit = aValues[a_idx].Segment.Limit; \
1859	(a_SReg).ValidSel = (a_SReg).Sel = aValues[a_idx].Segment.Selector; \
1860	(a_SReg).Attr.u = aValues[a_idx].Segment.Attributes; \
1861	(a_SReg).fFlags = CPUMSELREG_FLAGS_VALID; \
1862	} while (0)
1863	COPY_BACK_SEG(18, WHvX64RegisterEs, pCtx->es);
1864	COPY_BACK_SEG(19, WHvX64RegisterCs, pCtx->cs);
1865	COPY_BACK_SEG(20, WHvX64RegisterSs, pCtx->ss);
1866	COPY_BACK_SEG(21, WHvX64RegisterDs, pCtx->ds);
1867	COPY_BACK_SEG(22, WHvX64RegisterFs, pCtx->fs);
1868	COPY_BACK_SEG(23, WHvX64RegisterGs, pCtx->gs);
1869	COPY_BACK_SEG(24, WHvX64RegisterLdtr, pCtx->ldtr);
1870	COPY_BACK_SEG(25, WHvX64RegisterTr, pCtx->tr);
1871
1872	/* Descriptor tables. */
1873	Assert(aenmNames[26] == WHvX64RegisterIdtr);
1874	pCtx->idtr.cbIdt = aValues[26].Table.Limit;
1875	pCtx->idtr.pIdt = aValues[26].Table.Base;
1876	Assert(aenmNames[27] == WHvX64RegisterGdtr);
1877	pCtx->gdtr.cbGdt = aValues[27].Table.Limit;
1878	pCtx->gdtr.pGdt = aValues[27].Table.Base;
1879
1880	/* Control registers. */
1881	Assert(aenmNames[28] == WHvX64RegisterCr0);
1882	bool fMaybeChangedMode = false;
1883	bool fFlushTlb = false;
1884	bool fFlushGlobalTlb = false;
1885	if (pCtx->cr0 != aValues[28].Reg64)
1886	{
1887	CPUMSetGuestCR0(pVCpu, aValues[28].Reg64);
1888	fMaybeChangedMode = true;
1889	fFlushTlb = fFlushGlobalTlb = true; /// @todo fix this
1890	}
1891	Assert(aenmNames[29] == WHvX64RegisterCr2);
1892	pCtx->cr2 = aValues[29].Reg64;
1893	if (pCtx->cr3 != aValues[30].Reg64)
1894	{
1895	CPUMSetGuestCR3(pVCpu, aValues[30].Reg64);
1896	fFlushTlb = true;
1897	}
1898	if (pCtx->cr4 != aValues[31].Reg64)
1899	{
1900	CPUMSetGuestCR4(pVCpu, aValues[31].Reg64);
1901	fMaybeChangedMode = true;
1902	fFlushTlb = fFlushGlobalTlb = true; /// @todo fix this
1903	}
1904	APICSetTpr(pVCpu, (uint8_t)aValues[32].Reg64 << 4);
1905
1906	/* Debug registers. */
1907	Assert(aenmNames[33] == WHvX64RegisterDr0);
1908	/** @todo fixme */
1909	if (pCtx->dr[0] != aValues[33].Reg64)
1910	CPUMSetGuestDR0(pVCpu, aValues[33].Reg64);
1911	if (pCtx->dr[1] != aValues[34].Reg64)
1912	CPUMSetGuestDR1(pVCpu, aValues[34].Reg64);
1913	if (pCtx->dr[2] != aValues[35].Reg64)
1914	CPUMSetGuestDR2(pVCpu, aValues[35].Reg64);
1915	if (pCtx->dr[3] != aValues[36].Reg64)
1916	CPUMSetGuestDR3(pVCpu, aValues[36].Reg64);
1917	Assert(aenmNames[37] == WHvX64RegisterDr6);
1918	Assert(aenmNames[38] == WHvX64RegisterDr7);
1919	if (pCtx->dr[6] != aValues[37].Reg64)
1920	CPUMSetGuestDR6(pVCpu, aValues[37].Reg64);
1921	if (pCtx->dr[7] != aValues[38].Reg64)
1922	CPUMSetGuestDR6(pVCpu, aValues[38].Reg64);
1923
1924	/* Vector state. */
1925	Assert(aenmNames[39] == WHvX64RegisterXmm0);
1926	Assert(aenmNames[54] == WHvX64RegisterXmm15);
1927	pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Lo = aValues[39].Reg128.Low64;
1928	pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Hi = aValues[39].Reg128.High64;
1929	pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Lo = aValues[40].Reg128.Low64;
1930	pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Hi = aValues[40].Reg128.High64;
1931	pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Lo = aValues[41].Reg128.Low64;
1932	pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Hi = aValues[41].Reg128.High64;
1933	pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Lo = aValues[42].Reg128.Low64;
1934	pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Hi = aValues[42].Reg128.High64;
1935	pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Lo = aValues[43].Reg128.Low64;
1936	pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Hi = aValues[43].Reg128.High64;
1937	pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Lo = aValues[44].Reg128.Low64;
1938	pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Hi = aValues[44].Reg128.High64;
1939	pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Lo = aValues[45].Reg128.Low64;
1940	pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Hi = aValues[45].Reg128.High64;
1941	pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Lo = aValues[46].Reg128.Low64;
1942	pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Hi = aValues[46].Reg128.High64;
1943	pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Lo = aValues[47].Reg128.Low64;
1944	pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Hi = aValues[47].Reg128.High64;
1945	pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Lo = aValues[48].Reg128.Low64;
1946	pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Hi = aValues[48].Reg128.High64;
1947	pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Lo = aValues[49].Reg128.Low64;
1948	pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Hi = aValues[49].Reg128.High64;
1949	pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Lo = aValues[50].Reg128.Low64;
1950	pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Hi = aValues[50].Reg128.High64;
1951	pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Lo = aValues[51].Reg128.Low64;
1952	pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Hi = aValues[51].Reg128.High64;
1953	pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Lo = aValues[52].Reg128.Low64;
1954	pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Hi = aValues[52].Reg128.High64;
1955	pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Lo = aValues[53].Reg128.Low64;
1956	pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Hi = aValues[53].Reg128.High64;
1957	pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Lo = aValues[54].Reg128.Low64;
1958	pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Hi = aValues[54].Reg128.High64;
1959
1960	/* Floating point state. */
1961	Assert(aenmNames[55] == WHvX64RegisterFpMmx0);
1962	Assert(aenmNames[62] == WHvX64RegisterFpMmx7);
1963	pCtx->pXStateR3->x87.aRegs[0].au64[0] = aValues[55].Fp.AsUINT128.Low64;
1964	pCtx->pXStateR3->x87.aRegs[0].au64[1] = aValues[55].Fp.AsUINT128.High64;
1965	pCtx->pXStateR3->x87.aRegs[1].au64[0] = aValues[56].Fp.AsUINT128.Low64;
1966	pCtx->pXStateR3->x87.aRegs[1].au64[1] = aValues[56].Fp.AsUINT128.High64;
1967	pCtx->pXStateR3->x87.aRegs[2].au64[0] = aValues[57].Fp.AsUINT128.Low64;
1968	pCtx->pXStateR3->x87.aRegs[2].au64[1] = aValues[57].Fp.AsUINT128.High64;
1969	pCtx->pXStateR3->x87.aRegs[3].au64[0] = aValues[58].Fp.AsUINT128.Low64;
1970	pCtx->pXStateR3->x87.aRegs[3].au64[1] = aValues[58].Fp.AsUINT128.High64;
1971	pCtx->pXStateR3->x87.aRegs[4].au64[0] = aValues[59].Fp.AsUINT128.Low64;
1972	pCtx->pXStateR3->x87.aRegs[4].au64[1] = aValues[59].Fp.AsUINT128.High64;
1973	pCtx->pXStateR3->x87.aRegs[5].au64[0] = aValues[60].Fp.AsUINT128.Low64;
1974	pCtx->pXStateR3->x87.aRegs[5].au64[1] = aValues[60].Fp.AsUINT128.High64;
1975	pCtx->pXStateR3->x87.aRegs[6].au64[0] = aValues[61].Fp.AsUINT128.Low64;
1976	pCtx->pXStateR3->x87.aRegs[6].au64[1] = aValues[61].Fp.AsUINT128.High64;
1977	pCtx->pXStateR3->x87.aRegs[7].au64[0] = aValues[62].Fp.AsUINT128.Low64;
1978	pCtx->pXStateR3->x87.aRegs[7].au64[1] = aValues[62].Fp.AsUINT128.High64;
1979
1980	Assert(aenmNames[63] == WHvX64RegisterFpControlStatus);
1981	pCtx->pXStateR3->x87.FCW = aValues[63].FpControlStatus.FpControl;
1982	pCtx->pXStateR3->x87.FSW = aValues[63].FpControlStatus.FpStatus;
1983	pCtx->pXStateR3->x87.FTW = aValues[63].FpControlStatus.FpTag
1984	/\| (aValues[63].FpControlStatus.Reserved << 8)/;
1985	pCtx->pXStateR3->x87.FOP = aValues[63].FpControlStatus.LastFpOp;
1986	pCtx->pXStateR3->x87.FPUIP = (uint32_t)aValues[63].FpControlStatus.LastFpRip;
1987	pCtx->pXStateR3->x87.CS = (uint16_t)(aValues[63].FpControlStatus.LastFpRip >> 32);
1988	pCtx->pXStateR3->x87.Rsrvd1 = (uint16_t)(aValues[63].FpControlStatus.LastFpRip >> 48);
1989
1990	Assert(aenmNames[64] == WHvX64RegisterXmmControlStatus);
1991	pCtx->pXStateR3->x87.FPUDP = (uint32_t)aValues[64].XmmControlStatus.LastFpRdp;
1992	pCtx->pXStateR3->x87.DS = (uint16_t)(aValues[64].XmmControlStatus.LastFpRdp >> 32);
1993	pCtx->pXStateR3->x87.Rsrvd2 = (uint16_t)(aValues[64].XmmControlStatus.LastFpRdp >> 48);
1994	pCtx->pXStateR3->x87.MXCSR = aValues[64].XmmControlStatus.XmmStatusControl;
1995	pCtx->pXStateR3->x87.MXCSR_MASK = aValues[64].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
1996
1997	/* MSRs */
1998	// WHvX64RegisterTsc - don't touch
1999	Assert(aenmNames[65] == WHvX64RegisterEfer);
2000	if (aValues[65].Reg64 != pCtx->msrEFER)
2001	{
2002	pCtx->msrEFER = aValues[65].Reg64;
2003	fMaybeChangedMode = true;
2004	}
2005
2006	Assert(aenmNames[66] == WHvX64RegisterKernelGsBase);
2007	pCtx->msrKERNELGSBASE = aValues[66].Reg64;
2008
2009	Assert(aenmNames[67] == WHvX64RegisterApicBase);
2010	if (aValues[67].Reg64 != APICGetBaseMsrNoCheck(pVCpu))
2011	{
2012	VBOXSTRICTRC rc2 = APICSetBaseMsr(pVCpu, aValues[67].Reg64);
2013	Assert(rc2 == VINF_SUCCESS); NOREF(rc2);
2014	}
2015
2016	Assert(aenmNames[68] == WHvX64RegisterPat);
2017	pCtx->msrPAT = aValues[68].Reg64;
2018	/// @todo WHvX64RegisterSysenterCs
2019	/// @todo WHvX64RegisterSysenterEip
2020	/// @todo WHvX64RegisterSysenterEsp
2021	Assert(aenmNames[72] == WHvX64RegisterStar);
2022	pCtx->msrSTAR = aValues[72].Reg64;
2023	Assert(aenmNames[73] == WHvX64RegisterLstar);
2024	pCtx->msrLSTAR = aValues[73].Reg64;
2025	Assert(aenmNames[74] == WHvX64RegisterCstar);
2026	pCtx->msrCSTAR = aValues[74].Reg64;
2027	Assert(aenmNames[75] == WHvX64RegisterSfmask);
2028	pCtx->msrSFMASK = aValues[75].Reg64;
2029
2030	/// @todo WHvRegisterPendingInterruption
2031	Assert(aenmNames[76] == WHvRegisterPendingInterruption);
2032	WHV_X64_PENDING_INTERRUPTION_REGISTER const * pPendingInt = (WHV_X64_PENDING_INTERRUPTION_REGISTER const *)&aValues[76];
2033	if (pPendingInt->InterruptionPending)
2034	{
2035	Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
2036	pPendingInt->InterruptionType, pPendingInt->InterruptionVector, pPendingInt->DeliverErrorCode,
2037	pPendingInt->ErrorCode, pPendingInt->InstructionLength, pPendingInt->NestedEvent));
2038	AssertMsg((pPendingInt->AsUINT64 & UINT64_C(0xfc00)) == 0, ("%#RX64\n", pPendingInt->AsUINT64));
2039	}
2040
2041	/// @todo WHvRegisterInterruptState
2042	/// @todo WHvRegisterPendingEvent0
2043	/// @todo WHvRegisterPendingEvent1
2044
2045
2046	if (fMaybeChangedMode)
2047	{
2048	int rc = PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
2049	AssertRC(rc);
2050	}
2051	if (fFlushTlb)
2052	{
2053	int rc = PGMFlushTLB(pVCpu, pCtx->cr3, fFlushGlobalTlb);
2054	AssertRC(rc);
2055	}
2056
2057	return VINF_SUCCESS;
2058	}
2059
2060	AssertLogRelMsgFailed(("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
2061	pVM->nem.s.hPartition, pVCpu->idCpu, cRegs,
2062	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2063	return VERR_INTERNAL_ERROR;
2064	#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
2065	}
2066
2067
2068	#ifdef LOG_ENABLED
2069	/**
2070	* Get the virtual processor running status.
2071	*/
2072	DECLINLINE(VID_PROCESSOR_STATUS) nemR3WinCpuGetRunningStatus(PVMCPU pVCpu)
2073	{
2074	RTERRVARS Saved;
2075	RTErrVarsSave(&Saved);
2076
2077	/*
2078	* This API is disabled in release builds, it seems. On build 17101 it requires
2079	* the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00
2080	*/
2081	VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined;
2082	NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus);
2083	AssertRC(rcNt);
2084
2085	RTErrVarsRestore(&Saved);
2086	return enmCpuStatus;
2087	}
2088	#endif
2089
2090
2091	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
2092
2093	/**
2094	* Our own WHvCancelRunVirtualProcessor that can later be moved to ring-0.
2095	*
2096	* This is an experiment only.
2097	*
2098	* @returns VBox status code.
2099	* @param pVM The cross context VM structure.
2100	* @param pVCpu The cross context virtual CPU structure of the
2101	* calling EMT.
2102	*/
2103	static int nemR3WinCancelRunVirtualProcessor(PVM pVM, PVMCPU pVCpu)
2104	{
2105	/*
2106	* Work the state.
2107	*
2108	* From the looks of things, we should let the EMT call VidStopVirtualProcessor.
2109	* So, we just need to modify the state and kick the EMT if it's waiting on
2110	* messages. For the latter we use QueueUserAPC / KeAlterThread.
2111	*/
2112	for (;;)
2113	{
2114	VMCPUSTATE enmState = VMCPU_GET_STATE(pVCpu);
2115	switch (enmState)
2116	{
2117	case VMCPUSTATE_STARTED_EXEC_NEM:
2118	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM))
2119	{
2120	Log8(("nemR3WinCancelRunVirtualProcessor: Switched %u to canceled state\n", pVCpu->idCpu));
2121	return VINF_SUCCESS;
2122	}
2123	break;
2124
2125	case VMCPUSTATE_STARTED_EXEC_NEM_WAIT:
2126	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT))
2127	{
2128	NTSTATUS rcNt = NtAlertThread(pVCpu->nem.s.hNativeThreadHandle);
2129	Log8(("nemR3WinCancelRunVirtualProcessor: Alerted %u: %#x\n", pVCpu->idCpu, rcNt));
2130	Assert(rcNt == STATUS_SUCCESS);
2131	if (NT_SUCCESS(rcNt))
2132	return VINF_SUCCESS;
2133	AssertLogRelMsgFailedReturn(("NtAlertThread failed: %#x\n", rcNt), RTErrConvertFromNtStatus(rcNt));
2134	}
2135	break;
2136
2137	default:
2138	return VINF_SUCCESS;
2139	}
2140
2141	ASMNopPause();
2142	RT_NOREF(pVM);
2143	}
2144	}
2145
2146
2147	/**
2148	* Fills in WHV_VP_EXIT_CONTEXT from HV_X64_INTERCEPT_MESSAGE_HEADER.
2149	*/
2150	DECLINLINE(void) nemR3WinConvertX64MsgHdrToVpExitCtx(HV_X64_INTERCEPT_MESSAGE_HEADER const pHdr, WHV_VP_EXIT_CONTEXT pCtx)
2151	{
2152	pCtx->ExecutionState.AsUINT16 = pHdr->ExecutionState.AsUINT16;
2153	pCtx->InstructionLength = pHdr->InstructionLength;
2154	pCtx->Cs.Base = pHdr->CsSegment.Base;
2155	pCtx->Cs.Limit = pHdr->CsSegment.Limit;
2156	pCtx->Cs.Selector = pHdr->CsSegment.Selector;
2157	pCtx->Cs.Attributes = pHdr->CsSegment.Attributes;
2158	pCtx->Rip = pHdr->Rip;
2159	pCtx->Rflags = pHdr->Rflags;
2160	}
2161
2162
2163	/**
2164	* Convert hyper-V exit message to the WinHvPlatform structures.
2165	*
2166	* @returns VBox status code
2167	* @param pMsgHdr The message to convert.
2168	* @param pExitCtx The output structure. Assumes zeroed.
2169	*/
2170	static int nemR3WinRunVirtualProcessorConvertPending(HV_MESSAGE_HEADER const pMsgHdr, WHV_RUN_VP_EXIT_CONTEXT pExitCtx)
2171	{
2172	switch (pMsgHdr->MessageType)
2173	{
2174	case HvMessageTypeUnmappedGpa:
2175	case HvMessageTypeGpaIntercept:
2176	{
2177	PCHV_X64_MEMORY_INTERCEPT_MESSAGE pMemMsg = (PCHV_X64_MEMORY_INTERCEPT_MESSAGE)(pMsgHdr + 1);
2178	Assert(pMsgHdr->PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
2179
2180	pExitCtx->ExitReason = WHvRunVpExitReasonMemoryAccess;
2181	nemR3WinConvertX64MsgHdrToVpExitCtx(&pMemMsg->Header, &pExitCtx->MemoryAccess.VpContext);
2182	pExitCtx->MemoryAccess.InstructionByteCount = pMemMsg->InstructionByteCount;
2183	((uint64_t )pExitCtx->MemoryAccess.InstructionBytes)[0] = ((uint64_t const )pMemMsg->InstructionBytes)[0];
2184	((uint64_t )pExitCtx->MemoryAccess.InstructionBytes)[1] = ((uint64_t const )pMemMsg->InstructionBytes)[1];
2185
2186	pExitCtx->MemoryAccess.AccessInfo.AccessType = pMemMsg->Header.InterceptAccessType;
2187	pExitCtx->MemoryAccess.AccessInfo.GpaUnmapped = pMsgHdr->MessageType == HvMessageTypeUnmappedGpa;
2188	pExitCtx->MemoryAccess.AccessInfo.GvaValid = pMemMsg->MemoryAccessInfo.GvaValid;
2189	pExitCtx->MemoryAccess.AccessInfo.Reserved = pMemMsg->MemoryAccessInfo.Reserved;
2190	pExitCtx->MemoryAccess.Gpa = pMemMsg->GuestPhysicalAddress;
2191	pExitCtx->MemoryAccess.Gva = pMemMsg->GuestVirtualAddress;
2192	return VINF_SUCCESS;
2193	}
2194
2195	case HvMessageTypeX64IoPortIntercept:
2196	{
2197	PCHV_X64_IO_PORT_INTERCEPT_MESSAGE pPioMsg= (PCHV_X64_IO_PORT_INTERCEPT_MESSAGE)(pMsgHdr + 1);
2198	Assert(pMsgHdr->PayloadSize == sizeof(*pPioMsg));
2199
2200	pExitCtx->ExitReason = WHvRunVpExitReasonX64IoPortAccess;
2201	nemR3WinConvertX64MsgHdrToVpExitCtx(&pPioMsg->Header, &pExitCtx->IoPortAccess.VpContext);
2202	pExitCtx->IoPortAccess.InstructionByteCount = pPioMsg->InstructionByteCount;
2203	((uint64_t )pExitCtx->IoPortAccess.InstructionBytes)[0] = ((uint64_t const )pPioMsg->InstructionBytes)[0];
2204	((uint64_t )pExitCtx->IoPortAccess.InstructionBytes)[1] = ((uint64_t const )pPioMsg->InstructionBytes)[1];
2205
2206	pExitCtx->IoPortAccess.AccessInfo.IsWrite = pPioMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE;
2207	pExitCtx->IoPortAccess.AccessInfo.AccessSize = pPioMsg->AccessInfo.AccessSize;
2208	pExitCtx->IoPortAccess.AccessInfo.StringOp = pPioMsg->AccessInfo.StringOp;
2209	pExitCtx->IoPortAccess.AccessInfo.RepPrefix = pPioMsg->AccessInfo.RepPrefix;
2210	pExitCtx->IoPortAccess.AccessInfo.Reserved = pPioMsg->AccessInfo.Reserved;
2211	pExitCtx->IoPortAccess.PortNumber = pPioMsg->PortNumber;
2212	pExitCtx->IoPortAccess.Rax = pPioMsg->Rax;
2213	pExitCtx->IoPortAccess.Rcx = pPioMsg->Rcx;
2214	pExitCtx->IoPortAccess.Rsi = pPioMsg->Rsi;
2215	pExitCtx->IoPortAccess.Rdi = pPioMsg->Rdi;
2216	pExitCtx->IoPortAccess.Ds.Base = pPioMsg->DsSegment.Base;
2217	pExitCtx->IoPortAccess.Ds.Limit = pPioMsg->DsSegment.Limit;
2218	pExitCtx->IoPortAccess.Ds.Selector = pPioMsg->DsSegment.Selector;
2219	pExitCtx->IoPortAccess.Ds.Attributes = pPioMsg->DsSegment.Attributes;
2220	pExitCtx->IoPortAccess.Es.Base = pPioMsg->EsSegment.Base;
2221	pExitCtx->IoPortAccess.Es.Limit = pPioMsg->EsSegment.Limit;
2222	pExitCtx->IoPortAccess.Es.Selector = pPioMsg->EsSegment.Selector;
2223	pExitCtx->IoPortAccess.Es.Attributes = pPioMsg->EsSegment.Attributes;
2224	return VINF_SUCCESS;
2225	}
2226
2227	case HvMessageTypeX64Halt:
2228	{
2229	PCHV_X64_HALT_MESSAGE pHaltMsg = (PCHV_X64_HALT_MESSAGE)(pMsgHdr + 1);
2230	AssertMsg(pHaltMsg->u64Reserved == 0, ("HALT reserved: %#RX64\n", pHaltMsg->u64Reserved));
2231	pExitCtx->ExitReason = WHvRunVpExitReasonX64Halt;
2232	return VINF_SUCCESS;
2233	}
2234
2235	case HvMessageTypeX64InterruptWindow:
2236	AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n", pMsgHdr->MessageType), VERR_INTERNAL_ERROR_2);
2237
2238	case HvMessageTypeInvalidVpRegisterValue:
2239	case HvMessageTypeUnrecoverableException:
2240	case HvMessageTypeUnsupportedFeature:
2241	case HvMessageTypeTlbPageSizeMismatch:
2242	AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n", pMsgHdr->MessageType), VERR_INTERNAL_ERROR_2);
2243
2244	case HvMessageTypeX64MsrIntercept:
2245	case HvMessageTypeX64CpuidIntercept:
2246	case HvMessageTypeX64ExceptionIntercept:
2247	case HvMessageTypeX64ApicEoi:
2248	case HvMessageTypeX64LegacyFpError:
2249	case HvMessageTypeX64RegisterIntercept:
2250	case HvMessageTypeApicEoi:
2251	case HvMessageTypeFerrAsserted:
2252	case HvMessageTypeEventLogBufferComplete:
2253	case HvMessageTimerExpired:
2254	AssertLogRelMsgFailedReturn(("Unexpected message type #x!\n", pMsgHdr->MessageType), VERR_INTERNAL_ERROR_2);
2255
2256	default:
2257	AssertLogRelMsgFailedReturn(("Unknown message type #x!\n", pMsgHdr->MessageType), VERR_INTERNAL_ERROR_2);
2258	}
2259	}
2260
2261
2262	/**
2263	* Our own WHvRunVirtualProcessor that can later be moved to ring-0.
2264	*
2265	* This is an experiment only.
2266	*
2267	* @returns VBox status code.
2268	* @param pVM The cross context VM structure.
2269	* @param pVCpu The cross context virtual CPU structure of the
2270	* calling EMT.
2271	* @param pExitCtx Where to return exit information.
2272	* @param cbExitCtx Size of the exit information area.
2273	*/
2274	static int nemR3WinRunVirtualProcessor(PVM pVM, PVMCPU pVCpu, WHV_RUN_VP_EXIT_CONTEXT *pExitCtx, size_t cbExitCtx)
2275	{
2276	RT_BZERO(pExitCtx, cbExitCtx);
2277
2278	/*
2279	* Tell the CPU to execute stuff if we haven't got a pending message.
2280	*/
2281	VID_MESSAGE_MAPPING_HEADER volatile pMappingHeader = (VID_MESSAGE_MAPPING_HEADER volatile )pVCpu->nem.s.pvMsgSlotMapping;
2282	uint32_t fHandleAndGetFlags;
2283	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2284	{
2285	uint8_t const bMsgState = pVCpu->nem.s.bMsgState;
2286	if (bMsgState == NEM_WIN_MSG_STATE_PENDING_MSG)
2287	{
2288	Assert(pMappingHeader->enmVidMsgType == VidMessageHypervisorMessage);
2289	fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE \| VID_MSHAGN_F_HANDLE_MESSAGE;
2290	Log8(("nemR3WinRunVirtualProcessor: #1: msg pending, no need to start CPU (cpu state %u)\n", nemR3WinCpuGetRunningStatus(pVCpu) ));
2291	}
2292	else if (bMsgState != NEM_WIN_MSG_STATE_STARTED)
2293	{
2294	if (bMsgState == NEM_WIN_MSG_STATE_PENDING_STOP_AND_MSG)
2295	{
2296	Log8(("nemR3WinRunVirtualProcessor: #0: pending stop+message (cpu status %u)\n", nemR3WinCpuGetRunningStatus(pVCpu) ));
2297	/* ACK the pending message and get the stop message. */
2298	BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
2299	VID_MSHAGN_F_HANDLE_MESSAGE \| VID_MSHAGN_F_GET_NEXT_MESSAGE, 5000);
2300	AssertLogRelMsg(fWait, ("dwErr=%u (%#x) rcNt=%#x\n", RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()));
2301
2302	/* ACK the stop message. */
2303	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
2304	VID_MSHAGN_F_HANDLE_MESSAGE, 5000);
2305	AssertLogRelMsg(fWait, ("dwErr=%u (%#x) rcNt=%#x\n", RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()));
2306
2307	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_STOPPED;
2308	}
2309
2310	Log8(("nemR3WinRunVirtualProcessor: #1: starting CPU (cpu status %u)\n", nemR3WinCpuGetRunningStatus(pVCpu) ));
2311	if (g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu))
2312	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_STARTED;
2313	else
2314	{
2315	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM);
2316	AssertLogRelMsgFailedReturn(("VidStartVirtualProcessor failed for CPU #%u: rcNt=%#x dwErr=%u\n",
2317	pVCpu->idCpu, RTNtLastStatusValue(), RTNtLastErrorValue()),
2318	VERR_INTERNAL_ERROR_3);
2319	}
2320	fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
2321	}
2322	else
2323	{
2324	/* This shouldn't happen. */
2325	fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
2326	Log8(("nemR3WinRunVirtualProcessor: #1: NO MSG PENDING! No need to start CPU (cpu state %u)\n", nemR3WinCpuGetRunningStatus(pVCpu) ));
2327	}
2328	}
2329	else
2330	{
2331	Log8(("nemR3WinRunVirtualProcessor: #1: state=%u -> canceled (cpu status %u)\n",
2332	VMCPU_GET_STATE(pVCpu), nemR3WinCpuGetRunningStatus(pVCpu)));
2333	pExitCtx->ExitReason = WHvRunVpExitReasonCanceled;
2334	return VINF_SUCCESS;
2335	}
2336
2337	/*
2338	* Wait for it to stop and give us a reason to work with.
2339	*/
2340	uint32_t cMillies = 5000; // Starting low so we can experiment without getting stuck.
2341	for (;;)
2342	{
2343	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
2344	{
2345	Log8(("nemR3WinRunVirtualProcessor: #2: Waiting %#x (cpu status %u)...\n",
2346	fHandleAndGetFlags, nemR3WinCpuGetRunningStatus(pVCpu)));
2347	BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
2348	fHandleAndGetFlags, cMillies);
2349	if (fWait)
2350	{
2351	/* Not sure yet, but we have to check whether there is anything pending
2352	and retry if there isn't. */
2353	VID_MESSAGE_TYPE const enmVidMsgType = pMappingHeader->enmVidMsgType;
2354	if (enmVidMsgType == VidMessageHypervisorMessage)
2355	{
2356	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT))
2357	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2358	Log8(("nemR3WinRunVirtualProcessor: #3: wait succeeded: %#x / %#x (cpu status %u)\n",
2359	enmVidMsgType, ((HV_MESSAGE_HEADER const *)(pMappingHeader + 1))->MessageType,
2360	nemR3WinCpuGetRunningStatus(pVCpu) ));
2361	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_PENDING_MSG;
2362	return nemR3WinRunVirtualProcessorConvertPending((HV_MESSAGE_HEADER const *)(pMappingHeader + 1), pExitCtx);
2363	}
2364
2365	/* This shouldn't happen, and I think its wrong. */
2366	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
2367	#ifdef DEBUG_bird
2368	__debugbreak();
2369	#endif
2370	Log8(("nemR3WinRunVirtualProcessor: #3: wait succeeded, but nothing pending: %#x / %#x (cpu status %u)\n",
2371	enmVidMsgType, ((HV_MESSAGE_HEADER const *)(pMappingHeader + 1))->MessageType, nemR3WinCpuGetRunningStatus(pVCpu) ));
2372	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_STARTED;
2373	AssertLogRelMsgReturnStmt(enmVidMsgType == VidMessageStopRequestComplete,
2374	("enmVidMsgType=%#x\n", enmVidMsgType),
2375	g_pfnVidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
2376	VERR_INTERNAL_ERROR_3);
2377	fHandleAndGetFlags &= ~VID_MSHAGN_F_HANDLE_MESSAGE;
2378	}
2379	else
2380	{
2381	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
2382
2383	/* Note! VID.SYS merges STATUS_ALERTED and STATUS_USER_APC into STATUS_TIMEOUT. */
2384	DWORD const dwErr = RTNtLastErrorValue();
2385	AssertLogRelMsgReturnStmt( dwErr == STATUS_TIMEOUT
2386	\|\| dwErr == STATUS_ALERTED \|\| dwErr == STATUS_USER_APC, /* just in case */
2387	("dwErr=%u (%#x) (cpu status %u)\n", dwErr, dwErr, nemR3WinCpuGetRunningStatus(pVCpu)),
2388	g_pfnVidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
2389	VERR_INTERNAL_ERROR_3);
2390	Log8(("nemR3WinRunVirtualProcessor: #3: wait timed out (cpu status %u)\n", nemR3WinCpuGetRunningStatus(pVCpu) ));
2391	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_STARTED;
2392	fHandleAndGetFlags &= ~VID_MSHAGN_F_HANDLE_MESSAGE;
2393	}
2394	}
2395	else
2396	{
2397	/*
2398	* State changed and we need to return.
2399	*
2400	* We must ensure that the processor is not running while we
2401	* return, and that can be a bit complicated.
2402	*/
2403	Log8(("nemR3WinRunVirtualProcessor: #4: state changed to %u (cpu status %u)\n",
2404	VMCPU_GET_STATE(pVCpu), nemR3WinCpuGetRunningStatus(pVCpu) ));
2405	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2406
2407	/* If we haven't marked the pervious message as handled, simply return
2408	without doing anything special. */
2409	if (fHandleAndGetFlags & VID_MSHAGN_F_HANDLE_MESSAGE)
2410	{
2411	Log8(("nemR3WinRunVirtualProcessor: #5: Didn't resume previous message.\n"));
2412	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_PENDING_MSG;
2413	pExitCtx->ExitReason = WHvRunVpExitReasonCanceled;
2414	return VINF_SUCCESS;
2415	}
2416
2417	/* The processor is running, so try stop it. */
2418	BOOL fStop = g_pfnVidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu);
2419	if (fStop)
2420	{
2421	Log8(("nemR3WinRunVirtualProcessor: #5: Stopping CPU succeeded (cpu status %u)\n", nemR3WinCpuGetRunningStatus(pVCpu) ));
2422	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_STOPPED;
2423	pExitCtx->ExitReason = WHvRunVpExitReasonCanceled;
2424	return VINF_SUCCESS;
2425	}
2426
2427	/* Dang, the CPU stopped by itself with a message pending. */
2428	DWORD dwErr = RTNtLastErrorValue();
2429	Log8(("nemR3WinRunVirtualProcessor: #5: Stopping CPU failed (%u/%#x) - cpu status %u\n",
2430	dwErr, dwErr, nemR3WinCpuGetRunningStatus(pVCpu) ));
2431	pExitCtx->ExitReason = WHvRunVpExitReasonCanceled;
2432	AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u\n", dwErr), VERR_INTERNAL_ERROR_3);
2433
2434	/* Get the pending message. */
2435	BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
2436	VID_MSHAGN_F_GET_NEXT_MESSAGE, 5000);
2437	AssertLogRelMsgReturn(fWait, ("error=%#u\n", RTNtLastErrorValue()), VERR_INTERNAL_ERROR_3);
2438
2439	VID_MESSAGE_TYPE const enmVidMsgType = pMappingHeader->enmVidMsgType;
2440	if (enmVidMsgType == VidMessageHypervisorMessage)
2441	{
2442	Log8(("nemR3WinRunVirtualProcessor: #6: wait succeeded: %#x / %#x (cpu status %u)\n", enmVidMsgType,
2443	((HV_MESSAGE_HEADER const *)(pMappingHeader + 1))->MessageType, nemR3WinCpuGetRunningStatus(pVCpu) ));
2444	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_PENDING_STOP_AND_MSG;
2445	return nemR3WinRunVirtualProcessorConvertPending((HV_MESSAGE_HEADER const *)(pMappingHeader + 1), pExitCtx);
2446	}
2447
2448	/* ACK the stop message, if that's what it is. Don't think we'll ever get here. */
2449	Log8(("nemR3WinRunVirtualProcessor: #6b: wait succeeded: %#x / %#x (cpu status %u)\n", enmVidMsgType,
2450	((HV_MESSAGE_HEADER const *)(pMappingHeader + 1))->MessageType, nemR3WinCpuGetRunningStatus(pVCpu) ));
2451	AssertLogRelMsgReturn(enmVidMsgType == VidMessageStopRequestComplete, ("enmVidMsgType=%#x\n", enmVidMsgType),
2452	VERR_INTERNAL_ERROR_3);
2453	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
2454	VID_MSHAGN_F_HANDLE_MESSAGE, 5000);
2455	AssertLogRelMsgReturn(fWait, ("dwErr=%#u\n", RTNtLastErrorValue()), VERR_INTERNAL_ERROR_3);
2456
2457	pVCpu->nem.s.bMsgState = NEM_WIN_MSG_STATE_STOPPED;
2458	pExitCtx->ExitReason = WHvRunVpExitReasonCanceled;
2459	return VINF_SUCCESS;
2460	}
2461
2462	/** @todo check flags and stuff? */
2463	}
2464	}
2465
2466	#endif /* NEM_WIN_USE_OUR_OWN_RUN_API */
2467
2468	#ifdef LOG_ENABLED
2469
2470	/**
2471	* Log the full details of an exit reason.
2472	*
2473	* @param pExitReason The exit reason to log.
2474	*/
2475	static void nemR3WinLogExitReason(WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
2476	{
2477	bool fExitCtx = false;
2478	bool fExitInstr = false;
2479	switch (pExitReason->ExitReason)
2480	{
2481	case WHvRunVpExitReasonMemoryAccess:
2482	Log2(("Exit: Memory access: GCPhys=%RGp GCVirt=%RGv %s %s %s\n",
2483	pExitReason->MemoryAccess.Gpa, pExitReason->MemoryAccess.Gva,
2484	g_apszWHvMemAccesstypes[pExitReason->MemoryAccess.AccessInfo.AccessType],
2485	pExitReason->MemoryAccess.AccessInfo.GpaUnmapped ? "unmapped" : "mapped",
2486	pExitReason->MemoryAccess.AccessInfo.GvaValid ? "" : "invalid-gc-virt"));
2487	AssertMsg(!(pExitReason->MemoryAccess.AccessInfo.AsUINT32 & ~UINT32_C(0xf)),
2488	("MemoryAccess.AccessInfo=%#x\n", pExitReason->MemoryAccess.AccessInfo.AsUINT32));
2489	fExitCtx = fExitInstr = true;
2490	break;
2491
2492	case WHvRunVpExitReasonX64IoPortAccess:
2493	Log2(("Exit: I/O port access: IoPort=%#x LB %u %s%s%s rax=%#RX64 rcx=%#RX64 rsi=%#RX64 rdi=%#RX64\n",
2494	pExitReason->IoPortAccess.PortNumber,
2495	pExitReason->IoPortAccess.AccessInfo.AccessSize,
2496	pExitReason->IoPortAccess.AccessInfo.IsWrite ? "out" : "in",
2497	pExitReason->IoPortAccess.AccessInfo.StringOp ? " string" : "",
2498	pExitReason->IoPortAccess.AccessInfo.RepPrefix ? " rep" : "",
2499	pExitReason->IoPortAccess.Rax,
2500	pExitReason->IoPortAccess.Rcx,
2501	pExitReason->IoPortAccess.Rsi,
2502	pExitReason->IoPortAccess.Rdi));
2503	Log2(("Exit: + ds=%#x:{%#RX64 LB %#RX32, %#x} es=%#x:{%#RX64 LB %#RX32, %#x}\n",
2504	pExitReason->IoPortAccess.Ds.Selector,
2505	pExitReason->IoPortAccess.Ds.Base,
2506	pExitReason->IoPortAccess.Ds.Limit,
2507	pExitReason->IoPortAccess.Ds.Attributes,
2508	pExitReason->IoPortAccess.Es.Selector,
2509	pExitReason->IoPortAccess.Es.Base,
2510	pExitReason->IoPortAccess.Es.Limit,
2511	pExitReason->IoPortAccess.Es.Attributes ));
2512
2513	AssertMsg( pExitReason->IoPortAccess.AccessInfo.AccessSize == 1
2514	\|\| pExitReason->IoPortAccess.AccessInfo.AccessSize == 2
2515	\|\| pExitReason->IoPortAccess.AccessInfo.AccessSize == 4,
2516	("IoPortAccess.AccessInfo.AccessSize=%d\n", pExitReason->IoPortAccess.AccessInfo.AccessSize));
2517	AssertMsg(!(pExitReason->IoPortAccess.AccessInfo.AsUINT32 & ~UINT32_C(0x3f)),
2518	("IoPortAccess.AccessInfo=%#x\n", pExitReason->IoPortAccess.AccessInfo.AsUINT32));
2519	fExitCtx = fExitInstr = true;
2520	break;
2521
2522	# if 0
2523	case WHvRunVpExitReasonUnrecoverableException:
2524	case WHvRunVpExitReasonInvalidVpRegisterValue:
2525	case WHvRunVpExitReasonUnsupportedFeature:
2526	case WHvRunVpExitReasonX64InterruptWindow:
2527	case WHvRunVpExitReasonX64Halt:
2528	case WHvRunVpExitReasonX64MsrAccess:
2529	case WHvRunVpExitReasonX64Cpuid:
2530	case WHvRunVpExitReasonException:
2531	case WHvRunVpExitReasonCanceled:
2532	case WHvRunVpExitReasonAlerted:
2533	WHV_X64_MSR_ACCESS_CONTEXT MsrAccess;
2534	WHV_X64_CPUID_ACCESS_CONTEXT CpuidAccess;
2535	WHV_VP_EXCEPTION_CONTEXT VpException;
2536	WHV_X64_INTERRUPTION_DELIVERABLE_CONTEXT InterruptWindow;
2537	WHV_UNRECOVERABLE_EXCEPTION_CONTEXT UnrecoverableException;
2538	WHV_X64_UNSUPPORTED_FEATURE_CONTEXT UnsupportedFeature;
2539	WHV_RUN_VP_CANCELED_CONTEXT CancelReason;
2540	#endif
2541
2542	case WHvRunVpExitReasonNone:
2543	Log2(("Exit: No reason\n"));
2544	AssertFailed();
2545	break;
2546
2547	default:
2548	Log(("Exit: %#x\n", pExitReason->ExitReason));
2549	break;
2550	}
2551
2552	/*
2553	* Context and maybe instruction details.
2554	*/
2555	if (fExitCtx)
2556	{
2557	const WHV_VP_EXIT_CONTEXT *pVpCtx = &pExitReason->IoPortAccess.VpContext;
2558	Log2(("Exit: + CS:RIP=%04x:%08RX64 RFLAGS=%06RX64 cbInstr=%u CS={%RX64 L %#RX32, %#x}\n",
2559	pVpCtx->Cs.Selector,
2560	pVpCtx->Rip,
2561	pVpCtx->Rflags,
2562	pVpCtx->InstructionLength,
2563	pVpCtx->Cs.Base, pVpCtx->Cs.Limit, pVpCtx->Cs.Attributes));
2564	Log2(("Exit: + cpl=%d CR0.PE=%d CR0.AM=%d EFER.LMA=%d DebugActive=%d InterruptionPending=%d InterruptShadow=%d\n",
2565	pVpCtx->ExecutionState.Cpl,
2566	pVpCtx->ExecutionState.Cr0Pe,
2567	pVpCtx->ExecutionState.Cr0Am,
2568	pVpCtx->ExecutionState.EferLma,
2569	pVpCtx->ExecutionState.DebugActive,
2570	pVpCtx->ExecutionState.InterruptionPending,
2571	pVpCtx->ExecutionState.InterruptShadow));
2572	AssertMsg(!(pVpCtx->ExecutionState.AsUINT16 & ~UINT16_C(0x107f)),
2573	("ExecutionState.AsUINT16=%#x\n", pVpCtx->ExecutionState.AsUINT16));
2574
2575	/** @todo Someone at Microsoft please explain why the InstructionBytes fields
2576	* are 16 bytes long, when 15 would've been sufficent and saved 3-7 bytes of
2577	* alignment padding? Intel max length is 15, so is this sSome ARM stuff?
2578	* Aren't ARM
2579	* instructions max 32-bit wide? Confused. */
2580	if (fExitInstr && pExitReason->IoPortAccess.InstructionByteCount > 0)
2581	Log2(("Exit: + Instruction %.*Rhxs\n",
2582	pExitReason->IoPortAccess.InstructionByteCount, pExitReason->IoPortAccess.InstructionBytes));
2583	}
2584	}
2585
2586
2587	/**
2588	* Logs the current CPU state.
2589	*/
2590	static void nemR3WinLogState(PVM pVM, PVMCPU pVCpu)
2591	{
2592	if (LogIs3Enabled())
2593	{
2594	char szRegs[4096];
2595	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
2596	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
2597	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
2598	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
2599	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
2600	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
2601	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
2602	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
2603	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
2604	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
2605	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
2606	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
2607	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
2608	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
2609	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
2610	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
2611	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
2612	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
2613	" efer=%016VR{efer}\n"
2614	" pat=%016VR{pat}\n"
2615	" sf_mask=%016VR{sf_mask}\n"
2616	"krnl_gs_base=%016VR{krnl_gs_base}\n"
2617	" lstar=%016VR{lstar}\n"
2618	" star=%016VR{star} cstar=%016VR{cstar}\n"
2619	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
2620	);
2621
2622	char szInstr[256];
2623	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
2624	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
2625	szInstr, sizeof(szInstr), NULL);
2626	Log3(("%s%s\n", szRegs, szInstr));
2627	}
2628	}
2629
2630	#endif /* LOG_ENABLED */
2631
2632
2633	/**
2634	* Advances the guest RIP and clear EFLAGS.RF.
2635	*
2636	* This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
2637	*
2638	* @param pVCpu The cross context virtual CPU structure.
2639	* @param pCtx The CPU context to update.
2640	* @param pExitCtx The exit context.
2641	*/
2642	DECLINLINE(void) nemR3WinAdvanceGuestRipAndClearRF(PVMCPU pVCpu, PCPUMCTX pCtx, WHV_VP_EXIT_CONTEXT const *pExitCtx)
2643	{
2644	/* Advance the RIP. */
2645	Assert(pExitCtx->InstructionLength > 0 && pExitCtx->InstructionLength < 16);
2646	pCtx->rip += pExitCtx->InstructionLength;
2647	pCtx->rflags.Bits.u1RF = 0;
2648
2649	/* Update interrupt inhibition. */
2650	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2651	{ /* likely */ }
2652	else if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2653	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2654	}
2655
2656
2657	static VBOXSTRICTRC nemR3WinHandleHalt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2658	{
2659	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx);
2660	LogFlow(("nemR3WinHandleHalt\n"));
2661	return VINF_EM_HALT;
2662	}
2663
2664
2665	static DECLCALLBACK(int) nemR3WinUnmapOnePageCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, uint8_t pu2NemState, void pvUser)
2666	{
2667	RT_NOREF_PV(pvUser);
2668	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2669	int rc = nemR3WinHypercallUnmapPage(pVM, pVCpu, GCPhys);
2670	AssertRC(rc);
2671	if (RT_SUCCESS(rc))
2672	#else
2673	RT_NOREF_PV(pVCpu);
2674	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
2675	if (SUCCEEDED(hrc))
2676	#endif
2677	{
2678	Log5(("NEM GPA unmap all: %RGp (cMappedPages=%u)\n", GCPhys, pVM->nem.s.cMappedPages - 1));
2679	*pu2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
2680	}
2681	else
2682	{
2683	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2684	LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
2685	#else
2686	LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
2687	GCPhys, g_apszPageStates[*pu2NemState], hrc, hrc, RTNtLastStatusValue(),
2688	RTNtLastErrorValue(), pVM->nem.s.cMappedPages));
2689	#endif
2690	*pu2NemState = NEM_WIN_PAGE_STATE_NOT_SET;
2691	}
2692	if (pVM->nem.s.cMappedPages > 0)
2693	ASMAtomicDecU32(&pVM->nem.s.cMappedPages);
2694	return VINF_SUCCESS;
2695	}
2696
2697
2698	/**
2699	* State to pass between nemR3WinHandleMemoryAccess and
2700	* nemR3WinHandleMemoryAccessPageCheckerCallback.
2701	*/
2702	typedef struct NEMR3WINHMACPCCSTATE
2703	{
2704	/** Input: Write access. */
2705	bool fWriteAccess;
2706	/** Output: Set if we did something. */
2707	bool fDidSomething;
2708	/** Output: Set it we should resume. */
2709	bool fCanResume;
2710	} NEMR3WINHMACPCCSTATE;
2711
2712	/**
2713	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
2714	* Worker for nemR3WinHandleMemoryAccess; pvUser points to a
2715	* NEMR3WINHMACPCCSTATE structure. }
2716	*/
2717	static DECLCALLBACK(int) nemR3WinHandleMemoryAccessPageCheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys,
2718	PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
2719	{
2720	NEMR3WINHMACPCCSTATE pState = (NEMR3WINHMACPCCSTATE )pvUser;
2721	pState->fDidSomething = false;
2722	pState->fCanResume = false;
2723
2724	/* If A20 is disabled, we may need to make another query on the masked
2725	page to get the correct protection information. */
2726	uint8_t u2State = pInfo->u2NemState;
2727	RTGCPHYS GCPhysSrc;
2728	if ( pVM->nem.s.fA20Enabled
2729	\|\| !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
2730	GCPhysSrc = GCPhys;
2731	else
2732	{
2733	GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20);
2734	PGMPHYSNEMPAGEINFO Info2;
2735	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL);
2736	AssertRCReturn(rc, rc);
2737
2738	*pInfo = Info2;
2739	pInfo->u2NemState = u2State;
2740	}
2741
2742	/*
2743	* Consolidate current page state with actual page protection and access type.
2744	* We don't really consider downgrades here, as they shouldn't happen.
2745	*/
2746	#ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2747	/** @todo Someone at microsoft please explain:
2748	* I'm not sure WTF was going on, but I ended up in a loop if I remapped a
2749	* readonly page as writable (unmap, then map again). Specifically, this was an
2750	* issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1. So, in
2751	* a hope to work around that we no longer pre-map anything, just unmap stuff
2752	* and do it lazily here. And here we will first unmap, restart, and then remap
2753	* with new protection or backing.
2754	*/
2755	#endif
2756	int rc;
2757	switch (u2State)
2758	{
2759	case NEM_WIN_PAGE_STATE_UNMAPPED:
2760	case NEM_WIN_PAGE_STATE_NOT_SET:
2761	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
2762	{
2763	Log4(("nemR3WinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
2764	return VINF_SUCCESS;
2765	}
2766
2767	/* Don't bother remapping it if it's a write request to a non-writable page. */
2768	if ( pState->fWriteAccess
2769	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
2770	{
2771	Log4(("nemR3WinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
2772	return VINF_SUCCESS;
2773	}
2774
2775	/* Map the page. */
2776	rc = nemR3NativeSetPhysPage(pVM,
2777	pVCpu,
2778	GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
2779	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
2780	pInfo->fNemProt,
2781	&u2State,
2782	true /fBackingState/);
2783	pInfo->u2NemState = u2State;
2784	Log4(("nemR3WinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
2785	GCPhys, g_apszPageStates[u2State], rc));
2786	pState->fDidSomething = true;
2787	pState->fCanResume = true;
2788	return rc;
2789
2790	case NEM_WIN_PAGE_STATE_READABLE:
2791	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
2792	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
2793	{
2794	Log4(("nemR3WinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
2795	return VINF_SUCCESS;
2796	}
2797
2798	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2799	/* Upgrade page to writable. */
2800	/** @todo test this*/
2801	if ( (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
2802	&& pState->fWriteAccess)
2803	{
2804	rc = nemR3WinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhys,
2805	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
2806	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
2807	AssertRC(rc);
2808	if (RT_SUCCESS(rc))
2809	{
2810	pInfo->u2NemState = NEM_WIN_PAGE_STATE_WRITABLE;
2811	pState->fDidSomething = true;
2812	pState->fCanResume = true;
2813	Log5(("NEM GPA write-upgrade/exit: %RGp (was %s, cMappedPages=%u)\n",
2814	GCPhys, g_apszPageStates[u2State], pVM->nem.s.cMappedPages));
2815	}
2816	}
2817	else
2818	{
2819	/* Need to emulate the acces. */
2820	AssertBreak(pInfo->fNemProt != NEM_PAGE_PROT_NONE); /* There should be no downgrades. */
2821	rc = VINF_SUCCESS;
2822	}
2823	return rc;
2824	#else
2825	break;
2826	#endif
2827
2828	case NEM_WIN_PAGE_STATE_WRITABLE:
2829	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
2830	{
2831	Log4(("nemR3WinHandleMemoryAccessPageCheckerCallback: %RGp - #3\n", GCPhys));
2832	return VINF_SUCCESS;
2833	}
2834	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2835	AssertFailed(); /* There should be no downgrades. */
2836	#endif
2837	break;
2838
2839	default:
2840	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_INTERNAL_ERROR_3);
2841	}
2842
2843	/*
2844	* Unmap and restart the instruction.
2845	* If this fails, which it does every so often, just unmap everything for now.
2846	*/
2847	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2848	rc = nemR3WinHypercallUnmapPage(pVM, pVCpu, GCPhys);
2849	AssertRC(rc);
2850	if (RT_SUCCESS(rc))
2851	#else
2852	/** @todo figure out whether we mess up the state or if it's WHv. */
2853	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
2854	if (SUCCEEDED(hrc))
2855	#endif
2856	{
2857	pState->fDidSomething = true;
2858	pState->fCanResume = true;
2859	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
2860	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2861	Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
2862	return VINF_SUCCESS;
2863	}
2864	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2865	LogRel(("nemR3WinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhys, rc));
2866	return rc;
2867	#else
2868	LogRel(("nemR3WinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
2869	GCPhys, g_apszPageStates[u2State], hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue(),
2870	pVM->nem.s.cMappedPages));
2871
2872	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinUnmapOnePageCallback, NULL);
2873	Log(("nemR3WinHandleMemoryAccessPageCheckerCallback: Unmapped all (cMappedPages=%u)\n", pVM->nem.s.cMappedPages));
2874
2875	pState->fDidSomething = true;
2876	pState->fCanResume = true;
2877	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
2878	return VINF_SUCCESS;
2879	#endif
2880	}
2881
2882
2883	/**
2884	* Handles an memory access VMEXIT.
2885	*
2886	* This can be triggered by a number of things.
2887	*
2888	* @returns Strict VBox status code.
2889	* @param pVM The cross context VM structure.
2890	* @param pVCpu The cross context virtual CPU structure.
2891	* @param pCtx The CPU context to update.
2892	* @param pMemCtx The exit reason information.
2893	*/
2894	static VBOXSTRICTRC nemR3WinHandleMemoryAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_MEMORY_ACCESS_CONTEXT const *pMemCtx)
2895	{
2896	/*
2897	* Ask PGM for information about the given GCPhys. We need to check if we're
2898	* out of sync first.
2899	*/
2900	NEMR3WINHMACPCCSTATE State = { pMemCtx->AccessInfo.AccessType == WHvMemoryAccessWrite, false, false };
2901	PGMPHYSNEMPAGEINFO Info;
2902	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pMemCtx->Gpa, State.fWriteAccess, &Info,
2903	nemR3WinHandleMemoryAccessPageCheckerCallback, &State);
2904	if (RT_SUCCESS(rc))
2905	{
2906	if (Info.fNemProt & (pMemCtx->AccessInfo.AccessType == WHvMemoryAccessWrite ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
2907	{
2908	if (State.fCanResume)
2909	{
2910	Log4(("MemExit: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
2911	pMemCtx->Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2912	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2913	State.fDidSomething ? "" : " no-change", g_apszWHvMemAccesstypes[pMemCtx->AccessInfo.AccessType]));
2914	return VINF_SUCCESS;
2915	}
2916	}
2917	Log4(("MemExit: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
2918	pMemCtx->Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2919	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2920	State.fDidSomething ? "" : " no-change", g_apszWHvMemAccesstypes[pMemCtx->AccessInfo.AccessType]));
2921	}
2922	else
2923	Log4(("MemExit: %RGp rc=%Rrc%s; emulating (%s)\n", pMemCtx->Gpa, rc,
2924	State.fDidSomething ? " modified-backing" : "", g_apszWHvMemAccesstypes[pMemCtx->AccessInfo.AccessType]));
2925
2926	/*
2927	* Emulate the memory access, either access handler or special memory.
2928	*/
2929	VBOXSTRICTRC rcStrict;
2930	if (pMemCtx->InstructionByteCount > 0)
2931	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(pCtx), pMemCtx->VpContext.Rip,
2932	pMemCtx->InstructionBytes, pMemCtx->InstructionByteCount);
2933	else
2934	rcStrict = IEMExecOne(pVCpu);
2935	/** @todo do we need to do anything wrt debugging here? */
2936	return rcStrict;
2937	}
2938
2939
2940	/**
2941	* Handles an I/O port access VMEXIT.
2942	*
2943	* We ASSUME that the hypervisor has don't I/O port access control.
2944	*
2945	* @returns Strict VBox status code.
2946	* @param pVM The cross context VM structure.
2947	* @param pVCpu The cross context virtual CPU structure.
2948	* @param pCtx The CPU context to update.
2949	* @param pIoPortCtx The exit reason information.
2950	*/
2951	static VBOXSTRICTRC nemR3WinHandleIoPortAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
2952	WHV_X64_IO_PORT_ACCESS_CONTEXT const *pIoPortCtx)
2953	{
2954	Assert( pIoPortCtx->AccessInfo.AccessSize == 1
2955	\|\| pIoPortCtx->AccessInfo.AccessSize == 2
2956	\|\| pIoPortCtx->AccessInfo.AccessSize == 4);
2957
2958	VBOXSTRICTRC rcStrict;
2959	if (!pIoPortCtx->AccessInfo.StringOp)
2960	{
2961	/*
2962	* Simple port I/O.
2963	*/
2964	Assert(pCtx->rax == pIoPortCtx->Rax);
2965
2966	static uint32_t const s_fAndMask[8] =
2967	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
2968	uint32_t const fAndMask = s_fAndMask[pIoPortCtx->AccessInfo.AccessSize];
2969	if (pIoPortCtx->AccessInfo.IsWrite)
2970	{
2971	rcStrict = IOMIOPortWrite(pVM, pVCpu, pIoPortCtx->PortNumber, (uint32_t)pIoPortCtx->Rax & fAndMask,
2972	pIoPortCtx->AccessInfo.AccessSize);
2973	if (IOM_SUCCESS(rcStrict))
2974	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, pCtx, &pIoPortCtx->VpContext);
2975	}
2976	else
2977	{
2978	uint32_t uValue = 0;
2979	rcStrict = IOMIOPortRead(pVM, pVCpu, pIoPortCtx->PortNumber, &uValue,
2980	pIoPortCtx->AccessInfo.AccessSize);
2981	if (IOM_SUCCESS(rcStrict))
2982	{
2983	pCtx->eax = (pCtx->eax & ~fAndMask) \| (uValue & fAndMask);
2984	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, pCtx, &pIoPortCtx->VpContext);
2985	}
2986	}
2987	}
2988	else
2989	{
2990	/*
2991	* String port I/O.
2992	*/
2993	/** @todo Someone at Microsoft please explain how we can get the address mode
2994	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
2995	* getting the default mode, it can always be overridden by a prefix. This
2996	* forces us to interpret the instruction from opcodes, which is suboptimal.
2997	* Both AMD-V and VT-x includes the address size in the exit info, at least on
2998	* CPUs that are reasonably new. */
2999	Assert( pIoPortCtx->Ds.Base == pCtx->ds.u64Base
3000	&& pIoPortCtx->Ds.Limit == pCtx->ds.u32Limit
3001	&& pIoPortCtx->Ds.Selector == pCtx->ds.Sel);
3002	Assert( pIoPortCtx->Es.Base == pCtx->es.u64Base
3003	&& pIoPortCtx->Es.Limit == pCtx->es.u32Limit
3004	&& pIoPortCtx->Es.Selector == pCtx->es.Sel);
3005	Assert(pIoPortCtx->Rdi == pCtx->rdi);
3006	Assert(pIoPortCtx->Rsi == pCtx->rsi);
3007	Assert(pIoPortCtx->Rcx == pCtx->rcx);
3008	Assert(pIoPortCtx->Rcx == pCtx->rcx);
3009
3010	rcStrict = IEMExecOne(pVCpu);
3011	}
3012	if (IOM_SUCCESS(rcStrict))
3013	{
3014	/*
3015	* Do debug checks.
3016	*/
3017	if ( pIoPortCtx->VpContext.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflext DR7? */
3018	\|\| (pIoPortCtx->VpContext.Rflags & X86_EFL_TF)
3019	\|\| DBGFBpIsHwIoArmed(pVM) )
3020	{
3021	/** @todo Debugging. */
3022	}
3023	}
3024	return rcStrict;
3025	}
3026
3027
3028	static VBOXSTRICTRC nemR3WinHandleInterruptWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
3029	{
3030	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
3031	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
3032	}
3033
3034
3035	static VBOXSTRICTRC nemR3WinHandleMsrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
3036	{
3037	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
3038	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
3039	}
3040
3041
3042	static VBOXSTRICTRC nemR3WinHandleCpuId(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
3043	{
3044	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
3045	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
3046	}
3047
3048
3049	static VBOXSTRICTRC nemR3WinHandleException(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
3050	{
3051	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
3052	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
3053	}
3054
3055
3056	static VBOXSTRICTRC nemR3WinHandleUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
3057	{
3058	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
3059	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
3060	}
3061
3062
3063	static VBOXSTRICTRC nemR3WinHandleTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
3064	{
3065	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
3066	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
3067	}
3068
3069
3070	static VBOXSTRICTRC nemR3WinHandleInvalidState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
3071	{
3072	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
3073	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
3074	}
3075
3076
3077	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
3078	{
3079	#ifdef LOG_ENABLED
3080	if (LogIs3Enabled())
3081	{
3082	Log3(("nemR3NativeRunGC: Entering #%u\n", pVCpu->idCpu));
3083	nemR3WinLogState(pVM, pVCpu);
3084	}
3085	#endif
3086
3087	/*
3088	* The run loop.
3089	*
3090	* Current approach to state updating to use the sledgehammer and sync
3091	* everything every time. This will be optimized later.
3092	*/
3093	const bool fSingleStepping = false; /** @todo get this from somewhere. */
3094	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
3095	for (unsigned iLoop = 0;;iLoop++)
3096	{
3097	/*
3098	* Copy the state.
3099	*/
3100	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
3101	int rc2 = nemR3WinCopyStateToHyperV(pVM, pVCpu, pCtx);
3102	AssertRCBreakStmt(rc2, rcStrict = rc2);
3103
3104	/*
3105	* Run a bit.
3106	*/
3107	WHV_RUN_VP_EXIT_CONTEXT ExitReason;
3108	RT_ZERO(ExitReason);
3109	if ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
3110	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3111	{
3112	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
3113	int rc2 = nemR3WinRunVirtualProcessor(pVM, pVCpu, &ExitReason, sizeof(ExitReason));
3114	AssertRCBreakStmt(rc2, rcStrict = rc2);
3115	#else
3116	Log8(("Calling WHvRunVirtualProcessor\n"));
3117	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED);
3118	HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
3119	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM);
3120	AssertLogRelMsgBreakStmt(SUCCEEDED(hrc),
3121	("WHvRunVirtualProcessor(%p, %u,,) -> %Rhrc (Last=%#x/%u)\n", pVM->nem.s.hPartition, pVCpu->idCpu,
3122	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()),
3123	rcStrict = VERR_INTERNAL_ERROR);
3124	Log2(("WHvRunVirtualProcessor -> %#x; exit code %#x (%d) (cpu status %u)\n",
3125	hrc, ExitReason.ExitReason, ExitReason.ExitReason, nemR3WinCpuGetRunningStatus(pVCpu) ));
3126	#endif
3127	}
3128	else
3129	{
3130	LogFlow(("nemR3NativeRunGC: returning: pending FF (pre exec)\n"));
3131	break;
3132	}
3133
3134	/*
3135	* Copy back the state.
3136	*/
3137	rc2 = nemR3WinCopyStateFromHyperV(pVM, pVCpu, pCtx);
3138	AssertRCBreakStmt(rc2, rcStrict = rc2);
3139
3140	#ifdef LOG_ENABLED
3141	/*
3142	* Do some logging.
3143	*/
3144	if (LogIs2Enabled())
3145	nemR3WinLogExitReason(&ExitReason);
3146	if (LogIs3Enabled())
3147	nemR3WinLogState(pVM, pVCpu);
3148	#endif
3149
3150	#ifdef VBOX_STRICT
3151	/* Assert that the VpContext field makes sense. */
3152	switch (ExitReason.ExitReason)
3153	{
3154	case WHvRunVpExitReasonMemoryAccess:
3155	case WHvRunVpExitReasonX64IoPortAccess:
3156	case WHvRunVpExitReasonX64MsrAccess:
3157	case WHvRunVpExitReasonX64Cpuid:
3158	case WHvRunVpExitReasonException:
3159	case WHvRunVpExitReasonUnrecoverableException:
3160	Assert( ExitReason.IoPortAccess.VpContext.InstructionLength > 0
3161	\|\| ( ExitReason.ExitReason == WHvRunVpExitReasonMemoryAccess
3162	&& ExitReason.MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessExecute));
3163	Assert(ExitReason.IoPortAccess.VpContext.InstructionLength < 16);
3164	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Cpl == CPUMGetGuestCPL(pVCpu));
3165	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Cr0Pe == RT_BOOL(pCtx->cr0 & X86_CR0_PE));
3166	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Cr0Am == RT_BOOL(pCtx->cr0 & X86_CR0_AM));
3167	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.EferLma == RT_BOOL(pCtx->msrEFER & MSR_K6_EFER_LMA));
3168	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.DebugActive == RT_BOOL(pCtx->dr[7] & X86_DR7_ENABLED_MASK));
3169	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Reserved0 == 0);
3170	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Reserved1 == 0);
3171	Assert(ExitReason.IoPortAccess.VpContext.Rip == pCtx->rip);
3172	Assert(ExitReason.IoPortAccess.VpContext.Rflags == pCtx->rflags.u);
3173	Assert( ExitReason.IoPortAccess.VpContext.Cs.Base == pCtx->cs.u64Base
3174	&& ExitReason.IoPortAccess.VpContext.Cs.Limit == pCtx->cs.u32Limit
3175	&& ExitReason.IoPortAccess.VpContext.Cs.Selector == pCtx->cs.Sel);
3176	break;
3177	default: break; /* shut up compiler. */
3178	}
3179	#endif
3180
3181	/*
3182	* Deal with the exit.
3183	*/
3184	switch (ExitReason.ExitReason)
3185	{
3186	/* Frequent exits: */
3187	case WHvRunVpExitReasonCanceled:
3188	case WHvRunVpExitReasonAlerted:
3189	rcStrict = VINF_SUCCESS;
3190	break;
3191
3192	case WHvRunVpExitReasonX64Halt:
3193	rcStrict = nemR3WinHandleHalt(pVM, pVCpu, pCtx);
3194	break;
3195
3196	case WHvRunVpExitReasonMemoryAccess:
3197	rcStrict = nemR3WinHandleMemoryAccess(pVM, pVCpu, pCtx, &ExitReason.MemoryAccess);
3198	break;
3199
3200	case WHvRunVpExitReasonX64IoPortAccess:
3201	rcStrict = nemR3WinHandleIoPortAccess(pVM, pVCpu, pCtx, &ExitReason.IoPortAccess);
3202	break;
3203
3204	case WHvRunVpExitReasonX64InterruptWindow:
3205	rcStrict = nemR3WinHandleInterruptWindow(pVM, pVCpu, pCtx, &ExitReason);
3206	break;
3207
3208	case WHvRunVpExitReasonX64MsrAccess: /* needs configuring */
3209	rcStrict = nemR3WinHandleMsrAccess(pVM, pVCpu, pCtx, &ExitReason);
3210	break;
3211
3212	case WHvRunVpExitReasonX64Cpuid: /* needs configuring */
3213	rcStrict = nemR3WinHandleCpuId(pVM, pVCpu, pCtx, &ExitReason);
3214	break;
3215
3216	case WHvRunVpExitReasonException: /* needs configuring */
3217	rcStrict = nemR3WinHandleException(pVM, pVCpu, pCtx, &ExitReason);
3218	break;
3219
3220	/* Unlikely exits: */
3221	case WHvRunVpExitReasonUnsupportedFeature:
3222	rcStrict = nemR3WinHandleUD(pVM, pVCpu, pCtx, &ExitReason);
3223	break;
3224
3225	case WHvRunVpExitReasonUnrecoverableException:
3226	rcStrict = nemR3WinHandleTripleFault(pVM, pVCpu, pCtx, &ExitReason);
3227	break;
3228
3229	case WHvRunVpExitReasonInvalidVpRegisterValue:
3230	rcStrict = nemR3WinHandleInvalidState(pVM, pVCpu, pCtx, &ExitReason);
3231	break;
3232
3233	/* Undesired exits: */
3234	case WHvRunVpExitReasonNone:
3235	default:
3236	AssertLogRelMsgFailed(("Unknown ExitReason: %#x\n", ExitReason.ExitReason));
3237	rcStrict = VERR_INTERNAL_ERROR_3;
3238	break;
3239	}
3240	if (rcStrict != VINF_SUCCESS)
3241	{
3242	LogFlow(("nemR3NativeRunGC: returning: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
3243	break;
3244	}
3245
3246	#ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3247	/* Hack alert! */
3248	uint32_t const cMappedPages = pVM->nem.s.cMappedPages;
3249	if (cMappedPages < 4000)
3250	{ /* likely */ }
3251	else
3252	{
3253	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinUnmapOnePageCallback, NULL);
3254	Log(("nemR3NativeRunGC: Unmapped all; cMappedPages=%u -> %u\n", cMappedPages, pVM->nem.s.cMappedPages));
3255	}
3256	#endif
3257
3258	/* If any FF is pending, return to the EM loops. That's okay for the
3259	current sledgehammer approach. */
3260	if ( VM_FF_IS_PENDING( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3261	\|\| VMCPU_FF_IS_PENDING(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3262	{
3263	LogFlow(("nemR3NativeRunGC: returning: pending FF (%#x / %#x)\n", pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
3264	break;
3265	}
3266	}
3267
3268	return rcStrict;
3269	}
3270
3271
3272	bool nemR3NativeCanExecuteGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3273	{
3274	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx);
3275	return true;
3276	}
3277
3278
3279	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
3280	{
3281	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
3282	return false;
3283	}
3284
3285
3286	/**
3287	* Forced flag notification call from VMEmt.h.
3288	*
3289	* This is only called when pVCpu is in the VMCPUSTATE_STARTED_EXEC_NEM state.
3290	*
3291	* @param pVM The cross context VM structure.
3292	* @param pVCpu The cross context virtual CPU structure of the CPU
3293	* to be notified.
3294	* @param fFlags Notification flags, VMNOTIFYFF_FLAGS_XXX.
3295	*/
3296	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
3297	{
3298	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
3299	nemR3WinCancelRunVirtualProcessor(pVM, pVCpu);
3300	#else
3301	Log8(("nemR3NativeNotifyFF: canceling %u\n", pVCpu->idCpu));
3302	HRESULT hrc = WHvCancelRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, 0);
3303	AssertMsg(SUCCEEDED(hrc), ("WHvCancelRunVirtualProcessor -> hrc=%Rhrc\n", hrc));
3304	RT_NOREF_PV(hrc);
3305	#endif
3306	RT_NOREF_PV(fFlags);
3307	}
3308
3309
3310	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
3311	{
3312	PGMPAGEMAPLOCK Lock;
3313	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
3314	if (RT_SUCCESS(rc))
3315	PGMPhysReleasePageMappingLock(pVM, &Lock);
3316	return rc;
3317	}
3318
3319
3320	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
3321	{
3322	PGMPAGEMAPLOCK Lock;
3323	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
3324	if (RT_SUCCESS(rc))
3325	PGMPhysReleasePageMappingLock(pVM, &Lock);
3326	return rc;
3327	}
3328
3329
3330	int nemR3NativeNotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
3331	{
3332	Log5(("nemR3NativeNotifyPhysRamRegister: %RGp LB %RGp\n", GCPhys, cb));
3333	NOREF(pVM); NOREF(GCPhys); NOREF(cb);
3334	return VINF_SUCCESS;
3335	}
3336
3337
3338	int nemR3NativeNotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvMmio2)
3339	{
3340	Log5(("nemR3NativeNotifyPhysMmioExMap: %RGp LB %RGp fFlags=%#x pvMmio2=%p\n", GCPhys, cb, fFlags, pvMmio2));
3341	NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags); NOREF(pvMmio2);
3342	return VINF_SUCCESS;
3343	}
3344
3345
3346	int nemR3NativeNotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
3347	{
3348	Log5(("nemR3NativeNotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
3349	NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
3350	return VINF_SUCCESS;
3351	}
3352
3353
3354	/**
3355	* Called early during ROM registration, right after the pages have been
3356	* allocated and the RAM range updated.
3357	*
3358	* This will be succeeded by a number of NEMHCNotifyPhysPageProtChanged() calls
3359	* and finally a NEMR3NotifyPhysRomRegisterEarly().
3360	*
3361	* @returns VBox status code
3362	* @param pVM The cross context VM structure.
3363	* @param GCPhys The ROM address (page aligned).
3364	* @param cb The size (page aligned).
3365	* @param fFlags NEM_NOTIFY_PHYS_ROM_F_XXX.
3366	*/
3367	int nemR3NativeNotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
3368	{
3369	Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
3370	#if 0 /* Let's not do this after all. We'll protection change notifications for each page and if not we'll map them lazily. */
3371	RTGCPHYS const cPages = cb >> X86_PAGE_SHIFT;
3372	for (RTGCPHYS iPage = 0; iPage < cPages; iPage++, GCPhys += X86_PAGE_SIZE)
3373	{
3374	const void *pvPage;
3375	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhys, &pvPage);
3376	if (RT_SUCCESS(rc))
3377	{
3378	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhys, X86_PAGE_SIZE,
3379	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute);
3380	if (SUCCEEDED(hrc))
3381	{ /* likely */ }
3382	else
3383	{
3384	LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
3385	GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
3386	return VERR_NEM_INIT_FAILED;
3387	}
3388	}
3389	else
3390	{
3391	LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
3392	return rc;
3393	}
3394	}
3395	#else
3396	NOREF(pVM); NOREF(GCPhys); NOREF(cb);
3397	#endif
3398	RT_NOREF_PV(fFlags);
3399	return VINF_SUCCESS;
3400	}
3401
3402
3403	/**
3404	* Called after the ROM range has been fully completed.
3405	*
3406	* This will be preceeded by a NEMR3NotifyPhysRomRegisterEarly() call as well a
3407	* number of NEMHCNotifyPhysPageProtChanged calls.
3408	*
3409	* @returns VBox status code
3410	* @param pVM The cross context VM structure.
3411	* @param GCPhys The ROM address (page aligned).
3412	* @param cb The size (page aligned).
3413	* @param fFlags NEM_NOTIFY_PHYS_ROM_F_XXX.
3414	*/
3415	int nemR3NativeNotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
3416	{
3417	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
3418	NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
3419	return VINF_SUCCESS;
3420	}
3421
3422
3423	/**
3424	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
3425	*/
3426	static DECLCALLBACK(int) nemR3WinUnsetForA20CheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys,
3427	PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
3428	{
3429	/* We'll just unmap the memory. */
3430	if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
3431	{
3432	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3433	int rc = nemR3WinHypercallUnmapPage(pVM, pVCpu, GCPhys);
3434	AssertRC(rc);
3435	if (RT_SUCCESS(rc))
3436	#else
3437	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
3438	if (SUCCEEDED(hrc))
3439	#endif
3440	{
3441	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3442	Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
3443	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
3444	}
3445	else
3446	{
3447	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3448	LogRel(("nemR3WinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
3449	return rc;
3450	#else
3451	LogRel(("nemR3WinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
3452	GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
3453	return VERR_INTERNAL_ERROR_2;
3454	#endif
3455	}
3456	}
3457	RT_NOREF(pVCpu, pvUser);
3458	return VINF_SUCCESS;
3459	}
3460
3461
3462	/**
3463	* Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
3464	*
3465	* @returns The PGMPhysNemQueryPageInfo result.
3466	* @param pVM The cross context VM structure.
3467	* @param pVCpu The cross context virtual CPU structure.
3468	* @param GCPhys The page to unmap.
3469	*/
3470	static int nemR3WinUnmapPageForA20Gate(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3471	{
3472	PGMPHYSNEMPAGEINFO Info;
3473	return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /fMakeWritable/, &Info,
3474	nemR3WinUnsetForA20CheckerCallback, NULL);
3475	}
3476
3477
3478	/**
3479	* Called when the A20 state changes.
3480	*
3481	* Hyper-V doesn't seem to offer a simple way of implementing the A20 line
3482	* features of PCs. So, we do a very minimal emulation of the HMA to make DOS
3483	* happy.
3484	*
3485	* @param pVCpu The CPU the A20 state changed on.
3486	* @param fEnabled Whether it was enabled (true) or disabled.
3487	*/
3488	void nemR3NativeNotifySetA20(PVMCPU pVCpu, bool fEnabled)
3489	{
3490	Log(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
3491	PVM pVM = pVCpu->CTX_SUFF(pVM);
3492	if (!pVM->nem.s.fA20Fixed)
3493	{
3494	pVM->nem.s.fA20Enabled = fEnabled;
3495	for (RTGCPHYS GCPhys = _1M; GCPhys < _1M + _64K; GCPhys += X86_PAGE_SIZE)
3496	nemR3WinUnmapPageForA20Gate(pVM, pVCpu, GCPhys);
3497	}
3498	}
3499
3500
3501	void nemR3NativeNotifyHandlerPhysicalRegister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
3502	{
3503	Log5(("nemR3NativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
3504	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
3505	}
3506
3507
3508	void nemR3NativeNotifyHandlerPhysicalDeregister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
3509	int fRestoreAsRAM, bool fRestoreAsRAM2)
3510	{
3511	Log5(("nemR3NativeNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d fRestoreAsRAM=%d fRestoreAsRAM2=%d\n",
3512	GCPhys, cb, enmKind, fRestoreAsRAM, fRestoreAsRAM2));
3513	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb); NOREF(fRestoreAsRAM); NOREF(fRestoreAsRAM2);
3514	}
3515
3516
3517	void nemR3NativeNotifyHandlerPhysicalModify(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
3518	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
3519	{
3520	Log5(("nemR3NativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
3521	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
3522	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
3523	}
3524
3525
3526	/**
3527	* Worker that maps pages into Hyper-V.
3528	*
3529	* This is used by the PGM physical page notifications as well as the memory
3530	* access VMEXIT handlers.
3531	*
3532	* @returns VBox status code.
3533	* @param pVM The cross context VM structure.
3534	* @param pVCpu The cross context virtual CPU structure of the
3535	* calling EMT.
3536	* @param GCPhysSrc The source page address.
3537	* @param GCPhysDst The hyper-V destination page. This may differ from
3538	* GCPhysSrc when A20 is disabled.
3539	* @param fPageProt NEM_PAGE_PROT_XXX.
3540	* @param pu2State Our page state (input/output).
3541	* @param fBackingChanged Set if the page backing is being changed.
3542	* @thread EMT(pVCpu)
3543	*/
3544	static int nemR3NativeSetPhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fPageProt,
3545	uint8_t *pu2State, bool fBackingChanged)
3546	{
3547	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3548	/*
3549	* When using the hypercalls instead of the ring-3 APIs, we don't need to
3550	* unmap memory before modifying it. We still want to track the state though,
3551	* since unmap will fail when called an unmapped page and we don't want to redo
3552	* upgrades/downgrades.
3553	*/
3554	uint8_t const u2OldState = *pu2State;
3555	int rc;
3556	if (fPageProt == NEM_PAGE_PROT_NONE)
3557	{
3558	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
3559	{
3560	rc = nemR3WinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
3561	if (RT_SUCCESS(rc))
3562	{
3563	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
3564	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3565	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
3566	}
3567	else
3568	AssertLogRelMsgFailed(("nemR3NativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
3569	}
3570	else
3571	rc = VINF_SUCCESS;
3572	}
3573	else if (fPageProt & NEM_PAGE_PROT_WRITE)
3574	{
3575	if (u2OldState != NEM_WIN_PAGE_STATE_WRITABLE \|\| fBackingChanged)
3576	{
3577	rc = nemR3WinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
3578	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
3579	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
3580	if (RT_SUCCESS(rc))
3581	{
3582	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
3583	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
3584	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
3585	Log5(("NEM GPA writable/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
3586	NOREF(cMappedPages);
3587	}
3588	else
3589	AssertLogRelMsgFailed(("nemR3NativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
3590	}
3591	else
3592	rc = VINF_SUCCESS;
3593	}
3594	else
3595	{
3596	if (u2OldState != NEM_WIN_PAGE_STATE_READABLE \|\| fBackingChanged)
3597	{
3598	rc = nemR3WinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
3599	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
3600	if (RT_SUCCESS(rc))
3601	{
3602	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
3603	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
3604	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
3605	Log5(("NEM GPA read+exec/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
3606	NOREF(cMappedPages);
3607	}
3608	else
3609	AssertLogRelMsgFailed(("nemR3NativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
3610	}
3611	else
3612	rc = VINF_SUCCESS;
3613	}
3614
3615	return VINF_SUCCESS;
3616
3617	#else
3618	/*
3619	* Looks like we need to unmap a page before we can change the backing
3620	* or even modify the protection. This is going to be REALLY efficient.
3621	* PGM lends us two bits to keep track of the state here.
3622	*/
3623	uint8_t const u2OldState = *pu2State;
3624	uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
3625	: fPageProt & NEM_PAGE_PROT_READ ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED;
3626	if ( fBackingChanged
3627	\|\| u2NewState != u2OldState)
3628	{
3629	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
3630	{
3631	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3632	int rc = nemR3WinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
3633	AssertRC(rc);
3634	if (RT_SUCCESS(rc))
3635	{
3636	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
3637	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3638	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
3639	{
3640	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
3641	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
3642	return VINF_SUCCESS;
3643	}
3644	}
3645	else
3646	{
3647	LogRel(("nemR3NativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
3648	return rc;
3649	}
3650	# else
3651	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE);
3652	if (SUCCEEDED(hrc))
3653	{
3654	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
3655	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3656	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
3657	{
3658	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
3659	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
3660	return VINF_SUCCESS;
3661	}
3662	}
3663	else
3664	{
3665	LogRel(("nemR3NativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
3666	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
3667	return VERR_NEM_INIT_FAILED;
3668	}
3669	# endif
3670	}
3671	}
3672
3673	/*
3674	* Writeable mapping?
3675	*/
3676	if (fPageProt & NEM_PAGE_PROT_WRITE)
3677	{
3678	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3679	int rc = nemR3WinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
3680	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
3681	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
3682	AssertRC(rc);
3683	if (RT_SUCCESS(rc))
3684	{
3685	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
3686	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3687	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
3688	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
3689	return VINF_SUCCESS;
3690	}
3691	LogRel(("nemR3NativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
3692	return rc;
3693	# else
3694	void *pvPage;
3695	int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
3696	if (RT_SUCCESS(rc))
3697	{
3698	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE,
3699	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute \| WHvMapGpaRangeFlagWrite);
3700	if (SUCCEEDED(hrc))
3701	{
3702	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
3703	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3704	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
3705	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
3706	return VINF_SUCCESS;
3707	}
3708	LogRel(("nemR3NativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
3709	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
3710	return VERR_NEM_INIT_FAILED;
3711	}
3712	LogRel(("nemR3NativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
3713	return rc;
3714	# endif
3715	}
3716
3717	if (fPageProt & NEM_PAGE_PROT_READ)
3718	{
3719	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3720	int rc = nemR3WinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
3721	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
3722	AssertRC(rc);
3723	if (RT_SUCCESS(rc))
3724	{
3725	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
3726	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3727	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
3728	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
3729	return VINF_SUCCESS;
3730	}
3731	LogRel(("nemR3NativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
3732	return rc;
3733	# else
3734	const void *pvPage;
3735	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
3736	if (RT_SUCCESS(rc))
3737	{
3738	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE,
3739	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute);
3740	if (SUCCEEDED(hrc))
3741	{
3742	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
3743	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3744	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
3745	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
3746	return VINF_SUCCESS;
3747	}
3748	LogRel(("nemR3NativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
3749	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
3750	return VERR_NEM_INIT_FAILED;
3751	}
3752	LogRel(("nemR3NativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
3753	return rc;
3754	# endif
3755	}
3756
3757	/* We already unmapped it above. */
3758	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
3759	return VINF_SUCCESS;
3760	#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
3761	}
3762
3763
3764	static int nemR3JustUnmapPageFromHyperV(PVM pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
3765	{
3766	if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED)
3767	{
3768	Log5(("nemR3JustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst));
3769	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
3770	return VINF_SUCCESS;
3771	}
3772
3773	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3774	PVMCPU pVCpu = VMMGetCpu(pVM);
3775	int rc = nemR3WinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
3776	AssertRC(rc);
3777	if (RT_SUCCESS(rc))
3778	{
3779	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3780	Log5(("NEM GPA unmapped/just: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[*pu2State], cMappedPages));
3781	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
3782	return VINF_SUCCESS;
3783	}
3784	LogRel(("nemR3JustUnmapPageFromHyperV/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
3785	return rc;
3786	#else
3787	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
3788	if (SUCCEEDED(hrc))
3789	{
3790	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
3791	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
3792	Log5(("nemR3JustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
3793	return VINF_SUCCESS;
3794	}
3795	LogRel(("nemR3JustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n",
3796	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
3797	return VERR_INTERNAL_ERROR_3;
3798	#endif
3799	}
3800
3801
3802	int nemR3NativeNotifyPhysPageAllocated(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
3803	PGMPAGETYPE enmType, uint8_t *pu2State)
3804	{
3805	Log5(("nemR3NativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3806	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3807	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
3808
3809	int rc;
3810	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3811	PVMCPU pVCpu = VMMGetCpu(pVM);
3812	if ( pVM->nem.s.fA20Enabled
3813	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
3814	rc = nemR3NativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
3815	else
3816	{
3817	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
3818	rc = nemR3WinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
3819	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys) && RT_SUCCESS(rc))
3820	rc = nemR3NativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
3821
3822	}
3823	#else
3824	RT_NOREF_PV(fPageProt);
3825	if ( pVM->nem.s.fA20Enabled
3826	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
3827	rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
3828	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
3829	rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
3830	else
3831	rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */
3832	#endif
3833	return rc;
3834	}
3835
3836
3837	void nemR3NativeNotifyPhysPageProtChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
3838	PGMPAGETYPE enmType, uint8_t *pu2State)
3839	{
3840	Log5(("nemR3NativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3841	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3842	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
3843
3844	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3845	PVMCPU pVCpu = VMMGetCpu(pVM);
3846	if ( pVM->nem.s.fA20Enabled
3847	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
3848	nemR3NativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
3849	else
3850	{
3851	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
3852	nemR3WinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
3853	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
3854	nemR3NativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
3855	}
3856	#else
3857	RT_NOREF_PV(fPageProt);
3858	if ( pVM->nem.s.fA20Enabled
3859	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
3860	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
3861	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
3862	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
3863	/* else: ignore since we've got the alias page at this address. */
3864	#endif
3865	}
3866
3867
3868	void nemR3NativeNotifyPhysPageChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
3869	uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
3870	{
3871	Log5(("nemR3NativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3872	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
3873	RT_NOREF_PV(HCPhysPrev); RT_NOREF_PV(HCPhysNew); RT_NOREF_PV(enmType);
3874
3875	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
3876	PVMCPU pVCpu = VMMGetCpu(pVM);
3877	if ( pVM->nem.s.fA20Enabled
3878	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
3879	nemR3NativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
3880	else
3881	{
3882	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
3883	nemR3WinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
3884	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
3885	nemR3NativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
3886	}
3887	#else
3888	RT_NOREF_PV(fPageProt);
3889	if ( pVM->nem.s.fA20Enabled
3890	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
3891	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
3892	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
3893	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
3894	/* else: ignore since we've got the alias page at this address. */
3895	#endif
3896	}
3897

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp@ 71136

Download in other formats: