NEMR3Native-win.cpp@ 72526

Last change on this file since 72526 was 72526, checked in by vboxsync, 7 years ago
NEM,TM: More TSC adjustments for NEM/win. bugref:9044
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 127.3 KB

Line
1	/* $Id: NEMR3Native-win.cpp 72526 2018-06-12 13:06:02Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 Windows backend.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	* Log group 6: Ring-0 memory management
9	* Log group 12: API intercepts.
10	*/
11
12	/*
13	* Copyright (C) 2018 Oracle Corporation
14	*
15	* This file is part of VirtualBox Open Source Edition (OSE), as
16	* available from http://www.virtualbox.org. This file is free software;
17	* you can redistribute it and/or modify it under the terms of the GNU
18	* General Public License (GPL) as published by the Free Software
19	* Foundation, in version 2 as it comes in the "COPYING" file of the
20	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22	*/
23
24
25	/*********************************************************************************************************************************
26	* Header Files *
27	*********************************************************************************************************************************/
28	#define LOG_GROUP LOG_GROUP_NEM
29	#define VMCPU_INCL_CPUM_GST_CTX
30	#include <iprt/nt/nt-and-windows.h>
31	#include <iprt/nt/hyperv.h>
32	#include <iprt/nt/vid.h>
33	#include <WinHvPlatform.h>
34
35	#ifndef _WIN32_WINNT_WIN10
36	# error "Missing _WIN32_WINNT_WIN10"
37	#endif
38	#ifndef _WIN32_WINNT_WIN10_RS1 /* Missing define, causing trouble for us. */
39	# define _WIN32_WINNT_WIN10_RS1 (_WIN32_WINNT_WIN10 + 1)
40	#endif
41	#include <sysinfoapi.h>
42	#include <debugapi.h>
43	#include <errhandlingapi.h>
44	#include <fileapi.h>
45	#include <winerror.h> /* no api header for this. */
46
47	#include <VBox/vmm/nem.h>
48	#include <VBox/vmm/iem.h>
49	#include <VBox/vmm/em.h>
50	#include <VBox/vmm/apic.h>
51	#include <VBox/vmm/pdm.h>
52	#include "NEMInternal.h"
53	#include <VBox/vmm/vm.h>
54
55	#include <iprt/ldr.h>
56	#include <iprt/path.h>
57	#include <iprt/string.h>
58	#include <iprt/system.h>
59
60
61	/*********************************************************************************************************************************
62	* Defined Constants And Macros *
63	*********************************************************************************************************************************/
64	#ifdef LOG_ENABLED
65	# define NEM_WIN_INTERCEPT_NT_IO_CTLS
66	#endif
67
68	/** VID I/O control detection: Fake partition handle input. */
69	#define NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE ((HANDLE)(uintptr_t)38479125)
70	/** VID I/O control detection: Fake partition ID return. */
71	#define NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID UINT64_C(0xfa1e000042424242)
72	/** VID I/O control detection: Fake CPU index input. */
73	#define NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX UINT32_C(42)
74	/** VID I/O control detection: Fake timeout input. */
75	#define NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT UINT32_C(0x00080286)
76
77
78	/*********************************************************************************************************************************
79	* Global Variables *
80	*********************************************************************************************************************************/
81	/** @name APIs imported from WinHvPlatform.dll
82	* @{ */
83	static decltype(WHvGetCapability) * g_pfnWHvGetCapability;
84	static decltype(WHvCreatePartition) * g_pfnWHvCreatePartition;
85	static decltype(WHvSetupPartition) * g_pfnWHvSetupPartition;
86	static decltype(WHvDeletePartition) * g_pfnWHvDeletePartition;
87	static decltype(WHvGetPartitionProperty) * g_pfnWHvGetPartitionProperty;
88	static decltype(WHvSetPartitionProperty) * g_pfnWHvSetPartitionProperty;
89	static decltype(WHvMapGpaRange) * g_pfnWHvMapGpaRange;
90	static decltype(WHvUnmapGpaRange) * g_pfnWHvUnmapGpaRange;
91	static decltype(WHvTranslateGva) * g_pfnWHvTranslateGva;
92	#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
93	static decltype(WHvCreateVirtualProcessor) * g_pfnWHvCreateVirtualProcessor;
94	static decltype(WHvDeleteVirtualProcessor) * g_pfnWHvDeleteVirtualProcessor;
95	static decltype(WHvRunVirtualProcessor) * g_pfnWHvRunVirtualProcessor;
96	static decltype(WHvCancelRunVirtualProcessor) * g_pfnWHvCancelRunVirtualProcessor;
97	static decltype(WHvGetVirtualProcessorRegisters) * g_pfnWHvGetVirtualProcessorRegisters;
98	static decltype(WHvSetVirtualProcessorRegisters) * g_pfnWHvSetVirtualProcessorRegisters;
99	#endif
100	/** @} */
101
102	/** @name APIs imported from Vid.dll
103	* @{ */
104	static decltype(VidGetHvPartitionId) *g_pfnVidGetHvPartitionId;
105	static decltype(VidStartVirtualProcessor) *g_pfnVidStartVirtualProcessor;
106	static decltype(VidStopVirtualProcessor) *g_pfnVidStopVirtualProcessor;
107	static decltype(VidMessageSlotMap) *g_pfnVidMessageSlotMap;
108	static decltype(VidMessageSlotHandleAndGetNext) *g_pfnVidMessageSlotHandleAndGetNext;
109	#ifdef LOG_ENABLED
110	static decltype(VidGetVirtualProcessorState) *g_pfnVidGetVirtualProcessorState;
111	static decltype(VidSetVirtualProcessorState) *g_pfnVidSetVirtualProcessorState;
112	static decltype(VidGetVirtualProcessorRunningStatus) *g_pfnVidGetVirtualProcessorRunningStatus;
113	#endif
114	/** @} */
115
116	/** The Windows build number. */
117	static uint32_t g_uBuildNo = 17134;
118
119
120
121	/**
122	* Import instructions.
123	*/
124	static const struct
125	{
126	uint8_t idxDll; /*< 0 for WinHvPlatform.dll, 1 for vid.dll. /
127	bool fOptional; /*< Set if import is optional. /
128	PFNRT ppfn; /< The function pointer variable. /
129	const char pszName; /< The function name. /
130	} g_aImports[] =
131	{
132	#define NEM_WIN_IMPORT(a_idxDll, a_fOptional, a_Name) { (a_idxDll), (a_fOptional), (PFNRT *)&RT_CONCAT(g_pfn,a_Name), #a_Name }
133	NEM_WIN_IMPORT(0, false, WHvGetCapability),
134	NEM_WIN_IMPORT(0, false, WHvCreatePartition),
135	NEM_WIN_IMPORT(0, false, WHvSetupPartition),
136	NEM_WIN_IMPORT(0, false, WHvDeletePartition),
137	NEM_WIN_IMPORT(0, false, WHvGetPartitionProperty),
138	NEM_WIN_IMPORT(0, false, WHvSetPartitionProperty),
139	NEM_WIN_IMPORT(0, false, WHvMapGpaRange),
140	NEM_WIN_IMPORT(0, false, WHvUnmapGpaRange),
141	NEM_WIN_IMPORT(0, false, WHvTranslateGva),
142	#ifndef NEM_WIN_USE_OUR_OWN_RUN_API
143	NEM_WIN_IMPORT(0, false, WHvCreateVirtualProcessor),
144	NEM_WIN_IMPORT(0, false, WHvDeleteVirtualProcessor),
145	NEM_WIN_IMPORT(0, false, WHvRunVirtualProcessor),
146	NEM_WIN_IMPORT(0, false, WHvCancelRunVirtualProcessor),
147	NEM_WIN_IMPORT(0, false, WHvGetVirtualProcessorRegisters),
148	NEM_WIN_IMPORT(0, false, WHvSetVirtualProcessorRegisters),
149	#endif
150	NEM_WIN_IMPORT(1, false, VidGetHvPartitionId),
151	NEM_WIN_IMPORT(1, false, VidMessageSlotMap),
152	NEM_WIN_IMPORT(1, false, VidMessageSlotHandleAndGetNext),
153	NEM_WIN_IMPORT(1, false, VidStartVirtualProcessor),
154	NEM_WIN_IMPORT(1, false, VidStopVirtualProcessor),
155	#ifdef LOG_ENABLED
156	NEM_WIN_IMPORT(1, false, VidGetVirtualProcessorState),
157	NEM_WIN_IMPORT(1, false, VidSetVirtualProcessorState),
158	NEM_WIN_IMPORT(1, false, VidGetVirtualProcessorRunningStatus),
159	#endif
160	#undef NEM_WIN_IMPORT
161	};
162
163
164	/** The real NtDeviceIoControlFile API in NTDLL. */
165	static decltype(NtDeviceIoControlFile) *g_pfnNtDeviceIoControlFile;
166	/** Pointer to the NtDeviceIoControlFile import table entry. */
167	static decltype(NtDeviceIoControlFile) **g_ppfnVidNtDeviceIoControlFile;
168	/** Info about the VidGetHvPartitionId I/O control interface. */
169	static NEMWINIOCTL g_IoCtlGetHvPartitionId;
170	/** Info about the VidStartVirtualProcessor I/O control interface. */
171	static NEMWINIOCTL g_IoCtlStartVirtualProcessor;
172	/** Info about the VidStopVirtualProcessor I/O control interface. */
173	static NEMWINIOCTL g_IoCtlStopVirtualProcessor;
174	/** Info about the VidMessageSlotHandleAndGetNext I/O control interface. */
175	static NEMWINIOCTL g_IoCtlMessageSlotHandleAndGetNext;
176	#ifdef LOG_ENABLED
177	/** Info about the VidMessageSlotMap I/O control interface - for logging. */
178	static NEMWINIOCTL g_IoCtlMessageSlotMap;
179	/* Info about the VidGetVirtualProcessorState I/O control interface - for logging. */
180	static NEMWINIOCTL g_IoCtlGetVirtualProcessorState;
181	/* Info about the VidSetVirtualProcessorState I/O control interface - for logging. */
182	static NEMWINIOCTL g_IoCtlSetVirtualProcessorState;
183	/** Pointer to what nemR3WinIoctlDetector_ForLogging should fill in. */
184	static NEMWINIOCTL *g_pIoCtlDetectForLogging;
185	#endif
186
187	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
188	/** Mapping slot for CPU #0.
189	* @{ */
190	static VID_MESSAGE_MAPPING_HEADER *g_pMsgSlotMapping = NULL;
191	static const HV_MESSAGE_HEADER *g_pHvMsgHdr;
192	static const HV_X64_INTERCEPT_MESSAGE_HEADER *g_pX64MsgHdr;
193	/** @} */
194	#endif
195
196
197	/*
198	* Let the preprocessor alias the APIs to import variables for better autocompletion.
199	*/
200	#ifndef IN_SLICKEDIT
201	# define WHvGetCapability g_pfnWHvGetCapability
202	# define WHvCreatePartition g_pfnWHvCreatePartition
203	# define WHvSetupPartition g_pfnWHvSetupPartition
204	# define WHvDeletePartition g_pfnWHvDeletePartition
205	# define WHvGetPartitionProperty g_pfnWHvGetPartitionProperty
206	# define WHvSetPartitionProperty g_pfnWHvSetPartitionProperty
207	# define WHvMapGpaRange g_pfnWHvMapGpaRange
208	# define WHvUnmapGpaRange g_pfnWHvUnmapGpaRange
209	# define WHvTranslateGva g_pfnWHvTranslateGva
210	# define WHvCreateVirtualProcessor g_pfnWHvCreateVirtualProcessor
211	# define WHvDeleteVirtualProcessor g_pfnWHvDeleteVirtualProcessor
212	# define WHvRunVirtualProcessor g_pfnWHvRunVirtualProcessor
213	# define WHvGetRunExitContextSize g_pfnWHvGetRunExitContextSize
214	# define WHvCancelRunVirtualProcessor g_pfnWHvCancelRunVirtualProcessor
215	# define WHvGetVirtualProcessorRegisters g_pfnWHvGetVirtualProcessorRegisters
216	# define WHvSetVirtualProcessorRegisters g_pfnWHvSetVirtualProcessorRegisters
217
218	# define VidMessageSlotHandleAndGetNext g_pfnVidMessageSlotHandleAndGetNext
219	# define VidStartVirtualProcessor g_pfnVidStartVirtualProcessor
220	# define VidStopVirtualProcessor g_pfnVidStopVirtualProcessor
221
222	#endif
223
224	/** WHV_MEMORY_ACCESS_TYPE names */
225	static const char * const g_apszWHvMemAccesstypes[4] = { "read", "write", "exec", "!undefined!" };
226
227
228	/*********************************************************************************************************************************
229	* Internal Functions *
230	*********************************************************************************************************************************/
231
232	/*
233	* Instantate the code we share with ring-0.
234	*/
235	#include "../VMMAll/NEMAllNativeTemplate-win.cpp.h"
236
237
238
239	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
240	/**
241	* Wrapper that logs the call from VID.DLL.
242	*
243	* This is very handy for figuring out why an API call fails.
244	*/
245	static NTSTATUS WINAPI
246	nemR3WinLogWrapper_NtDeviceIoControlFile(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
247	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
248	PVOID pvOutput, ULONG cbOutput)
249	{
250
251	char szFunction[32];
252	const char *pszFunction;
253	if (uFunction == g_IoCtlMessageSlotHandleAndGetNext.uFunction)
254	pszFunction = "VidMessageSlotHandleAndGetNext";
255	else if (uFunction == g_IoCtlStartVirtualProcessor.uFunction)
256	pszFunction = "VidStartVirtualProcessor";
257	else if (uFunction == g_IoCtlStopVirtualProcessor.uFunction)
258	pszFunction = "VidStopVirtualProcessor";
259	else if (uFunction == g_IoCtlMessageSlotMap.uFunction)
260	pszFunction = "VidMessageSlotMap";
261	else if (uFunction == g_IoCtlGetVirtualProcessorState.uFunction)
262	pszFunction = "VidGetVirtualProcessorState";
263	else if (uFunction == g_IoCtlSetVirtualProcessorState.uFunction)
264	pszFunction = "VidSetVirtualProcessorState";
265	else
266	{
267	RTStrPrintf(szFunction, sizeof(szFunction), "%#x", uFunction);
268	pszFunction = szFunction;
269	}
270
271	if (cbInput > 0 && pvInput)
272	Log12(("VID!NtDeviceIoControlFile: %s/input: %.*Rhxs\n", pszFunction, RT_MIN(cbInput, 32), pvInput));
273	NTSTATUS rcNt = g_pfnNtDeviceIoControlFile(hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, uFunction,
274	pvInput, cbInput, pvOutput, cbOutput);
275	if (!hEvt && !pfnApcCallback && !pvApcCtx)
276	Log12(("VID!NtDeviceIoControlFile: hFile=%#zx pIos=%p->{s:%#x, i:%#zx} uFunction=%s Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
277	hFile, pIos, pIos->Status, pIos->Information, pszFunction, pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
278	else
279	Log12(("VID!NtDeviceIoControlFile: hFile=%#zx hEvt=%#zx Apc=%p/%p pIos=%p->{s:%#x, i:%#zx} uFunction=%s Input=%p LB %#x Output=%p LB %#x) -> %#x; Caller=%p\n",
280	hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, pIos->Status, pIos->Information, pszFunction,
281	pvInput, cbInput, pvOutput, cbOutput, rcNt, ASMReturnAddress()));
282	if (cbOutput > 0 && pvOutput)
283	{
284	Log12(("VID!NtDeviceIoControlFile: %s/output: %.*Rhxs\n", pszFunction, RT_MIN(cbOutput, 32), pvOutput));
285	if (uFunction == 0x2210cc && g_pMsgSlotMapping == NULL && cbOutput >= sizeof(void *))
286	{
287	g_pMsgSlotMapping = (VID_MESSAGE_MAPPING_HEADER *)pvOutput;
288	g_pHvMsgHdr = (const HV_MESSAGE_HEADER *)(g_pMsgSlotMapping + 1);
289	g_pX64MsgHdr = (const HV_X64_INTERCEPT_MESSAGE_HEADER *)(g_pHvMsgHdr + 1);
290	Log12(("VID!NtDeviceIoControlFile: Message slot mapping: %p\n", g_pMsgSlotMapping));
291	}
292	}
293	if ( g_pMsgSlotMapping
294	&& ( uFunction == g_IoCtlMessageSlotHandleAndGetNext.uFunction
295	\|\| uFunction == g_IoCtlStopVirtualProcessor.uFunction
296	\|\| uFunction == g_IoCtlMessageSlotMap.uFunction
297	))
298	Log12(("VID!NtDeviceIoControlFile: enmVidMsgType=%#x cb=%#x msg=%#x payload=%u cs:rip=%04x:%08RX64 (%s)\n",
299	g_pMsgSlotMapping->enmVidMsgType, g_pMsgSlotMapping->cbMessage,
300	g_pHvMsgHdr->MessageType, g_pHvMsgHdr->PayloadSize,
301	g_pX64MsgHdr->CsSegment.Selector, g_pX64MsgHdr->Rip, pszFunction));
302
303	return rcNt;
304	}
305	#endif /* NEM_WIN_INTERCEPT_NT_IO_CTLS */
306
307
308	/**
309	* Patches the call table of VID.DLL so we can intercept NtDeviceIoControlFile.
310	*
311	* This is for used to figure out the I/O control codes and in logging builds
312	* for logging API calls that WinHvPlatform.dll does.
313	*
314	* @returns VBox status code.
315	* @param hLdrModVid The VID module handle.
316	* @param pErrInfo Where to return additional error information.
317	*/
318	static int nemR3WinInitVidIntercepts(RTLDRMOD hLdrModVid, PRTERRINFO pErrInfo)
319	{
320	/*
321	* Locate the real API.
322	*/
323	g_pfnNtDeviceIoControlFile = (decltype(NtDeviceIoControlFile) *)RTLdrGetSystemSymbol("NTDLL.DLL", "NtDeviceIoControlFile");
324	AssertReturn(g_pfnNtDeviceIoControlFile != NULL,
325	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Failed to resolve NtDeviceIoControlFile from NTDLL.DLL"));
326
327	/*
328	* Locate the PE header and get what we need from it.
329	*/
330	uint8_t const pbImage = (uint8_t const )RTLdrGetNativeHandle(hLdrModVid);
331	IMAGE_DOS_HEADER const pMzHdr = (IMAGE_DOS_HEADER const )pbImage;
332	AssertReturn(pMzHdr->e_magic == IMAGE_DOS_SIGNATURE,
333	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL mapping doesn't start with MZ signature: %#x", pMzHdr->e_magic));
334	IMAGE_NT_HEADERS const pNtHdrs = (IMAGE_NT_HEADERS const )&pbImage[pMzHdr->e_lfanew];
335	AssertReturn(pNtHdrs->Signature == IMAGE_NT_SIGNATURE,
336	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL has invalid PE signaturre: %#x @%#x",
337	pNtHdrs->Signature, pMzHdr->e_lfanew));
338
339	uint32_t const cbImage = pNtHdrs->OptionalHeader.SizeOfImage;
340	IMAGE_DATA_DIRECTORY const ImportDir = pNtHdrs->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
341
342	/*
343	* Walk the import descriptor table looking for NTDLL.DLL.
344	*/
345	AssertReturn( ImportDir.Size > 0
346	&& ImportDir.Size < cbImage,
347	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory size: %#x", ImportDir.Size));
348	AssertReturn( ImportDir.VirtualAddress > 0
349	&& ImportDir.VirtualAddress <= cbImage - ImportDir.Size,
350	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory RVA: %#x", ImportDir.VirtualAddress));
351
352	for (PIMAGE_IMPORT_DESCRIPTOR pImps = (PIMAGE_IMPORT_DESCRIPTOR)&pbImage[ImportDir.VirtualAddress];
353	pImps->Name != 0 && pImps->FirstThunk != 0;
354	pImps++)
355	{
356	AssertReturn(pImps->Name < cbImage,
357	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad import directory entry name: %#x", pImps->Name));
358	const char pszModName = (const char )&pbImage[pImps->Name];
359	if (RTStrICmpAscii(pszModName, "ntdll.dll"))
360	continue;
361	AssertReturn(pImps->FirstThunk < cbImage,
362	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
363	AssertReturn(pImps->OriginalFirstThunk < cbImage,
364	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
365
366	/*
367	* Walk the thunks table(s) looking for NtDeviceIoControlFile.
368	*/
369	PIMAGE_THUNK_DATA pFirstThunk = (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]; /* update this. */
370	PIMAGE_THUNK_DATA pThunk = pImps->OriginalFirstThunk == 0 /* read from this. */
371	? (PIMAGE_THUNK_DATA)&pbImage[pImps->FirstThunk]
372	: (PIMAGE_THUNK_DATA)&pbImage[pImps->OriginalFirstThunk];
373	while (pThunk->u1.Ordinal != 0)
374	{
375	if (!(pThunk->u1.Ordinal & IMAGE_ORDINAL_FLAG32))
376	{
377	AssertReturn(pThunk->u1.Ordinal > 0 && pThunk->u1.Ordinal < cbImage,
378	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "VID.DLL bad FirstThunk: %#x", pImps->FirstThunk));
379
380	const char pszSymbol = (const char )&pbImage[(uintptr_t)pThunk->u1.AddressOfData + 2];
381	if (strcmp(pszSymbol, "NtDeviceIoControlFile") == 0)
382	{
383	DWORD fOldProt = PAGE_READONLY;
384	VirtualProtect(&pFirstThunk->u1.Function, sizeof(uintptr_t), PAGE_EXECUTE_READWRITE, &fOldProt);
385	g_ppfnVidNtDeviceIoControlFile = (decltype(NtDeviceIoControlFile) **)&pFirstThunk->u1.Function;
386	/* Don't restore the protection here, so we modify the NtDeviceIoControlFile pointer later. */
387	}
388	}
389
390	pThunk++;
391	pFirstThunk++;
392	}
393	}
394
395	if (*g_ppfnVidNtDeviceIoControlFile)
396	{
397	#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
398	*g_ppfnVidNtDeviceIoControlFile = nemR3WinLogWrapper_NtDeviceIoControlFile;
399	#endif
400	return VINF_SUCCESS;
401	}
402	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Failed to patch NtDeviceIoControlFile import in VID.DLL!");
403	}
404
405
406	/**
407	* Worker for nemR3NativeInit that probes and load the native API.
408	*
409	* @returns VBox status code.
410	* @param fForced Whether the HMForced flag is set and we should
411	* fail if we cannot initialize.
412	* @param pErrInfo Where to always return error info.
413	*/
414	static int nemR3WinInitProbeAndLoad(bool fForced, PRTERRINFO pErrInfo)
415	{
416	/*
417	* Check that the DLL files we need are present, but without loading them.
418	* We'd like to avoid loading them unnecessarily.
419	*/
420	WCHAR wszPath[MAX_PATH + 64];
421	UINT cwcPath = GetSystemDirectoryW(wszPath, MAX_PATH);
422	if (cwcPath >= MAX_PATH \|\| cwcPath < 2)
423	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "GetSystemDirectoryW failed (%#x / %u)", cwcPath, GetLastError());
424
425	if (wszPath[cwcPath - 1] != '\\' \|\| wszPath[cwcPath - 1] != '/')
426	wszPath[cwcPath++] = '\\';
427	RTUtf16CopyAscii(&wszPath[cwcPath], RT_ELEMENTS(wszPath) - cwcPath, "WinHvPlatform.dll");
428	if (GetFileAttributesW(wszPath) == INVALID_FILE_ATTRIBUTES)
429	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "The native API dll was not found (%ls)", wszPath);
430
431	/*
432	* Check that we're in a VM and that the hypervisor identifies itself as Hyper-V.
433	*/
434	if (!ASMHasCpuId())
435	return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "No CPUID support");
436	if (!ASMIsValidStdRange(ASMCpuId_EAX(0)))
437	return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "No CPUID leaf #1");
438	if (!(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_HVP))
439	return RTErrInfoSet(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Not in a hypervisor partition (HVP=0)");
440
441	uint32_t cMaxHyperLeaf = 0;
442	uint32_t uEbx = 0;
443	uint32_t uEcx = 0;
444	uint32_t uEdx = 0;
445	ASMCpuIdExSlow(0x40000000, 0, 0, 0, &cMaxHyperLeaf, &uEbx, &uEcx, &uEdx);
446	if (!ASMIsValidHypervisorRange(cMaxHyperLeaf))
447	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Invalid hypervisor CPUID range (%#x %#x %#x %#x)",
448	cMaxHyperLeaf, uEbx, uEcx, uEdx);
449	if ( uEbx != UINT32_C(0x7263694d) /* Micr */
450	\|\| uEcx != UINT32_C(0x666f736f) /* osof */
451	\|\| uEdx != UINT32_C(0x76482074) /* t Hv */)
452	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE,
453	"Not Hyper-V CPUID signature: %#x %#x %#x (expected %#x %#x %#x)",
454	uEbx, uEcx, uEdx, UINT32_C(0x7263694d), UINT32_C(0x666f736f), UINT32_C(0x76482074));
455	if (cMaxHyperLeaf < UINT32_C(0x40000005))
456	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "Too narrow hypervisor CPUID range (%#x)", cMaxHyperLeaf);
457
458	/** @todo would be great if we could recognize a root partition from the
459	* CPUID info, but I currently don't dare do that. */
460
461	/*
462	* Now try load the DLLs and resolve the APIs.
463	*/
464	static const char * const s_apszDllNames[2] = { "WinHvPlatform.dll", "vid.dll" };
465	RTLDRMOD ahMods[2] = { NIL_RTLDRMOD, NIL_RTLDRMOD };
466	int rc = VINF_SUCCESS;
467	for (unsigned i = 0; i < RT_ELEMENTS(s_apszDllNames); i++)
468	{
469	int rc2 = RTLdrLoadSystem(s_apszDllNames[i], true /fNoUnload/, &ahMods[i]);
470	if (RT_FAILURE(rc2))
471	{
472	if (!RTErrInfoIsSet(pErrInfo))
473	RTErrInfoSetF(pErrInfo, rc2, "Failed to load API DLL: %s: %Rrc", s_apszDllNames[i], rc2);
474	else
475	RTErrInfoAddF(pErrInfo, rc2, "; %s: %Rrc", s_apszDllNames[i], rc2);
476	ahMods[i] = NIL_RTLDRMOD;
477	rc = VERR_NEM_INIT_FAILED;
478	}
479	}
480	if (RT_SUCCESS(rc))
481	rc = nemR3WinInitVidIntercepts(ahMods[1], pErrInfo);
482	if (RT_SUCCESS(rc))
483	{
484	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
485	{
486	int rc2 = RTLdrGetSymbol(ahMods[g_aImports[i].idxDll], g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
487	if (RT_FAILURE(rc2))
488	{
489	*g_aImports[i].ppfn = NULL;
490
491	LogRel(("NEM: %s: Failed to import %s!%s: %Rrc",
492	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
493	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName, rc2));
494	if (!g_aImports[i].fOptional)
495	{
496	if (RTErrInfoIsSet(pErrInfo))
497	RTErrInfoAddF(pErrInfo, rc2, ", %s!%s",
498	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
499	else
500	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: %s!%s",
501	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
502	Assert(RT_FAILURE(rc));
503	}
504	}
505	}
506	if (RT_SUCCESS(rc))
507	{
508	Assert(!RTErrInfoIsSet(pErrInfo));
509	}
510	}
511
512	for (unsigned i = 0; i < RT_ELEMENTS(ahMods); i++)
513	RTLdrClose(ahMods[i]);
514	return rc;
515	}
516
517
518	/**
519	* Wrapper for different WHvGetCapability signatures.
520	*/
521	DECLINLINE(HRESULT) WHvGetCapabilityWrapper(WHV_CAPABILITY_CODE enmCap, WHV_CAPABILITY *pOutput, uint32_t cbOutput)
522	{
523	return g_pfnWHvGetCapability(enmCap, pOutput, cbOutput, NULL);
524	}
525
526
527	/**
528	* Worker for nemR3NativeInit that gets the hypervisor capabilities.
529	*
530	* @returns VBox status code.
531	* @param pVM The cross context VM structure.
532	* @param pErrInfo Where to always return error info.
533	*/
534	static int nemR3WinInitCheckCapabilities(PVM pVM, PRTERRINFO pErrInfo)
535	{
536	#define NEM_LOG_REL_CAP_EX(a_szField, a_szFmt, a_Value) LogRel(("NEM: %-38s= " a_szFmt "\n", a_szField, a_Value))
537	#define NEM_LOG_REL_CAP_SUB_EX(a_szField, a_szFmt, a_Value) LogRel(("NEM: %36s: " a_szFmt "\n", a_szField, a_Value))
538	#define NEM_LOG_REL_CAP_SUB(a_szField, a_Value) NEM_LOG_REL_CAP_SUB_EX(a_szField, "%d", a_Value)
539
540	/*
541	* Is the hypervisor present with the desired capability?
542	*
543	* In build 17083 this translates into:
544	* - CPUID[0x00000001].HVP is set
545	* - CPUID[0x40000000] == "Microsoft Hv"
546	* - CPUID[0x40000001].eax == "Hv#1"
547	* - CPUID[0x40000003].ebx[12] is set.
548	* - VidGetExoPartitionProperty(INVALID_HANDLE_VALUE, 0x60000, &Ignored) returns
549	* a non-zero value.
550	*/
551	/**
552	* @todo Someone at Microsoft please explain weird API design:
553	* 1. Pointless CapabilityCode duplication int the output;
554	* 2. No output size.
555	*/
556	WHV_CAPABILITY Caps;
557	RT_ZERO(Caps);
558	SetLastError(0);
559	HRESULT hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeHypervisorPresent, &Caps, sizeof(Caps));
560	DWORD rcWin = GetLastError();
561	if (FAILED(hrc))
562	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
563	"WHvGetCapability/WHvCapabilityCodeHypervisorPresent failed: %Rhrc (Last=%#x/%u)",
564	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
565	if (!Caps.HypervisorPresent)
566	{
567	if (!RTPathExists(RTPATH_NT_PASSTHRU_PREFIX "Device\\VidExo"))
568	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE,
569	"WHvCapabilityCodeHypervisorPresent is FALSE! Make sure you have enabled the 'Windows Hypervisor Platform' feature.");
570	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "WHvCapabilityCodeHypervisorPresent is FALSE! (%u)", rcWin);
571	}
572	LogRel(("NEM: WHvCapabilityCodeHypervisorPresent is TRUE, so this might work...\n"));
573
574
575	/*
576	* Check what extended VM exits are supported.
577	*/
578	RT_ZERO(Caps);
579	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeExtendedVmExits, &Caps, sizeof(Caps));
580	if (FAILED(hrc))
581	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
582	"WHvGetCapability/WHvCapabilityCodeExtendedVmExits failed: %Rhrc (Last=%#x/%u)",
583	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
584	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeExtendedVmExits", "%'#018RX64", Caps.ExtendedVmExits.AsUINT64);
585	pVM->nem.s.fExtendedMsrExit = RT_BOOL(Caps.ExtendedVmExits.X64MsrExit);
586	pVM->nem.s.fExtendedCpuIdExit = RT_BOOL(Caps.ExtendedVmExits.X64CpuidExit);
587	pVM->nem.s.fExtendedXcptExit = RT_BOOL(Caps.ExtendedVmExits.ExceptionExit);
588	NEM_LOG_REL_CAP_SUB("fExtendedMsrExit", pVM->nem.s.fExtendedMsrExit);
589	NEM_LOG_REL_CAP_SUB("fExtendedCpuIdExit", pVM->nem.s.fExtendedCpuIdExit);
590	NEM_LOG_REL_CAP_SUB("fExtendedXcptExit", pVM->nem.s.fExtendedXcptExit);
591	if (Caps.ExtendedVmExits.AsUINT64 & ~(uint64_t)7)
592	LogRel(("NEM: Warning! Unknown VM exit definitions: %#RX64\n", Caps.ExtendedVmExits.AsUINT64));
593	/** @todo RECHECK: WHV_EXTENDED_VM_EXITS typedef. */
594
595	/*
596	* Check features in case they end up defining any.
597	*/
598	RT_ZERO(Caps);
599	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeFeatures, &Caps, sizeof(Caps));
600	if (FAILED(hrc))
601	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
602	"WHvGetCapability/WHvCapabilityCodeFeatures failed: %Rhrc (Last=%#x/%u)",
603	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
604	if (Caps.Features.AsUINT64 & ~(uint64_t)0)
605	LogRel(("NEM: Warning! Unknown feature definitions: %#RX64\n", Caps.Features.AsUINT64));
606	/** @todo RECHECK: WHV_CAPABILITY_FEATURES typedef. */
607
608	/*
609	* Check supported exception exit bitmap bits.
610	* We don't currently require this, so we just log failure.
611	*/
612	RT_ZERO(Caps);
613	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeExceptionExitBitmap, &Caps, sizeof(Caps));
614	if (SUCCEEDED(hrc))
615	LogRel(("NEM: Supported exception exit bitmap: %#RX64\n", Caps.ExceptionExitBitmap));
616	else
617	LogRel(("NEM: Warning! WHvGetCapability/WHvCapabilityCodeExceptionExitBitmap failed: %Rhrc (Last=%#x/%u)",
618	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
619
620	/*
621	* Check that the CPU vendor is supported.
622	*/
623	RT_ZERO(Caps);
624	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorVendor, &Caps, sizeof(Caps));
625	if (FAILED(hrc))
626	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
627	"WHvGetCapability/WHvCapabilityCodeProcessorVendor failed: %Rhrc (Last=%#x/%u)",
628	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
629	switch (Caps.ProcessorVendor)
630	{
631	/** @todo RECHECK: WHV_PROCESSOR_VENDOR typedef. */
632	case WHvProcessorVendorIntel:
633	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d - Intel", Caps.ProcessorVendor);
634	pVM->nem.s.enmCpuVendor = CPUMCPUVENDOR_INTEL;
635	break;
636	case WHvProcessorVendorAmd:
637	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d - AMD", Caps.ProcessorVendor);
638	pVM->nem.s.enmCpuVendor = CPUMCPUVENDOR_AMD;
639	break;
640	default:
641	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d", Caps.ProcessorVendor);
642	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unknown processor vendor: %d", Caps.ProcessorVendor);
643	}
644
645	/*
646	* CPU features, guessing these are virtual CPU features?
647	*/
648	RT_ZERO(Caps);
649	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorFeatures, &Caps, sizeof(Caps));
650	if (FAILED(hrc))
651	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
652	"WHvGetCapability/WHvCapabilityCodeProcessorFeatures failed: %Rhrc (Last=%#x/%u)",
653	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
654	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorFeatures", "%'#018RX64", Caps.ProcessorFeatures.AsUINT64);
655	#define NEM_LOG_REL_CPU_FEATURE(a_Field) NEM_LOG_REL_CAP_SUB(#a_Field, Caps.ProcessorFeatures.a_Field)
656	NEM_LOG_REL_CPU_FEATURE(Sse3Support);
657	NEM_LOG_REL_CPU_FEATURE(LahfSahfSupport);
658	NEM_LOG_REL_CPU_FEATURE(Ssse3Support);
659	NEM_LOG_REL_CPU_FEATURE(Sse4_1Support);
660	NEM_LOG_REL_CPU_FEATURE(Sse4_2Support);
661	NEM_LOG_REL_CPU_FEATURE(Sse4aSupport);
662	NEM_LOG_REL_CPU_FEATURE(XopSupport);
663	NEM_LOG_REL_CPU_FEATURE(PopCntSupport);
664	NEM_LOG_REL_CPU_FEATURE(Cmpxchg16bSupport);
665	NEM_LOG_REL_CPU_FEATURE(Altmovcr8Support);
666	NEM_LOG_REL_CPU_FEATURE(LzcntSupport);
667	NEM_LOG_REL_CPU_FEATURE(MisAlignSseSupport);
668	NEM_LOG_REL_CPU_FEATURE(MmxExtSupport);
669	NEM_LOG_REL_CPU_FEATURE(Amd3DNowSupport);
670	NEM_LOG_REL_CPU_FEATURE(ExtendedAmd3DNowSupport);
671	NEM_LOG_REL_CPU_FEATURE(Page1GbSupport);
672	NEM_LOG_REL_CPU_FEATURE(AesSupport);
673	NEM_LOG_REL_CPU_FEATURE(PclmulqdqSupport);
674	NEM_LOG_REL_CPU_FEATURE(PcidSupport);
675	NEM_LOG_REL_CPU_FEATURE(Fma4Support);
676	NEM_LOG_REL_CPU_FEATURE(F16CSupport);
677	NEM_LOG_REL_CPU_FEATURE(RdRandSupport);
678	NEM_LOG_REL_CPU_FEATURE(RdWrFsGsSupport);
679	NEM_LOG_REL_CPU_FEATURE(SmepSupport);
680	NEM_LOG_REL_CPU_FEATURE(EnhancedFastStringSupport);
681	NEM_LOG_REL_CPU_FEATURE(Bmi1Support);
682	NEM_LOG_REL_CPU_FEATURE(Bmi2Support);
683	/* two reserved bits here, see below */
684	NEM_LOG_REL_CPU_FEATURE(MovbeSupport);
685	NEM_LOG_REL_CPU_FEATURE(Npiep1Support);
686	NEM_LOG_REL_CPU_FEATURE(DepX87FPUSaveSupport);
687	NEM_LOG_REL_CPU_FEATURE(RdSeedSupport);
688	NEM_LOG_REL_CPU_FEATURE(AdxSupport);
689	NEM_LOG_REL_CPU_FEATURE(IntelPrefetchSupport);
690	NEM_LOG_REL_CPU_FEATURE(SmapSupport);
691	NEM_LOG_REL_CPU_FEATURE(HleSupport);
692	NEM_LOG_REL_CPU_FEATURE(RtmSupport);
693	NEM_LOG_REL_CPU_FEATURE(RdtscpSupport);
694	NEM_LOG_REL_CPU_FEATURE(ClflushoptSupport);
695	NEM_LOG_REL_CPU_FEATURE(ClwbSupport);
696	NEM_LOG_REL_CPU_FEATURE(ShaSupport);
697	NEM_LOG_REL_CPU_FEATURE(X87PointersSavedSupport);
698	#undef NEM_LOG_REL_CPU_FEATURE
699	if (Caps.ProcessorFeatures.AsUINT64 & (~(RT_BIT_64(43) - 1) \| RT_BIT_64(27) \| RT_BIT_64(28)))
700	LogRel(("NEM: Warning! Unknown CPU features: %#RX64\n", Caps.ProcessorFeatures.AsUINT64));
701	pVM->nem.s.uCpuFeatures.u64 = Caps.ProcessorFeatures.AsUINT64;
702	/** @todo RECHECK: WHV_PROCESSOR_FEATURES typedef. */
703
704	/*
705	* The cache line flush size.
706	*/
707	RT_ZERO(Caps);
708	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorClFlushSize, &Caps, sizeof(Caps));
709	if (FAILED(hrc))
710	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
711	"WHvGetCapability/WHvCapabilityCodeProcessorClFlushSize failed: %Rhrc (Last=%#x/%u)",
712	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
713	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorClFlushSize", "2^%u", Caps.ProcessorClFlushSize);
714	if (Caps.ProcessorClFlushSize < 8 && Caps.ProcessorClFlushSize > 9)
715	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unsupported cache line flush size: %u", Caps.ProcessorClFlushSize);
716	pVM->nem.s.cCacheLineFlushShift = Caps.ProcessorClFlushSize;
717
718	/*
719	* See if they've added more properties that we're not aware of.
720	*/
721	/** @todo RECHECK: WHV_CAPABILITY_CODE typedef. */
722	if (!IsDebuggerPresent()) /* Too noisy when in debugger, so skip. */
723	{
724	static const struct
725	{
726	uint32_t iMin, iMax; } s_aUnknowns[] =
727	{
728	{ 0x0004, 0x000f },
729	{ 0x1003, 0x100f },
730	{ 0x2000, 0x200f },
731	{ 0x3000, 0x300f },
732	{ 0x4000, 0x400f },
733	};
734	for (uint32_t j = 0; j < RT_ELEMENTS(s_aUnknowns); j++)
735	for (uint32_t i = s_aUnknowns[j].iMin; i <= s_aUnknowns[j].iMax; i++)
736	{
737	RT_ZERO(Caps);
738	hrc = WHvGetCapabilityWrapper((WHV_CAPABILITY_CODE)i, &Caps, sizeof(Caps));
739	if (SUCCEEDED(hrc))
740	LogRel(("NEM: Warning! Unknown capability %#x returning: %.*Rhxs\n", i, sizeof(Caps), &Caps));
741	}
742	}
743
744	/*
745	* For proper operation, we require CPUID exits.
746	*/
747	if (!pVM->nem.s.fExtendedCpuIdExit)
748	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Missing required extended CPUID exit support");
749	if (!pVM->nem.s.fExtendedMsrExit)
750	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Missing required extended MSR exit support");
751	if (!pVM->nem.s.fExtendedXcptExit)
752	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Missing required extended exception exit support");
753
754	#undef NEM_LOG_REL_CAP_EX
755	#undef NEM_LOG_REL_CAP_SUB_EX
756	#undef NEM_LOG_REL_CAP_SUB
757	return VINF_SUCCESS;
758	}
759
760
761	/**
762	* Used to fill in g_IoCtlGetHvPartitionId.
763	*/
764	static NTSTATUS WINAPI
765	nemR3WinIoctlDetector_GetHvPartitionId(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
766	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
767	PVOID pvOutput, ULONG cbOutput)
768	{
769	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
770	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
771	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
772	AssertLogRelMsgReturn(cbInput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
773	RT_NOREF(pvInput);
774
775	AssertLogRelMsgReturn(RT_VALID_PTR(pvOutput), ("pvOutput=%p\n", pvOutput), STATUS_INVALID_PARAMETER_9);
776	AssertLogRelMsgReturn(cbOutput == sizeof(HV_PARTITION_ID), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
777	(HV_PARTITION_ID )pvOutput = NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID;
778
779	g_IoCtlGetHvPartitionId.cbInput = cbInput;
780	g_IoCtlGetHvPartitionId.cbOutput = cbOutput;
781	g_IoCtlGetHvPartitionId.uFunction = uFunction;
782
783	return STATUS_SUCCESS;
784	}
785
786
787	/**
788	* Used to fill in g_IoCtlStartVirtualProcessor.
789	*/
790	static NTSTATUS WINAPI
791	nemR3WinIoctlDetector_StartVirtualProcessor(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
792	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
793	PVOID pvOutput, ULONG cbOutput)
794	{
795	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
796	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
797	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
798	AssertLogRelMsgReturn(cbInput == sizeof(HV_VP_INDEX), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
799	AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
800	AssertLogRelMsgReturn((HV_VP_INDEX )pvInput == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
801	("piCpu=%u\n", (HV_VP_INDEX *)pvInput), STATUS_INVALID_PARAMETER_9);
802	AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
803	RT_NOREF(pvOutput);
804
805	g_IoCtlStartVirtualProcessor.cbInput = cbInput;
806	g_IoCtlStartVirtualProcessor.cbOutput = cbOutput;
807	g_IoCtlStartVirtualProcessor.uFunction = uFunction;
808
809	return STATUS_SUCCESS;
810	}
811
812
813	/**
814	* Used to fill in g_IoCtlStartVirtualProcessor.
815	*/
816	static NTSTATUS WINAPI
817	nemR3WinIoctlDetector_StopVirtualProcessor(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
818	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
819	PVOID pvOutput, ULONG cbOutput)
820	{
821	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
822	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
823	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
824	AssertLogRelMsgReturn(cbInput == sizeof(HV_VP_INDEX), ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_8);
825	AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
826	AssertLogRelMsgReturn((HV_VP_INDEX )pvInput == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
827	("piCpu=%u\n", (HV_VP_INDEX *)pvInput), STATUS_INVALID_PARAMETER_9);
828	AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
829	RT_NOREF(pvOutput);
830
831	g_IoCtlStopVirtualProcessor.cbInput = cbInput;
832	g_IoCtlStopVirtualProcessor.cbOutput = cbOutput;
833	g_IoCtlStopVirtualProcessor.uFunction = uFunction;
834
835	return STATUS_SUCCESS;
836	}
837
838
839	/**
840	* Used to fill in g_IoCtlMessageSlotHandleAndGetNext
841	*/
842	static NTSTATUS WINAPI
843	nemR3WinIoctlDetector_MessageSlotHandleAndGetNext(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
844	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
845	PVOID pvOutput, ULONG cbOutput)
846	{
847	AssertLogRelMsgReturn(hFile == NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, ("hFile=%p\n", hFile), STATUS_INVALID_PARAMETER_1);
848	RT_NOREF(hEvt); RT_NOREF(pfnApcCallback); RT_NOREF(pvApcCtx);
849	AssertLogRelMsgReturn(RT_VALID_PTR(pIos), ("pIos=%p\n", pIos), STATUS_INVALID_PARAMETER_5);
850
851	AssertLogRelMsgReturn(cbInput == sizeof(VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT), ("cbInput=%#x\n", cbInput),
852	STATUS_INVALID_PARAMETER_8);
853	AssertLogRelMsgReturn(RT_VALID_PTR(pvInput), ("pvInput=%p\n", pvInput), STATUS_INVALID_PARAMETER_9);
854	PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT pVidIn = (PCVID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT)pvInput;
855	AssertLogRelMsgReturn( pVidIn->iCpu == NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX
856	&& pVidIn->fFlags == VID_MSHAGN_F_HANDLE_MESSAGE
857	&& pVidIn->cMillies == NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT,
858	("iCpu=%u fFlags=%#x cMillies=%#x\n", pVidIn->iCpu, pVidIn->fFlags, pVidIn->cMillies),
859	STATUS_INVALID_PARAMETER_9);
860	AssertLogRelMsgReturn(cbOutput == 0, ("cbInput=%#x\n", cbInput), STATUS_INVALID_PARAMETER_10);
861	RT_NOREF(pvOutput);
862
863	g_IoCtlMessageSlotHandleAndGetNext.cbInput = cbInput;
864	g_IoCtlMessageSlotHandleAndGetNext.cbOutput = cbOutput;
865	g_IoCtlMessageSlotHandleAndGetNext.uFunction = uFunction;
866
867	return STATUS_SUCCESS;
868	}
869
870
871	#ifdef LOG_ENABLED
872	/**
873	* Used to fill in what g_pIoCtlDetectForLogging points to.
874	*/
875	static NTSTATUS WINAPI nemR3WinIoctlDetector_ForLogging(HANDLE hFile, HANDLE hEvt, PIO_APC_ROUTINE pfnApcCallback, PVOID pvApcCtx,
876	PIO_STATUS_BLOCK pIos, ULONG uFunction, PVOID pvInput, ULONG cbInput,
877	PVOID pvOutput, ULONG cbOutput)
878	{
879	RT_NOREF(hFile, hEvt, pfnApcCallback, pvApcCtx, pIos, pvInput, pvOutput);
880
881	g_pIoCtlDetectForLogging->cbInput = cbInput;
882	g_pIoCtlDetectForLogging->cbOutput = cbOutput;
883	g_pIoCtlDetectForLogging->uFunction = uFunction;
884
885	return STATUS_SUCCESS;
886	}
887	#endif
888
889
890	/**
891	* Worker for nemR3NativeInit that detect I/O control function numbers for VID.
892	*
893	* We use the function numbers directly in ring-0 and to name functions when
894	* logging NtDeviceIoControlFile calls.
895	*
896	* @note We could alternatively do this by disassembling the respective
897	* functions, but hooking NtDeviceIoControlFile and making fake calls
898	* more easily provides the desired information.
899	*
900	* @returns VBox status code.
901	* @param pVM The cross context VM structure. Will set I/O
902	* control info members.
903	* @param pErrInfo Where to always return error info.
904	*/
905	static int nemR3WinInitDiscoverIoControlProperties(PVM pVM, PRTERRINFO pErrInfo)
906	{
907	/*
908	* Probe the I/O control information for select VID APIs so we can use
909	* them directly from ring-0 and better log them.
910	*
911	*/
912	decltype(NtDeviceIoControlFile) * const pfnOrg = *g_ppfnVidNtDeviceIoControlFile;
913
914	/* VidGetHvPartitionId */
915	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_GetHvPartitionId;
916	HV_PARTITION_ID idHvPartition = HV_PARTITION_ID_INVALID;
917	BOOL fRet = g_pfnVidGetHvPartitionId(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, &idHvPartition);
918	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
919	AssertReturn(fRet && idHvPartition == NEM_WIN_IOCTL_DETECTOR_FAKE_PARTITION_ID && g_IoCtlGetHvPartitionId.uFunction != 0,
920	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
921	"Problem figuring out VidGetHvPartitionId: fRet=%u idHvPartition=%#x dwErr=%u",
922	fRet, idHvPartition, GetLastError()) );
923	LogRel(("NEM: VidGetHvPartitionId -> fun:%#x in:%#x out:%#x\n",
924	g_IoCtlGetHvPartitionId.uFunction, g_IoCtlGetHvPartitionId.cbInput, g_IoCtlGetHvPartitionId.cbOutput));
925
926	/* VidStartVirtualProcessor */
927	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_StartVirtualProcessor;
928	fRet = g_pfnVidStartVirtualProcessor(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
929	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
930	AssertReturn(fRet && g_IoCtlStartVirtualProcessor.uFunction != 0,
931	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
932	"Problem figuring out VidStartVirtualProcessor: fRet=%u dwErr=%u",
933	fRet, GetLastError()) );
934	LogRel(("NEM: VidStartVirtualProcessor -> fun:%#x in:%#x out:%#x\n", g_IoCtlStartVirtualProcessor.uFunction,
935	g_IoCtlStartVirtualProcessor.cbInput, g_IoCtlStartVirtualProcessor.cbOutput));
936
937	/* VidStopVirtualProcessor */
938	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_StopVirtualProcessor;
939	fRet = g_pfnVidStopVirtualProcessor(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
940	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
941	AssertReturn(fRet && g_IoCtlStopVirtualProcessor.uFunction != 0,
942	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
943	"Problem figuring out VidStopVirtualProcessor: fRet=%u dwErr=%u",
944	fRet, GetLastError()) );
945	LogRel(("NEM: VidStopVirtualProcessor -> fun:%#x in:%#x out:%#x\n", g_IoCtlStopVirtualProcessor.uFunction,
946	g_IoCtlStopVirtualProcessor.cbInput, g_IoCtlStopVirtualProcessor.cbOutput));
947
948	/* VidMessageSlotHandleAndGetNext */
949	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_MessageSlotHandleAndGetNext;
950	fRet = g_pfnVidMessageSlotHandleAndGetNext(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE,
951	NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX, VID_MSHAGN_F_HANDLE_MESSAGE,
952	NEM_WIN_IOCTL_DETECTOR_FAKE_TIMEOUT);
953	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
954	AssertReturn(fRet && g_IoCtlMessageSlotHandleAndGetNext.uFunction != 0,
955	RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
956	"Problem figuring out VidMessageSlotHandleAndGetNext: fRet=%u dwErr=%u",
957	fRet, GetLastError()) );
958	LogRel(("NEM: VidMessageSlotHandleAndGetNext -> fun:%#x in:%#x out:%#x\n",
959	g_IoCtlMessageSlotHandleAndGetNext.uFunction, g_IoCtlMessageSlotHandleAndGetNext.cbInput,
960	g_IoCtlMessageSlotHandleAndGetNext.cbOutput));
961
962	#ifdef LOG_ENABLED
963	/* The following are only for logging: */
964	union
965	{
966	VID_MAPPED_MESSAGE_SLOT MapSlot;
967	HV_REGISTER_NAME Name;
968	HV_REGISTER_VALUE Value;
969	} uBuf;
970
971	/* VidMessageSlotMap */
972	g_pIoCtlDetectForLogging = &g_IoCtlMessageSlotMap;
973	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
974	fRet = g_pfnVidMessageSlotMap(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, &uBuf.MapSlot, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX);
975	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
976	Assert(fRet);
977	LogRel(("NEM: VidMessageSlotMap -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
978	g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
979
980	/* VidGetVirtualProcessorState */
981	uBuf.Name = HvRegisterExplicitSuspend;
982	g_pIoCtlDetectForLogging = &g_IoCtlGetVirtualProcessorState;
983	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
984	fRet = g_pfnVidGetVirtualProcessorState(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
985	&uBuf.Name, 1, &uBuf.Value);
986	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
987	Assert(fRet);
988	LogRel(("NEM: VidGetVirtualProcessorState -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
989	g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
990
991	/* VidSetVirtualProcessorState */
992	uBuf.Name = HvRegisterExplicitSuspend;
993	g_pIoCtlDetectForLogging = &g_IoCtlSetVirtualProcessorState;
994	*g_ppfnVidNtDeviceIoControlFile = nemR3WinIoctlDetector_ForLogging;
995	fRet = g_pfnVidSetVirtualProcessorState(NEM_WIN_IOCTL_DETECTOR_FAKE_HANDLE, NEM_WIN_IOCTL_DETECTOR_FAKE_VP_INDEX,
996	&uBuf.Name, 1, &uBuf.Value);
997	*g_ppfnVidNtDeviceIoControlFile = pfnOrg;
998	Assert(fRet);
999	LogRel(("NEM: VidSetVirtualProcessorState -> fun:%#x in:%#x out:%#x\n", g_pIoCtlDetectForLogging->uFunction,
1000	g_pIoCtlDetectForLogging->cbInput, g_pIoCtlDetectForLogging->cbOutput));
1001
1002	g_pIoCtlDetectForLogging = NULL;
1003	#endif
1004
1005	/* Done. */
1006	pVM->nem.s.IoCtlGetHvPartitionId = g_IoCtlGetHvPartitionId;
1007	pVM->nem.s.IoCtlStartVirtualProcessor = g_IoCtlStartVirtualProcessor;
1008	pVM->nem.s.IoCtlStopVirtualProcessor = g_IoCtlStopVirtualProcessor;
1009	pVM->nem.s.IoCtlMessageSlotHandleAndGetNext = g_IoCtlMessageSlotHandleAndGetNext;
1010	return VINF_SUCCESS;
1011	}
1012
1013
1014	/**
1015	* Creates and sets up a Hyper-V (exo) partition.
1016	*
1017	* @returns VBox status code.
1018	* @param pVM The cross context VM structure.
1019	* @param pErrInfo Where to always return error info.
1020	*/
1021	static int nemR3WinInitCreatePartition(PVM pVM, PRTERRINFO pErrInfo)
1022	{
1023	AssertReturn(!pVM->nem.s.hPartition, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
1024	AssertReturn(!pVM->nem.s.hPartitionDevice, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
1025
1026	/*
1027	* Create the partition.
1028	*/
1029	WHV_PARTITION_HANDLE hPartition;
1030	HRESULT hrc = WHvCreatePartition(&hPartition);
1031	if (FAILED(hrc))
1032	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "WHvCreatePartition failed with %Rhrc (Last=%#x/%u)",
1033	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1034
1035	int rc;
1036
1037	/*
1038	* Set partition properties, most importantly the CPU count.
1039	*/
1040	/**
1041	* @todo Someone at Microsoft please explain another weird API:
1042	* - Why this API doesn't take the WHV_PARTITION_PROPERTY_CODE value as an
1043	* argument rather than as part of the struct. That is so weird if you've
1044	* used any other NT or windows API, including WHvGetCapability().
1045	* - Why use PVOID when WHV_PARTITION_PROPERTY is what's expected. We
1046	* technically only need 9 bytes for setting/getting
1047	* WHVPartitionPropertyCodeProcessorClFlushSize, but the API insists on 16. */
1048	WHV_PARTITION_PROPERTY Property;
1049	RT_ZERO(Property);
1050	Property.ProcessorCount = pVM->cCpus;
1051	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorCount, &Property, sizeof(Property));
1052	if (SUCCEEDED(hrc))
1053	{
1054	RT_ZERO(Property);
1055	Property.ExtendedVmExits.X64CpuidExit = pVM->nem.s.fExtendedCpuIdExit; /** @todo Register fixed results and restrict cpuid exits */
1056	Property.ExtendedVmExits.X64MsrExit = pVM->nem.s.fExtendedMsrExit;
1057	Property.ExtendedVmExits.ExceptionExit = pVM->nem.s.fExtendedXcptExit;
1058	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeExtendedVmExits, &Property, sizeof(Property));
1059	if (SUCCEEDED(hrc))
1060	{
1061	/*
1062	* We'll continue setup in nemR3NativeInitAfterCPUM.
1063	*/
1064	pVM->nem.s.fCreatedEmts = false;
1065	pVM->nem.s.hPartition = hPartition;
1066	LogRel(("NEM: Created partition %p.\n", hPartition));
1067	return VINF_SUCCESS;
1068	}
1069
1070	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
1071	"Failed setting WHvPartitionPropertyCodeExtendedVmExits to %'#RX64: %Rhrc",
1072	Property.ExtendedVmExits.AsUINT64, hrc);
1073	}
1074	else
1075	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
1076	"Failed setting WHvPartitionPropertyCodeProcessorCount to %u: %Rhrc (Last=%#x/%u)",
1077	pVM->cCpus, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1078	WHvDeletePartition(hPartition);
1079
1080	Assert(!pVM->nem.s.hPartitionDevice);
1081	Assert(!pVM->nem.s.hPartition);
1082	return rc;
1083	}
1084
1085
1086	/**
1087	* Makes sure APIC and firmware will not allow X2APIC mode.
1088	*
1089	* This is rather ugly.
1090	*
1091	* @returns VBox status code
1092	* @param pVM The cross context VM structure.
1093	*/
1094	static int nemR3WinDisableX2Apic(PVM pVM)
1095	{
1096	/*
1097	* First make sure the 'Mode' config value of the APIC isn't set to X2APIC.
1098	* This defaults to APIC, so no need to change unless it's X2APIC.
1099	*/
1100	PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/Devices/apic/0/Config");
1101	if (pCfg)
1102	{
1103	uint8_t bMode = 0;
1104	int rc = CFGMR3QueryU8(pCfg, "Mode", &bMode);
1105	AssertLogRelMsgReturn(RT_SUCCESS(rc) \|\| rc == VERR_CFGM_VALUE_NOT_FOUND, ("%Rrc\n", rc), rc);
1106	if (RT_SUCCESS(rc) && bMode == PDMAPICMODE_X2APIC)
1107	{
1108	LogRel(("NEM: Adjusting APIC configuration from X2APIC to APIC max mode. X2APIC is not supported by the WinHvPlatform API!\n"));
1109	LogRel(("NEM: Disable Hyper-V if you need X2APIC for your guests!\n"));
1110	rc = CFGMR3RemoveValue(pCfg, "Mode");
1111	rc = CFGMR3InsertInteger(pCfg, "Mode", PDMAPICMODE_APIC);
1112	AssertLogRelRCReturn(rc, rc);
1113	}
1114	}
1115
1116	/*
1117	* Now the firmwares.
1118	* These also defaults to APIC and only needs adjusting if configured to X2APIC (2).
1119	*/
1120	static const char * const s_apszFirmwareConfigs[] =
1121	{
1122	"/Devices/efi/0/Config",
1123	"/Devices/pcbios/0/Config",
1124	};
1125	for (unsigned i = 0; i < RT_ELEMENTS(s_apszFirmwareConfigs); i++)
1126	{
1127	pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/Devices/APIC/0/Config");
1128	if (pCfg)
1129	{
1130	uint8_t bMode = 0;
1131	int rc = CFGMR3QueryU8(pCfg, "APIC", &bMode);
1132	AssertLogRelMsgReturn(RT_SUCCESS(rc) \|\| rc == VERR_CFGM_VALUE_NOT_FOUND, ("%Rrc\n", rc), rc);
1133	if (RT_SUCCESS(rc) && bMode == 2)
1134	{
1135	LogRel(("NEM: Adjusting %s/Mode from 2 (X2APIC) to 1 (APIC).\n", s_apszFirmwareConfigs[i]));
1136	rc = CFGMR3RemoveValue(pCfg, "APIC");
1137	rc = CFGMR3InsertInteger(pCfg, "APIC", 1);
1138	AssertLogRelRCReturn(rc, rc);
1139	}
1140	}
1141	}
1142
1143	return VINF_SUCCESS;
1144	}
1145
1146
1147	/**
1148	* Try initialize the native API.
1149	*
1150	* This may only do part of the job, more can be done in
1151	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
1152	*
1153	* @returns VBox status code.
1154	* @param pVM The cross context VM structure.
1155	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
1156	* the latter we'll fail if we cannot initialize.
1157	* @param fForced Whether the HMForced flag is set and we should
1158	* fail if we cannot initialize.
1159	*/
1160	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
1161	{
1162	g_uBuildNo = RTSystemGetNtBuildNo();
1163
1164	/*
1165	* Some state init.
1166	*/
1167	for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
1168	{
1169	PNEMCPU pNemCpu = &pVM->aCpus[iCpu].nem.s;
1170	pNemCpu->uPendingApicBase = UINT64_MAX;
1171	}
1172
1173	/*
1174	* Error state.
1175	* The error message will be non-empty on failure and 'rc' will be set too.
1176	*/
1177	RTERRINFOSTATIC ErrInfo;
1178	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
1179	int rc = nemR3WinInitProbeAndLoad(fForced, pErrInfo);
1180	if (RT_SUCCESS(rc))
1181	{
1182	/*
1183	* Check the capabilties of the hypervisor, starting with whether it's present.
1184	*/
1185	rc = nemR3WinInitCheckCapabilities(pVM, pErrInfo);
1186	if (RT_SUCCESS(rc))
1187	{
1188	/*
1189	* Discover the VID I/O control function numbers we need.
1190	*/
1191	rc = nemR3WinInitDiscoverIoControlProperties(pVM, pErrInfo);
1192	if (RT_SUCCESS(rc))
1193	{
1194	/*
1195	* Check out our ring-0 capabilities.
1196	*/
1197	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_NEM_INIT_VM, 0, NULL);
1198	if (RT_SUCCESS(rc))
1199	{
1200	/*
1201	* Create and initialize a partition.
1202	*/
1203	rc = nemR3WinInitCreatePartition(pVM, pErrInfo);
1204	if (RT_SUCCESS(rc))
1205	{
1206	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
1207	Log(("NEM: Marked active!\n"));
1208	nemR3WinDisableX2Apic(pVM);
1209
1210	/* Register release statistics */
1211	for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
1212	{
1213	PNEMCPU pNemCpu = &pVM->aCpus[iCpu].nem.s;
1214	STAMR3RegisterF(pVM, &pNemCpu->StatExitPortIo, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of port I/O exits", "/NEM/CPU%u/ExitPortIo", iCpu);
1215	STAMR3RegisterF(pVM, &pNemCpu->StatExitMemUnmapped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unmapped memory exits", "/NEM/CPU%u/ExitMemUnmapped", iCpu);
1216	STAMR3RegisterF(pVM, &pNemCpu->StatExitMemIntercept, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of intercepted memory exits", "/NEM/CPU%u/ExitMemIntercept", iCpu);
1217	STAMR3RegisterF(pVM, &pNemCpu->StatExitHalt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of HLT exits", "/NEM/CPU%u/ExitHalt", iCpu);
1218	STAMR3RegisterF(pVM, &pNemCpu->StatExitInterruptWindow, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of HLT exits", "/NEM/CPU%u/ExitInterruptWindow", iCpu);
1219	STAMR3RegisterF(pVM, &pNemCpu->StatExitCpuId, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of CPUID exits", "/NEM/CPU%u/ExitCpuId", iCpu);
1220	STAMR3RegisterF(pVM, &pNemCpu->StatExitMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of MSR access exits", "/NEM/CPU%u/ExitMsr", iCpu);
1221	STAMR3RegisterF(pVM, &pNemCpu->StatExitException, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of exception exits", "/NEM/CPU%u/ExitException", iCpu);
1222	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionBp, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #BP exits", "/NEM/CPU%u/ExitExceptionBp", iCpu);
1223	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionDb, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #DB exits", "/NEM/CPU%u/ExitExceptionDb", iCpu);
1224	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUd, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #UD exits", "/NEM/CPU%u/ExitExceptionUd", iCpu);
1225	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUdHandled, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of handled #UD exits", "/NEM/CPU%u/ExitExceptionUdHandled", iCpu);
1226	STAMR3RegisterF(pVM, &pNemCpu->StatExitUnrecoverable, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unrecoverable exits", "/NEM/CPU%u/ExitUnrecoverable", iCpu);
1227	STAMR3RegisterF(pVM, &pNemCpu->StatGetMsgTimeout, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of get message timeouts/alerts", "/NEM/CPU%u/GetMsgTimeout", iCpu);
1228	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuSuccess, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of successful CPU stops", "/NEM/CPU%u/StopCpuSuccess", iCpu);
1229	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPending, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pending CPU stops", "/NEM/CPU%u/StopCpuPending", iCpu);
1230	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPendingOdd, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of odd pending CPU stops (see code)", "/NEM/CPU%u/StopCpuPendingOdd", iCpu);
1231	STAMR3RegisterF(pVM, &pNemCpu->StatCancelChangedState, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel changed state", "/NEM/CPU%u/CancelChangedState", iCpu);
1232	STAMR3RegisterF(pVM, &pNemCpu->StatCancelAlertedThread, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel alerted EMT", "/NEM/CPU%u/CancelAlertedEMT", iCpu);
1233	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPre, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pre execution FF breaks", "/NEM/CPU%u/BreakOnFFPre", iCpu);
1234	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPost, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of post execution FF breaks", "/NEM/CPU%u/BreakOnFFPost", iCpu);
1235	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnCancel, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel execution breaks", "/NEM/CPU%u/BreakOnCancel", iCpu);
1236	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnStatus, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of status code breaks", "/NEM/CPU%u/BreakOnStatus", iCpu);
1237	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnDemand, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of on-demand state imports", "/NEM/CPU%u/ImportOnDemand", iCpu);
1238	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of state imports on loop return", "/NEM/CPU%u/ImportOnReturn", iCpu);
1239	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturnSkipped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of skipped state imports on loop return", "/NEM/CPU%u/ImportOnReturnSkipped", iCpu);
1240	STAMR3RegisterF(pVM, &pNemCpu->StatQueryCpuTick, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of TSC queries", "/NEM/CPU%u/QueryCpuTick", iCpu);
1241	}
1242
1243	PUVM pUVM = pVM->pUVM;
1244	STAMR3RegisterRefresh(pUVM, &pVM->nem.s.R0Stats.cPagesAvailable, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,
1245	STAMUNIT_PAGES, STAM_REFRESH_GRP_NEM, "Free pages available to the hypervisor",
1246	"/NEM/R0Stats/cPagesAvailable");
1247	STAMR3RegisterRefresh(pUVM, &pVM->nem.s.R0Stats.cPagesInUse, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,
1248	STAMUNIT_PAGES, STAM_REFRESH_GRP_NEM, "Pages in use by hypervisor",
1249	"/NEM/R0Stats/cPagesInUse");
1250	}
1251	}
1252	}
1253	}
1254	}
1255
1256	/*
1257	* We only fail if in forced mode, otherwise just log the complaint and return.
1258	*/
1259	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
1260	if ( (fForced \|\| !fFallback)
1261	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
1262	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
1263
1264	if (RTErrInfoIsSet(pErrInfo))
1265	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
1266	return VINF_SUCCESS;
1267	}
1268
1269
1270	/**
1271	* This is called after CPUMR3Init is done.
1272	*
1273	* @returns VBox status code.
1274	* @param pVM The VM handle..
1275	*/
1276	int nemR3NativeInitAfterCPUM(PVM pVM)
1277	{
1278	/*
1279	* Validate sanity.
1280	*/
1281	WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
1282	AssertReturn(hPartition != NULL, VERR_WRONG_ORDER);
1283	AssertReturn(!pVM->nem.s.hPartitionDevice, VERR_WRONG_ORDER);
1284	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
1285	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
1286
1287	/*
1288	* Continue setting up the partition now that we've got most of the CPUID feature stuff.
1289	*/
1290	WHV_PARTITION_PROPERTY Property;
1291	HRESULT hrc;
1292
1293	#if 0
1294	/* Not sure if we really need to set the vendor.
1295	Update: Apparently we don't. WHvPartitionPropertyCodeProcessorVendor was removed in 17110. */
1296	RT_ZERO(Property);
1297	Property.ProcessorVendor = pVM->nem.s.enmCpuVendor == CPUMCPUVENDOR_AMD ? WHvProcessorVendorAmd
1298	: WHvProcessorVendorIntel;
1299	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorVendor, &Property, sizeof(Property));
1300	if (FAILED(hrc))
1301	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1302	"Failed to set WHvPartitionPropertyCodeProcessorVendor to %u: %Rhrc (Last=%#x/%u)",
1303	Property.ProcessorVendor, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1304	#endif
1305
1306	/* Not sure if we really need to set the cache line flush size. */
1307	RT_ZERO(Property);
1308	Property.ProcessorClFlushSize = pVM->nem.s.cCacheLineFlushShift;
1309	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorClFlushSize, &Property, sizeof(Property));
1310	if (FAILED(hrc))
1311	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1312	"Failed to set WHvPartitionPropertyCodeProcessorClFlushSize to %u: %Rhrc (Last=%#x/%u)",
1313	pVM->nem.s.cCacheLineFlushShift, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1314
1315	/* Intercept #DB, #BP and #UD exceptions. */
1316	RT_ZERO(Property);
1317	Property.ExceptionExitBitmap = RT_BIT_64(WHvX64ExceptionTypeDebugTrapOrFault)
1318	\| RT_BIT_64(WHvX64ExceptionTypeBreakpointTrap)
1319	\| RT_BIT_64(WHvX64ExceptionTypeInvalidOpcodeFault);
1320	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeExceptionExitBitmap, &Property, sizeof(Property));
1321	if (FAILED(hrc))
1322	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1323	"Failed to set WHvPartitionPropertyCodeExceptionExitBitmap to %#RX64: %Rhrc (Last=%#x/%u)",
1324	Property.ExceptionExitBitmap, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1325
1326
1327	/*
1328	* Sync CPU features with CPUM.
1329	*/
1330	/** @todo sync CPU features with CPUM. */
1331
1332	/* Set the partition property. */
1333	RT_ZERO(Property);
1334	Property.ProcessorFeatures.AsUINT64 = pVM->nem.s.uCpuFeatures.u64;
1335	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorFeatures, &Property, sizeof(Property));
1336	if (FAILED(hrc))
1337	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1338	"Failed to set WHvPartitionPropertyCodeProcessorFeatures to %'#RX64: %Rhrc (Last=%#x/%u)",
1339	pVM->nem.s.uCpuFeatures.u64, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1340
1341	/*
1342	* Set up the partition and create EMTs.
1343	*
1344	* Seems like this is where the partition is actually instantiated and we get
1345	* a handle to it.
1346	*/
1347	hrc = WHvSetupPartition(hPartition);
1348	if (FAILED(hrc))
1349	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1350	"Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)",
1351	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
1352
1353	/* Get the handle. */
1354	HANDLE hPartitionDevice;
1355	__try
1356	{
1357	hPartitionDevice = ((HANDLE *)hPartition)[1];
1358	}
1359	__except(EXCEPTION_EXECUTE_HANDLER)
1360	{
1361	hrc = GetExceptionCode();
1362	hPartitionDevice = NULL;
1363	}
1364	if ( hPartitionDevice == NULL
1365	\|\| hPartitionDevice == (HANDLE)(intptr_t)-1)
1366	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1367	"Failed to get device handle for partition %p: %Rhrc", hPartition, hrc);
1368
1369	HV_PARTITION_ID idHvPartition = HV_PARTITION_ID_INVALID;
1370	if (!g_pfnVidGetHvPartitionId(hPartitionDevice, &idHvPartition))
1371	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1372	"Failed to get device handle and/or partition ID for %p (hPartitionDevice=%p, Last=%#x/%u)",
1373	hPartition, hPartitionDevice, RTNtLastStatusValue(), RTNtLastErrorValue());
1374	pVM->nem.s.hPartitionDevice = hPartitionDevice;
1375	pVM->nem.s.idHvPartition = idHvPartition;
1376
1377	/*
1378	* Setup the EMTs.
1379	*/
1380	VMCPUID iCpu;
1381	for (iCpu = 0; iCpu < pVM->cCpus; iCpu++)
1382	{
1383	PVMCPU pVCpu = &pVM->aCpus[iCpu];
1384
1385	pVCpu->nem.s.hNativeThreadHandle = (RTR3PTR)RTThreadGetNativeHandle(VMR3GetThreadHandle(pVCpu->pUVCpu));
1386	Assert((HANDLE)pVCpu->nem.s.hNativeThreadHandle != INVALID_HANDLE_VALUE);
1387
1388	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
1389	VID_MAPPED_MESSAGE_SLOT MappedMsgSlot = { NULL, UINT32_MAX, UINT32_MAX };
1390	if (g_pfnVidMessageSlotMap(hPartitionDevice, &MappedMsgSlot, iCpu))
1391	{
1392	AssertLogRelMsg(MappedMsgSlot.iCpu == iCpu && MappedMsgSlot.uParentAdvisory == UINT32_MAX,
1393	("%#x %#x (iCpu=%#x)\n", MappedMsgSlot.iCpu, MappedMsgSlot.uParentAdvisory, iCpu));
1394	pVCpu->nem.s.pvMsgSlotMapping = MappedMsgSlot.pMsgBlock;
1395	}
1396	else
1397	{
1398	NTSTATUS const rcNtLast = RTNtLastStatusValue();
1399	DWORD const dwErrLast = RTNtLastErrorValue();
1400	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1401	"Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)", hrc, rcNtLast, dwErrLast);
1402	}
1403	#else
1404	hrc = WHvCreateVirtualProcessor(hPartition, iCpu, 0 /fFlags/);
1405	if (FAILED(hrc))
1406	{
1407	NTSTATUS const rcNtLast = RTNtLastStatusValue();
1408	DWORD const dwErrLast = RTNtLastErrorValue();
1409	while (iCpu-- > 0)
1410	{
1411	HRESULT hrc2 = WHvDeleteVirtualProcessor(hPartition, iCpu);
1412	AssertLogRelMsg(SUCCEEDED(hrc2), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
1413	hPartition, iCpu, hrc2, RTNtLastStatusValue(),
1414	RTNtLastErrorValue()));
1415	}
1416	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
1417	"Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)", hrc, rcNtLast, dwErrLast);
1418	}
1419	#endif /* !NEM_WIN_USE_OUR_OWN_RUN_API */
1420	}
1421	pVM->nem.s.fCreatedEmts = true;
1422
1423	/*
1424	* Do some more ring-0 initialization now that we've got the partition handle.
1425	*/
1426	int rc = VMMR3CallR0Emt(pVM, &pVM->aCpus[0], VMMR0_DO_NEM_INIT_VM_PART_2, 0, NULL);
1427	if (RT_SUCCESS(rc))
1428	{
1429	LogRel(("NEM: Successfully set up partition (device handle %p, partition ID %#llx)\n", hPartitionDevice, idHvPartition));
1430
1431	#if 1
1432	VMMR3CallR0Emt(pVM, &pVM->aCpus[0], VMMR0_DO_NEM_UPDATE_STATISTICS, 0, NULL);
1433	LogRel(("NEM: Memory balance: %#RX64 out of %#RX64 pages in use\n",
1434	pVM->nem.s.R0Stats.cPagesInUse, pVM->nem.s.R0Stats.cPagesAvailable));
1435	#endif
1436
1437	/*
1438	* Register statistics on shared pages.
1439	*/
1440	/** @todo HvCallMapStatsPage */
1441
1442	/*
1443	* Adjust features.
1444	* Note! We've already disabled X2APIC via CFGM during the first init call.
1445	*/
1446	return VINF_SUCCESS;
1447	}
1448	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to NEMR0InitVMPart2 failed: %Rrc", rc);
1449	}
1450
1451
1452	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
1453	{
1454	//BOOL fRet = SetThreadPriority(GetCurrentThread(), 0);
1455	//AssertLogRel(fRet);
1456
1457	NOREF(pVM); NOREF(enmWhat);
1458	return VINF_SUCCESS;
1459	}
1460
1461
1462	int nemR3NativeTerm(PVM pVM)
1463	{
1464	/*
1465	* Delete the partition.
1466	*/
1467	WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
1468	pVM->nem.s.hPartition = NULL;
1469	pVM->nem.s.hPartitionDevice = NULL;
1470	if (hPartition != NULL)
1471	{
1472	VMCPUID iCpu = pVM->nem.s.fCreatedEmts ? pVM->cCpus : 0;
1473	LogRel(("NEM: Destroying partition %p with its %u VCpus...\n", hPartition, iCpu));
1474	while (iCpu-- > 0)
1475	{
1476	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
1477	pVM->aCpus[iCpu].nem.s.pvMsgSlotMapping = NULL;
1478	#else
1479	HRESULT hrc = WHvDeleteVirtualProcessor(hPartition, iCpu);
1480	AssertLogRelMsg(SUCCEEDED(hrc), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
1481	hPartition, iCpu, hrc, RTNtLastStatusValue(),
1482	RTNtLastErrorValue()));
1483	#endif
1484	}
1485	WHvDeletePartition(hPartition);
1486	}
1487	pVM->nem.s.fCreatedEmts = false;
1488	return VINF_SUCCESS;
1489	}
1490
1491
1492	/**
1493	* VM reset notification.
1494	*
1495	* @param pVM The cross context VM structure.
1496	*/
1497	void nemR3NativeReset(PVM pVM)
1498	{
1499	/* Unfix the A20 gate. */
1500	pVM->nem.s.fA20Fixed = false;
1501	}
1502
1503
1504	/**
1505	* Reset CPU due to INIT IPI or hot (un)plugging.
1506	*
1507	* @param pVCpu The cross context virtual CPU structure of the CPU being
1508	* reset.
1509	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
1510	*/
1511	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
1512	{
1513	/* Lock the A20 gate if INIT IPI, make sure it's enabled. */
1514	if (fInitIpi && pVCpu->idCpu > 0)
1515	{
1516	PVM pVM = pVCpu->CTX_SUFF(pVM);
1517	if (!pVM->nem.s.fA20Enabled)
1518	nemR3NativeNotifySetA20(pVCpu, true);
1519	pVM->nem.s.fA20Enabled = true;
1520	pVM->nem.s.fA20Fixed = true;
1521	}
1522	}
1523
1524	#if 0 //ndef NEM_WIN_USE_OUR_OWN_RUN_API - migrating to NEMAllNativeTemplate-win.cpp.h */
1525
1526	# ifdef LOG_ENABLED
1527	/**
1528	* Log the full details of an exit reason.
1529	*
1530	* @param pExitReason The exit reason to log.
1531	*/
1532	static void nemR3WinLogWHvExitReason(WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1533	{
1534	bool fExitCtx = false;
1535	bool fExitInstr = false;
1536	switch (pExitReason->ExitReason)
1537	{
1538	case WHvRunVpExitReasonMemoryAccess:
1539	Log2(("Exit: Memory access: GCPhys=%RGp GCVirt=%RGv %s %s %s\n",
1540	pExitReason->MemoryAccess.Gpa, pExitReason->MemoryAccess.Gva,
1541	g_apszWHvMemAccesstypes[pExitReason->MemoryAccess.AccessInfo.AccessType],
1542	pExitReason->MemoryAccess.AccessInfo.GpaUnmapped ? "unmapped" : "mapped",
1543	pExitReason->MemoryAccess.AccessInfo.GvaValid ? "" : "invalid-gc-virt"));
1544	AssertMsg(!(pExitReason->MemoryAccess.AccessInfo.AsUINT32 & ~UINT32_C(0xf)),
1545	("MemoryAccess.AccessInfo=%#x\n", pExitReason->MemoryAccess.AccessInfo.AsUINT32));
1546	fExitCtx = fExitInstr = true;
1547	break;
1548
1549	case WHvRunVpExitReasonX64IoPortAccess:
1550	Log2(("Exit: I/O port access: IoPort=%#x LB %u %s%s%s rax=%#RX64 rcx=%#RX64 rsi=%#RX64 rdi=%#RX64\n",
1551	pExitReason->IoPortAccess.PortNumber,
1552	pExitReason->IoPortAccess.AccessInfo.AccessSize,
1553	pExitReason->IoPortAccess.AccessInfo.IsWrite ? "out" : "in",
1554	pExitReason->IoPortAccess.AccessInfo.StringOp ? " string" : "",
1555	pExitReason->IoPortAccess.AccessInfo.RepPrefix ? " rep" : "",
1556	pExitReason->IoPortAccess.Rax,
1557	pExitReason->IoPortAccess.Rcx,
1558	pExitReason->IoPortAccess.Rsi,
1559	pExitReason->IoPortAccess.Rdi));
1560	Log2(("Exit: + ds=%#x:{%#RX64 LB %#RX32, %#x} es=%#x:{%#RX64 LB %#RX32, %#x}\n",
1561	pExitReason->IoPortAccess.Ds.Selector,
1562	pExitReason->IoPortAccess.Ds.Base,
1563	pExitReason->IoPortAccess.Ds.Limit,
1564	pExitReason->IoPortAccess.Ds.Attributes,
1565	pExitReason->IoPortAccess.Es.Selector,
1566	pExitReason->IoPortAccess.Es.Base,
1567	pExitReason->IoPortAccess.Es.Limit,
1568	pExitReason->IoPortAccess.Es.Attributes ));
1569
1570	AssertMsg( pExitReason->IoPortAccess.AccessInfo.AccessSize == 1
1571	\|\| pExitReason->IoPortAccess.AccessInfo.AccessSize == 2
1572	\|\| pExitReason->IoPortAccess.AccessInfo.AccessSize == 4,
1573	("IoPortAccess.AccessInfo.AccessSize=%d\n", pExitReason->IoPortAccess.AccessInfo.AccessSize));
1574	AssertMsg(!(pExitReason->IoPortAccess.AccessInfo.AsUINT32 & ~UINT32_C(0x3f)),
1575	("IoPortAccess.AccessInfo=%#x\n", pExitReason->IoPortAccess.AccessInfo.AsUINT32));
1576	fExitCtx = fExitInstr = true;
1577	break;
1578
1579	# if 0
1580	case WHvRunVpExitReasonUnrecoverableException:
1581	case WHvRunVpExitReasonInvalidVpRegisterValue:
1582	case WHvRunVpExitReasonUnsupportedFeature:
1583	case WHvRunVpExitReasonX64InterruptWindow:
1584	case WHvRunVpExitReasonX64Halt:
1585	case WHvRunVpExitReasonX64MsrAccess:
1586	case WHvRunVpExitReasonX64Cpuid:
1587	case WHvRunVpExitReasonException:
1588	case WHvRunVpExitReasonCanceled:
1589	case WHvRunVpExitReasonAlerted:
1590	WHV_X64_MSR_ACCESS_CONTEXT MsrAccess;
1591	WHV_X64_CPUID_ACCESS_CONTEXT CpuidAccess;
1592	WHV_VP_EXCEPTION_CONTEXT VpException;
1593	WHV_X64_INTERRUPTION_DELIVERABLE_CONTEXT InterruptWindow;
1594	WHV_UNRECOVERABLE_EXCEPTION_CONTEXT UnrecoverableException;
1595	WHV_X64_UNSUPPORTED_FEATURE_CONTEXT UnsupportedFeature;
1596	WHV_RUN_VP_CANCELED_CONTEXT CancelReason;
1597	# endif
1598
1599	case WHvRunVpExitReasonNone:
1600	Log2(("Exit: No reason\n"));
1601	AssertFailed();
1602	break;
1603
1604	default:
1605	Log(("Exit: %#x\n", pExitReason->ExitReason));
1606	break;
1607	}
1608
1609	/*
1610	* Context and maybe instruction details.
1611	*/
1612	if (fExitCtx)
1613	{
1614	const WHV_VP_EXIT_CONTEXT *pVpCtx = &pExitReason->VpContext;
1615	Log2(("Exit: + CS:RIP=%04x:%08RX64 RFLAGS=%06RX64 cbInstr=%u CS={%RX64 L %#RX32, %#x}\n",
1616	pVpCtx->Cs.Selector,
1617	pVpCtx->Rip,
1618	pVpCtx->Rflags,
1619	pVpCtx->InstructionLength,
1620	pVpCtx->Cs.Base, pVpCtx->Cs.Limit, pVpCtx->Cs.Attributes));
1621	Log2(("Exit: + cpl=%d CR0.PE=%d CR0.AM=%d EFER.LMA=%d DebugActive=%d InterruptionPending=%d InterruptShadow=%d\n",
1622	pVpCtx->ExecutionState.Cpl,
1623	pVpCtx->ExecutionState.Cr0Pe,
1624	pVpCtx->ExecutionState.Cr0Am,
1625	pVpCtx->ExecutionState.EferLma,
1626	pVpCtx->ExecutionState.DebugActive,
1627	pVpCtx->ExecutionState.InterruptionPending,
1628	pVpCtx->ExecutionState.InterruptShadow));
1629	AssertMsg(!(pVpCtx->ExecutionState.AsUINT16 & ~UINT16_C(0x107f)),
1630	("ExecutionState.AsUINT16=%#x\n", pVpCtx->ExecutionState.AsUINT16));
1631
1632	/** @todo Someone at Microsoft please explain why the InstructionBytes fields
1633	* are 16 bytes long, when 15 would've been sufficent and saved 3-7 bytes of
1634	* alignment padding? Intel max length is 15, so is this sSome ARM stuff?
1635	* Aren't ARM
1636	* instructions max 32-bit wide? Confused. */
1637	if (fExitInstr && pExitReason->IoPortAccess.InstructionByteCount > 0)
1638	Log2(("Exit: + Instruction %.*Rhxs\n",
1639	pExitReason->IoPortAccess.InstructionByteCount, &pExitReason->IoPortAccess.InstructionBytes[0]));
1640	}
1641	}
1642	# endif /* LOG_ENABLED */
1643
1644
1645	static VBOXSTRICTRC nemR3WinWHvHandleHalt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1646	{
1647	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx);
1648	LogFlow(("nemR3WinWHvHandleHalt\n"));
1649	return VINF_EM_HALT;
1650	}
1651
1652
1653	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1654	/**
1655	* @callback_method_impl{FNPGMPHYSNEMENUMCALLBACK,
1656	* Hack to unmap all pages when/before we run into quota (WHv only).}
1657	*/
1658	static DECLCALLBACK(int) nemR3WinWHvUnmapOnePageCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, uint8_t pu2NemState, void pvUser)
1659	{
1660	RT_NOREF_PV(pvUser);
1661	RT_NOREF_PV(pVCpu);
1662	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1663	if (SUCCEEDED(hrc))
1664	{
1665	Log5(("NEM GPA unmap all: %RGp (cMappedPages=%u)\n", GCPhys, pVM->nem.s.cMappedPages - 1));
1666	*pu2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1667	}
1668	else
1669	{
1670	LogRel(("nemR3WinWHvUnmapOnePageCallback: GCPhys=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1671	GCPhys, g_apszPageStates[*pu2NemState], hrc, hrc, RTNtLastStatusValue(),
1672	RTNtLastErrorValue(), pVM->nem.s.cMappedPages));
1673	*pu2NemState = NEM_WIN_PAGE_STATE_NOT_SET;
1674	}
1675	if (pVM->nem.s.cMappedPages > 0)
1676	ASMAtomicDecU32(&pVM->nem.s.cMappedPages);
1677	return VINF_SUCCESS;
1678	}
1679	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
1680
1681
1682	/**
1683	* Handles an memory access VMEXIT.
1684	*
1685	* This can be triggered by a number of things.
1686	*
1687	* @returns Strict VBox status code.
1688	* @param pVM The cross context VM structure.
1689	* @param pVCpu The cross context virtual CPU structure.
1690	* @param pCtx The CPU context to update.
1691	* @param pMemCtx The exit reason information.
1692	* @param pVpContext The processor context info associated with the exit.
1693	*/
1694	static VBOXSTRICTRC nemR3WinWHvHandleMemoryAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_MEMORY_ACCESS_CONTEXT const *pMemCtx,
1695	WHV_VP_EXIT_CONTEXT const *pVpContext)
1696	{
1697	/*
1698	* Ask PGM for information about the given GCPhys. We need to check if we're
1699	* out of sync first.
1700	*/
1701	NEMHCWINHMACPCCSTATE State = { pMemCtx->AccessInfo.AccessType == WHvMemoryAccessWrite, false, false };
1702	PGMPHYSNEMPAGEINFO Info;
1703	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pMemCtx->Gpa, State.fWriteAccess, &Info,
1704	nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
1705	if (RT_SUCCESS(rc))
1706	{
1707	if (Info.fNemProt & (pMemCtx->AccessInfo.AccessType == WHvMemoryAccessWrite ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
1708	{
1709	if (State.fCanResume)
1710	{
1711	Log4(("MemExit: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
1712	pMemCtx->Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
1713	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
1714	State.fDidSomething ? "" : " no-change", g_apszWHvMemAccesstypes[pMemCtx->AccessInfo.AccessType]));
1715	return VINF_SUCCESS;
1716	}
1717	}
1718	Log4(("MemExit: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
1719	pMemCtx->Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
1720	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
1721	State.fDidSomething ? "" : " no-change", g_apszWHvMemAccesstypes[pMemCtx->AccessInfo.AccessType]));
1722	}
1723	else
1724	Log4(("MemExit: %RGp rc=%Rrc%s; emulating (%s)\n", pMemCtx->Gpa, rc,
1725	State.fDidSomething ? " modified-backing" : "", g_apszWHvMemAccesstypes[pMemCtx->AccessInfo.AccessType]));
1726
1727	/*
1728	* Emulate the memory access, either access handler or special memory.
1729	*/
1730	rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
1731	AssertRCReturn(rc, rc);
1732
1733	VBOXSTRICTRC rcStrict;
1734	if (pMemCtx->InstructionByteCount > 0)
1735	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(pCtx), pVpContext->Rip,
1736	&pMemCtx->InstructionBytes[0], pMemCtx->InstructionByteCount);
1737	else
1738	rcStrict = IEMExecOne(pVCpu);
1739	/** @todo do we need to do anything wrt debugging here? */
1740	return rcStrict;
1741	}
1742
1743
1744	/**
1745	* Handles an I/O port access VMEXIT.
1746	*
1747	* We ASSUME that the hypervisor has don't I/O port access control.
1748	*
1749	* @returns Strict VBox status code.
1750	* @param pVM The cross context VM structure.
1751	* @param pVCpu The cross context virtual CPU structure.
1752	* @param pCtx The CPU context to update.
1753	* @param pIoPortCtx The exit reason information.
1754	* @param pVpContext The processor context info associated with the exit.
1755	*/
1756	static VBOXSTRICTRC
1757	nemR3WinWHvHandleIoPortAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_X64_IO_PORT_ACCESS_CONTEXT const *pIoPortCtx,
1758	WHV_VP_EXIT_CONTEXT const *pVpContext)
1759	{
1760	Assert( pIoPortCtx->AccessInfo.AccessSize == 1
1761	\|\| pIoPortCtx->AccessInfo.AccessSize == 2
1762	\|\| pIoPortCtx->AccessInfo.AccessSize == 4);
1763
1764	VBOXSTRICTRC rcStrict;
1765	if (!pIoPortCtx->AccessInfo.StringOp)
1766	{
1767	/*
1768	* Simple port I/O.
1769	*/
1770	//Assert(pCtx->rax == pIoPortCtx->Rax); - sledgehammer
1771
1772	static uint32_t const s_fAndMask[8] =
1773	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
1774	uint32_t const fAndMask = s_fAndMask[pIoPortCtx->AccessInfo.AccessSize];
1775	if (pIoPortCtx->AccessInfo.IsWrite)
1776	{
1777	rcStrict = IOMIOPortWrite(pVM, pVCpu, pIoPortCtx->PortNumber, (uint32_t)pIoPortCtx->Rax & fAndMask,
1778	pIoPortCtx->AccessInfo.AccessSize);
1779	if (IOM_SUCCESS(rcStrict))
1780	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, pCtx, pVpContext);
1781	}
1782	else
1783	{
1784	uint32_t uValue = 0;
1785	rcStrict = IOMIOPortRead(pVM, pVCpu, pIoPortCtx->PortNumber, &uValue,
1786	pIoPortCtx->AccessInfo.AccessSize);
1787	if (IOM_SUCCESS(rcStrict))
1788	{
1789	pCtx->eax = (pCtx->eax & ~fAndMask) \| (uValue & fAndMask);
1790	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, pCtx, pVpContext);
1791	}
1792	}
1793	}
1794	else
1795	{
1796	/*
1797	* String port I/O.
1798	*/
1799	/** @todo Someone at Microsoft please explain how we can get the address mode
1800	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
1801	* getting the default mode, it can always be overridden by a prefix. This
1802	* forces us to interpret the instruction from opcodes, which is suboptimal.
1803	* Both AMD-V and VT-x includes the address size in the exit info, at least on
1804	* CPUs that are reasonably new. */
1805	# if 0 // requires sledgehammer
1806	Assert( pIoPortCtx->Ds.Base == pCtx->ds.u64Base
1807	&& pIoPortCtx->Ds.Limit == pCtx->ds.u32Limit
1808	&& pIoPortCtx->Ds.Selector == pCtx->ds.Sel);
1809	Assert( pIoPortCtx->Es.Base == pCtx->es.u64Base
1810	&& pIoPortCtx->Es.Limit == pCtx->es.u32Limit
1811	&& pIoPortCtx->Es.Selector == pCtx->es.Sel);
1812	Assert(pIoPortCtx->Rdi == pCtx->rdi);
1813	Assert(pIoPortCtx->Rsi == pCtx->rsi);
1814	Assert(pIoPortCtx->Rcx == pCtx->rcx);
1815	Assert(pIoPortCtx->Rcx == pCtx->rcx);
1816	# endif
1817
1818	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
1819	AssertRCReturn(rc, rc);
1820
1821	rcStrict = IEMExecOne(pVCpu);
1822	}
1823	if (IOM_SUCCESS(rcStrict))
1824	{
1825	/*
1826	* Do debug checks.
1827	*/
1828	if ( pVpContext->ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflext DR7? */
1829	\|\| (pVpContext->Rflags & X86_EFL_TF)
1830	\|\| DBGFBpIsHwIoArmed(pVM) )
1831	{
1832	/** @todo Debugging. */
1833	}
1834	}
1835	return rcStrict;
1836	}
1837
1838
1839	static VBOXSTRICTRC nemR3WinWHvHandleInterruptWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1840	{
1841	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
1842	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
1843	}
1844
1845
1846	static VBOXSTRICTRC nemR3WinWHvHandleMsrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1847	{
1848	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
1849	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
1850	}
1851
1852
1853	static VBOXSTRICTRC nemR3WinWHvHandleCpuId(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1854	{
1855	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
1856	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
1857	}
1858
1859
1860	static VBOXSTRICTRC nemR3WinWHvHandleException(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1861	{
1862	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
1863	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
1864	}
1865
1866
1867	static VBOXSTRICTRC nemR3WinWHvHandleUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1868	{
1869	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
1870	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
1871	}
1872
1873
1874	static VBOXSTRICTRC nemR3WinWHvHandleTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1875	{
1876	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
1877	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
1878	}
1879
1880
1881	static VBOXSTRICTRC nemR3WinWHvHandleInvalidState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, WHV_RUN_VP_EXIT_CONTEXT const *pExitReason)
1882	{
1883	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx); NOREF(pExitReason);
1884	AssertLogRelFailedReturn(VERR_NOT_IMPLEMENTED);
1885	}
1886
1887
1888	VBOXSTRICTRC nemR3WinWHvRunGC(PVM pVM, PVMCPU pVCpu)
1889	{
1890	# ifdef LOG_ENABLED
1891	if (LogIs3Enabled())
1892	{
1893	Log3(("nemR3NativeRunGC: Entering #%u\n", pVCpu->idCpu));
1894	nemHCWinLogState(pVM, pVCpu);
1895	}
1896	# endif
1897
1898	/*
1899	* The run loop.
1900	*/
1901	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1902	const bool fSingleStepping = false; /** @todo get this from somewhere. */
1903	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1904	for (unsigned iLoop = 0;;iLoop++)
1905	{
1906	/*
1907	* Copy the state.
1908	*/
1909	int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu, pCtx);
1910	AssertRCBreakStmt(rc2, rcStrict = rc2);
1911
1912	/*
1913	* Run a bit.
1914	*/
1915	WHV_RUN_VP_EXIT_CONTEXT ExitReason;
1916	RT_ZERO(ExitReason);
1917	if ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
1918	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
1919	{
1920	Log8(("Calling WHvRunVirtualProcessor\n"));
1921	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED);
1922	HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
1923	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM);
1924	AssertLogRelMsgBreakStmt(SUCCEEDED(hrc),
1925	("WHvRunVirtualProcessor(%p, %u,,) -> %Rhrc (Last=%#x/%u)\n", pVM->nem.s.hPartition, pVCpu->idCpu,
1926	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()),
1927	rcStrict = VERR_INTERNAL_ERROR);
1928	Log2(("WHvRunVirtualProcessor -> %#x; exit code %#x (%d) (cpu status %u)\n",
1929	hrc, ExitReason.ExitReason, ExitReason.ExitReason, nemHCWinCpuGetRunningStatus(pVCpu) ));
1930	}
1931	else
1932	{
1933	LogFlow(("nemR3NativeRunGC: returning: pending FF (pre exec)\n"));
1934	break;
1935	}
1936
1937	# if 0 /* sledgehammer approach */
1938	/*
1939	* Copy back the state.
1940	*/
1941	rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, UINT64_MAX);
1942	AssertRCBreakStmt(rc2, rcStrict = rc2);
1943	# endif
1944
1945	# ifdef LOG_ENABLED
1946	/*
1947	* Do some logging.
1948	*/
1949	if (LogIs2Enabled())
1950	nemR3WinLogWHvExitReason(&ExitReason);
1951	if (LogIs3Enabled())
1952	nemHCWinLogState(pVM, pVCpu);
1953	# endif
1954
1955	# if 0 //def VBOX_STRICT - requires sledgehammer
1956	/* Assert that the VpContext field makes sense. */
1957	switch (ExitReason.ExitReason)
1958	{
1959	case WHvRunVpExitReasonMemoryAccess:
1960	case WHvRunVpExitReasonX64IoPortAccess:
1961	case WHvRunVpExitReasonX64MsrAccess:
1962	case WHvRunVpExitReasonX64Cpuid:
1963	case WHvRunVpExitReasonException:
1964	case WHvRunVpExitReasonUnrecoverableException:
1965	Assert( ExitReason.IoPortAccess.VpContext.InstructionLength > 0
1966	\|\| ( ExitReason.ExitReason == WHvRunVpExitReasonMemoryAccess
1967	&& ExitReason.MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessExecute));
1968	Assert(ExitReason.IoPortAccess.VpContext.InstructionLength < 16);
1969	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Cpl == CPUMGetGuestCPL(pVCpu));
1970	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Cr0Pe == RT_BOOL(pCtx->cr0 & X86_CR0_PE));
1971	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Cr0Am == RT_BOOL(pCtx->cr0 & X86_CR0_AM));
1972	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.EferLma == RT_BOOL(pCtx->msrEFER & MSR_K6_EFER_LMA));
1973	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.DebugActive == RT_BOOL(pCtx->dr[7] & X86_DR7_ENABLED_MASK));
1974	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Reserved0 == 0);
1975	Assert(ExitReason.IoPortAccess.VpContext.ExecutionState.Reserved1 == 0);
1976	Assert(ExitReason.IoPortAccess.VpContext.Rip == pCtx->rip);
1977	Assert(ExitReason.IoPortAccess.VpContext.Rflags == pCtx->rflags.u);
1978	Assert( ExitReason.IoPortAccess.VpContext.Cs.Base == pCtx->cs.u64Base
1979	&& ExitReason.IoPortAccess.VpContext.Cs.Limit == pCtx->cs.u32Limit
1980	&& ExitReason.IoPortAccess.VpContext.Cs.Selector == pCtx->cs.Sel);
1981	break;
1982	default: break; /* shut up compiler. */
1983	}
1984	# endif
1985
1986	/*
1987	* Deal with the exit.
1988	*/
1989	switch (ExitReason.ExitReason)
1990	{
1991	/* Frequent exits: */
1992	case WHvRunVpExitReasonCanceled:
1993	//case WHvRunVpExitReasonAlerted:
1994	rcStrict = VINF_SUCCESS;
1995	break;
1996
1997	case WHvRunVpExitReasonX64Halt:
1998	rcStrict = nemR3WinWHvHandleHalt(pVM, pVCpu, pCtx);
1999	break;
2000
2001	case WHvRunVpExitReasonMemoryAccess:
2002	rcStrict = nemR3WinWHvHandleMemoryAccess(pVM, pVCpu, pCtx, &ExitReason.MemoryAccess, &ExitReason.VpContext);
2003	break;
2004
2005	case WHvRunVpExitReasonX64IoPortAccess:
2006	rcStrict = nemR3WinWHvHandleIoPortAccess(pVM, pVCpu, pCtx, &ExitReason.IoPortAccess, &ExitReason.VpContext);
2007	break;
2008
2009	case WHvRunVpExitReasonX64InterruptWindow:
2010	rcStrict = nemR3WinWHvHandleInterruptWindow(pVM, pVCpu, pCtx, &ExitReason);
2011	break;
2012
2013	case WHvRunVpExitReasonX64MsrAccess: /* needs configuring */
2014	rcStrict = nemR3WinWHvHandleMsrAccess(pVM, pVCpu, pCtx, &ExitReason);
2015	break;
2016
2017	case WHvRunVpExitReasonX64Cpuid: /* needs configuring */
2018	rcStrict = nemR3WinWHvHandleCpuId(pVM, pVCpu, pCtx, &ExitReason);
2019	break;
2020
2021	case WHvRunVpExitReasonException: /* needs configuring */
2022	rcStrict = nemR3WinWHvHandleException(pVM, pVCpu, pCtx, &ExitReason);
2023	break;
2024
2025	/* Unlikely exits: */
2026	case WHvRunVpExitReasonUnsupportedFeature:
2027	rcStrict = nemR3WinWHvHandleUD(pVM, pVCpu, pCtx, &ExitReason);
2028	break;
2029
2030	case WHvRunVpExitReasonUnrecoverableException:
2031	rcStrict = nemR3WinWHvHandleTripleFault(pVM, pVCpu, pCtx, &ExitReason);
2032	break;
2033
2034	case WHvRunVpExitReasonInvalidVpRegisterValue:
2035	rcStrict = nemR3WinWHvHandleInvalidState(pVM, pVCpu, pCtx, &ExitReason);
2036	break;
2037
2038	/* Undesired exits: */
2039	case WHvRunVpExitReasonNone:
2040	default:
2041	AssertLogRelMsgFailed(("Unknown ExitReason: %#x\n", ExitReason.ExitReason));
2042	rcStrict = VERR_INTERNAL_ERROR_3;
2043	break;
2044	}
2045	if (rcStrict != VINF_SUCCESS)
2046	{
2047	LogFlow(("nemR3NativeRunGC: returning: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
2048	break;
2049	}
2050
2051	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2052	/* Hack alert! */
2053	uint32_t const cMappedPages = pVM->nem.s.cMappedPages;
2054	if (cMappedPages < 4000)
2055	{ /* likely */ }
2056	else
2057	{
2058	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinWHvUnmapOnePageCallback, NULL);
2059	Log(("nemR3NativeRunGC: Unmapped all; cMappedPages=%u -> %u\n", cMappedPages, pVM->nem.s.cMappedPages));
2060	}
2061	# endif
2062
2063	/* If any FF is pending, return to the EM loops. That's okay for the
2064	current sledgehammer approach. */
2065	if ( VM_FF_IS_PENDING( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2066	\|\| VMCPU_FF_IS_PENDING(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2067	{
2068	LogFlow(("nemR3NativeRunGC: returning: pending FF (%#x / %#x)\n", pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
2069	break;
2070	}
2071	}
2072
2073
2074	/*
2075	* Copy back the state before returning.
2076	*/
2077	if (pCtx->fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
2078	{
2079	int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK);
2080	if (RT_SUCCESS(rc2))
2081	pCtx->fExtrn = 0;
2082	else if (RT_SUCCESS(rcStrict))
2083	rcStrict = rc2;
2084	}
2085	else
2086	pCtx->fExtrn = 0;
2087
2088	return rcStrict;
2089	}
2090
2091	#endif /* !NEM_WIN_USE_OUR_OWN_RUN_API - migrating to NEMAllNativeTemplate-win.cpp.h*/
2092
2093
2094	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2095	{
2096	#if !defined(NEM_WIN_USE_OUR_OWN_RUN_API) \|\| 0
2097	return nemHCWinRunGC(pVM, pVCpu, NULL /pGVM/, NULL /pGVCpu/);
2098	#else
2099	for (;;)
2100	{
2101	VBOXSTRICTRC rcStrict = VMMR3CallR0EmtFast(pVM, pVCpu, VMMR0_DO_NEM_RUN);
2102	if (RT_SUCCESS(rcStrict))
2103	{
2104	/*
2105	* We deal with VINF_NEM_CHANGE_PGM_MODE, VINF_NEM_FLUSH_TLB and
2106	* VINF_NEM_UPDATE_APIC_BASE here, since we're running the risk of
2107	* getting these while we already got another RC (I/O ports).
2108	*
2109	* The APIC base update and a PGM update can happen at the same time, so
2110	* we don't depend on the status code for that and always checks it first.
2111	*/
2112	/* APIC base: */
2113	if (pVCpu->nem.s.uPendingApicBase != UINT64_MAX)
2114	{
2115	LogFlow(("nemR3NativeRunGC: calling APICSetBaseMsr(,%RX64)...\n", pVCpu->nem.s.uPendingApicBase));
2116	VBOXSTRICTRC rc2 = APICSetBaseMsr(pVCpu, pVCpu->nem.s.uPendingApicBase);
2117	AssertLogRelMsg(rc2 == VINF_SUCCESS, ("rc2=%Rrc [%#RX64]\n", VBOXSTRICTRC_VAL(rc2), pVCpu->nem.s.uPendingApicBase));
2118	pVCpu->nem.s.uPendingApicBase = UINT64_MAX;
2119	}
2120
2121	/* Status codes: */
2122	VBOXSTRICTRC rcPending = pVCpu->nem.s.rcPending;
2123	pVCpu->nem.s.rcPending = VINF_SUCCESS;
2124	if ( rcStrict == VINF_NEM_CHANGE_PGM_MODE
2125	\|\| rcStrict == VINF_PGM_CHANGE_MODE
2126	\|\| rcPending == VINF_NEM_CHANGE_PGM_MODE )
2127	{
2128	LogFlow(("nemR3NativeRunGC: calling PGMChangeMode...\n"));
2129	int rc = PGMChangeMode(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR4(pVCpu), CPUMGetGuestEFER(pVCpu));
2130	AssertRCReturn(rc, rc);
2131	if (rcStrict == VINF_NEM_CHANGE_PGM_MODE \|\| rcStrict == VINF_NEM_FLUSH_TLB)
2132	{
2133	if ( !VM_FF_IS_PENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK \| VM_FF_HP_R0_PRE_HM_MASK)
2134	&& !VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_HIGH_PRIORITY_POST_MASK \| VMCPU_FF_HP_R0_PRE_HM_MASK)
2135	& ~VMCPU_FF_RESUME_GUEST_MASK))
2136	{
2137	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
2138	continue;
2139	}
2140	rcStrict = VINF_SUCCESS;
2141	}
2142	}
2143	else if (rcStrict == VINF_NEM_FLUSH_TLB \|\| rcPending == VINF_NEM_FLUSH_TLB)
2144	{
2145	LogFlow(("nemR3NativeRunGC: calling PGMFlushTLB...\n"));
2146	int rc = PGMFlushTLB(pVCpu, CPUMGetGuestCR3(pVCpu), true);
2147	AssertRCReturn(rc, rc);
2148	if (rcStrict == VINF_NEM_FLUSH_TLB \|\| rcStrict == VINF_NEM_CHANGE_PGM_MODE)
2149	{
2150	if ( !VM_FF_IS_PENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK \| VM_FF_HP_R0_PRE_HM_MASK)
2151	&& !VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_HIGH_PRIORITY_POST_MASK \| VMCPU_FF_HP_R0_PRE_HM_MASK)
2152	& ~VMCPU_FF_RESUME_GUEST_MASK))
2153	{
2154	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
2155	continue;
2156	}
2157	rcStrict = VINF_SUCCESS;
2158	}
2159	}
2160	else if (rcStrict == VINF_NEM_UPDATE_APIC_BASE \|\| rcPending == VERR_NEM_UPDATE_APIC_BASE)
2161	continue;
2162	else
2163	AssertMsg(rcPending == VINF_SUCCESS, ("rcPending=%Rrc\n", VBOXSTRICTRC_VAL(rcPending) ));
2164	}
2165	LogFlow(("nemR3NativeRunGC: returns %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
2166	return rcStrict;
2167	}
2168	#endif
2169	}
2170
2171
2172	bool nemR3NativeCanExecuteGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2173	{
2174	NOREF(pVM); NOREF(pVCpu); NOREF(pCtx);
2175	return true;
2176	}
2177
2178
2179	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
2180	{
2181	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
2182	return false;
2183	}
2184
2185
2186	/**
2187	* Forced flag notification call from VMEmt.h.
2188	*
2189	* This is only called when pVCpu is in the VMCPUSTATE_STARTED_EXEC_NEM state.
2190	*
2191	* @param pVM The cross context VM structure.
2192	* @param pVCpu The cross context virtual CPU structure of the CPU
2193	* to be notified.
2194	* @param fFlags Notification flags, VMNOTIFYFF_FLAGS_XXX.
2195	*/
2196	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
2197	{
2198	#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
2199	nemHCWinCancelRunVirtualProcessor(pVM, pVCpu);
2200	#else
2201	Log8(("nemR3NativeNotifyFF: canceling %u\n", pVCpu->idCpu));
2202	HRESULT hrc = WHvCancelRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, 0);
2203	AssertMsg(SUCCEEDED(hrc), ("WHvCancelRunVirtualProcessor -> hrc=%Rhrc\n", hrc));
2204	RT_NOREF_PV(hrc);
2205	#endif
2206	RT_NOREF_PV(fFlags);
2207	}
2208
2209
2210	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
2211	{
2212	PGMPAGEMAPLOCK Lock;
2213	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
2214	if (RT_SUCCESS(rc))
2215	PGMPhysReleasePageMappingLock(pVM, &Lock);
2216	return rc;
2217	}
2218
2219
2220	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
2221	{
2222	PGMPAGEMAPLOCK Lock;
2223	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
2224	if (RT_SUCCESS(rc))
2225	PGMPhysReleasePageMappingLock(pVM, &Lock);
2226	return rc;
2227	}
2228
2229
2230	int nemR3NativeNotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2231	{
2232	Log5(("nemR3NativeNotifyPhysRamRegister: %RGp LB %RGp\n", GCPhys, cb));
2233	NOREF(pVM); NOREF(GCPhys); NOREF(cb);
2234	return VINF_SUCCESS;
2235	}
2236
2237
2238	int nemR3NativeNotifyPhysMmioExMap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvMmio2)
2239	{
2240	Log5(("nemR3NativeNotifyPhysMmioExMap: %RGp LB %RGp fFlags=%#x pvMmio2=%p\n", GCPhys, cb, fFlags, pvMmio2));
2241	NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags); NOREF(pvMmio2);
2242	return VINF_SUCCESS;
2243	}
2244
2245
2246	int nemR3NativeNotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
2247	{
2248	Log5(("nemR3NativeNotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
2249	NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
2250	return VINF_SUCCESS;
2251	}
2252
2253
2254	/**
2255	* Called early during ROM registration, right after the pages have been
2256	* allocated and the RAM range updated.
2257	*
2258	* This will be succeeded by a number of NEMHCNotifyPhysPageProtChanged() calls
2259	* and finally a NEMR3NotifyPhysRomRegisterEarly().
2260	*
2261	* @returns VBox status code
2262	* @param pVM The cross context VM structure.
2263	* @param GCPhys The ROM address (page aligned).
2264	* @param cb The size (page aligned).
2265	* @param fFlags NEM_NOTIFY_PHYS_ROM_F_XXX.
2266	*/
2267	int nemR3NativeNotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
2268	{
2269	Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
2270	#if 0 /* Let's not do this after all. We'll protection change notifications for each page and if not we'll map them lazily. */
2271	RTGCPHYS const cPages = cb >> X86_PAGE_SHIFT;
2272	for (RTGCPHYS iPage = 0; iPage < cPages; iPage++, GCPhys += X86_PAGE_SIZE)
2273	{
2274	const void *pvPage;
2275	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhys, &pvPage);
2276	if (RT_SUCCESS(rc))
2277	{
2278	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhys, X86_PAGE_SIZE,
2279	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute);
2280	if (SUCCEEDED(hrc))
2281	{ /* likely */ }
2282	else
2283	{
2284	LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
2285	GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2286	return VERR_NEM_INIT_FAILED;
2287	}
2288	}
2289	else
2290	{
2291	LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
2292	return rc;
2293	}
2294	}
2295	#else
2296	NOREF(pVM); NOREF(GCPhys); NOREF(cb);
2297	#endif
2298	RT_NOREF_PV(fFlags);
2299	return VINF_SUCCESS;
2300	}
2301
2302
2303	/**
2304	* Called after the ROM range has been fully completed.
2305	*
2306	* This will be preceeded by a NEMR3NotifyPhysRomRegisterEarly() call as well a
2307	* number of NEMHCNotifyPhysPageProtChanged calls.
2308	*
2309	* @returns VBox status code
2310	* @param pVM The cross context VM structure.
2311	* @param GCPhys The ROM address (page aligned).
2312	* @param cb The size (page aligned).
2313	* @param fFlags NEM_NOTIFY_PHYS_ROM_F_XXX.
2314	*/
2315	int nemR3NativeNotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags)
2316	{
2317	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp fFlags=%#x\n", GCPhys, cb, fFlags));
2318	NOREF(pVM); NOREF(GCPhys); NOREF(cb); NOREF(fFlags);
2319	return VINF_SUCCESS;
2320	}
2321
2322
2323	/**
2324	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
2325	*/
2326	static DECLCALLBACK(int) nemR3WinUnsetForA20CheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys,
2327	PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
2328	{
2329	/* We'll just unmap the memory. */
2330	if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
2331	{
2332	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2333	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
2334	AssertRC(rc);
2335	if (RT_SUCCESS(rc))
2336	#else
2337	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
2338	if (SUCCEEDED(hrc))
2339	#endif
2340	{
2341	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2342	Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
2343	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
2344	}
2345	else
2346	{
2347	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2348	LogRel(("nemR3WinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
2349	return rc;
2350	#else
2351	LogRel(("nemR3WinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
2352	GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2353	return VERR_INTERNAL_ERROR_2;
2354	#endif
2355	}
2356	}
2357	RT_NOREF(pVCpu, pvUser);
2358	return VINF_SUCCESS;
2359	}
2360
2361
2362	/**
2363	* Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
2364	*
2365	* @returns The PGMPhysNemQueryPageInfo result.
2366	* @param pVM The cross context VM structure.
2367	* @param pVCpu The cross context virtual CPU structure.
2368	* @param GCPhys The page to unmap.
2369	*/
2370	static int nemR3WinUnmapPageForA20Gate(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
2371	{
2372	PGMPHYSNEMPAGEINFO Info;
2373	return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /fMakeWritable/, &Info,
2374	nemR3WinUnsetForA20CheckerCallback, NULL);
2375	}
2376
2377
2378	/**
2379	* Called when the A20 state changes.
2380	*
2381	* Hyper-V doesn't seem to offer a simple way of implementing the A20 line
2382	* features of PCs. So, we do a very minimal emulation of the HMA to make DOS
2383	* happy.
2384	*
2385	* @param pVCpu The CPU the A20 state changed on.
2386	* @param fEnabled Whether it was enabled (true) or disabled.
2387	*/
2388	void nemR3NativeNotifySetA20(PVMCPU pVCpu, bool fEnabled)
2389	{
2390	Log(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
2391	PVM pVM = pVCpu->CTX_SUFF(pVM);
2392	if (!pVM->nem.s.fA20Fixed)
2393	{
2394	pVM->nem.s.fA20Enabled = fEnabled;
2395	for (RTGCPHYS GCPhys = _1M; GCPhys < _1M + _64K; GCPhys += X86_PAGE_SIZE)
2396	nemR3WinUnmapPageForA20Gate(pVM, pVCpu, GCPhys);
2397	}
2398	}
2399
2400
2401	/** @page pg_nem_win NEM/win - Native Execution Manager, Windows.
2402	*
2403	* On Windows the Hyper-V root partition (dom0 in zen terminology) does not have
2404	* nested VT-x or AMD-V capabilities. For a while raw-mode worked inside it,
2405	* but for a while now we've been getting \#GP when trying to modify CR4 in the
2406	* world switcher. So, when Hyper-V is active on Windows we have little choice
2407	* but to use Hyper-V to run our VMs.
2408	*
2409	*
2410	* @section sub_nem_win_whv The WinHvPlatform API
2411	*
2412	* Since Windows 10 build 17083 there is a documented API for managing Hyper-V
2413	* VMs, header file WinHvPlatform.h and implementation in WinHvPlatform.dll.
2414	* This interface is a wrapper around the undocumented Virtualization
2415	* Infrastructure Driver (VID) API - VID.DLL and VID.SYS. The wrapper is
2416	* written in C++, namespaced, early versions (at least) was using standard C++
2417	* container templates in several places.
2418	*
2419	* When creating a VM using WHvCreatePartition, it will only create the
2420	* WinHvPlatform structures for it, to which you get an abstract pointer. The
2421	* VID API that actually creates the partition is first engaged when you call
2422	* WHvSetupPartition after first setting a lot of properties using
2423	* WHvSetPartitionProperty. Since the VID API is just a very thin wrapper
2424	* around CreateFile and NtDeviceIoControlFile, it returns an actual HANDLE for
2425	* the partition WinHvPlatform. We fish this HANDLE out of the WinHvPlatform
2426	* partition structures because we need to talk directly to VID for reasons
2427	* we'll get to in a bit. (Btw. we could also intercept the CreateFileW or
2428	* NtDeviceIoControlFile calls from VID.DLL to get the HANDLE should fishing in
2429	* the partition structures become difficult.)
2430	*
2431	* The WinHvPlatform API requires us to both set the number of guest CPUs before
2432	* setting up the partition and call WHvCreateVirtualProcessor for each of them.
2433	* The CPU creation function boils down to a VidMessageSlotMap call that sets up
2434	* and maps a message buffer into ring-3 for async communication with hyper-V
2435	* and/or the VID.SYS thread actually running the CPU thru
2436	* WinHvRunVpDispatchLoop(). When for instance a VMEXIT is encountered, hyper-V
2437	* sends a message that the WHvRunVirtualProcessor API retrieves (and later
2438	* acknowledges) via VidMessageSlotHandleAndGetNext. It should be noteded that
2439	* WHvDeleteVirtualProcessor doesn't do much as there seems to be no partner
2440	* function VidMessagesSlotMap that reverses what it did.
2441	*
2442	* Memory is managed thru calls to WHvMapGpaRange and WHvUnmapGpaRange (GPA does
2443	* not mean grade point average here, but rather guest physical addressspace),
2444	* which corresponds to VidCreateVaGpaRangeSpecifyUserVa and VidDestroyGpaRange
2445	* respectively. As 'UserVa' indicates, the functions works on user process
2446	* memory. The mappings are also subject to quota restrictions, so the number
2447	* of ranges are limited and probably their total size as well. Obviously
2448	* VID.SYS keeps track of the ranges, but so does WinHvPlatform, which means
2449	* there is a bit of overhead involved and quota restrctions makes sense. For
2450	* some reason though, regions are lazily mapped on VMEXIT/memory by
2451	* WHvRunVirtualProcessor.
2452	*
2453	* Running guest code is done thru the WHvRunVirtualProcessor function. It
2454	* asynchronously starts or resumes hyper-V CPU execution and then waits for an
2455	* VMEXIT message. Hyper-V / VID.SYS will return information about the message
2456	* in the message buffer mapping, and WHvRunVirtualProcessor will convert that
2457	* finto it's own WHV_RUN_VP_EXIT_CONTEXT format.
2458	*
2459	* Other threads can interrupt the execution by using WHvCancelVirtualProcessor,
2460	* which which case the thread in WHvRunVirtualProcessor is woken up via a dummy
2461	* QueueUserAPC and will call VidStopVirtualProcessor to asynchronously end
2462	* execution. The stop CPU call not immediately succeed if the CPU encountered
2463	* a VMEXIT before the stop was processed, in which case the VMEXIT needs to be
2464	* processed first, and the pending stop will be processed in a subsequent call
2465	* to WHvRunVirtualProcessor.
2466	*
2467	* Registers are retrieved and set via WHvGetVirtualProcessorRegisters and
2468	* WHvSetVirtualProcessorRegisters. In addition, several VMEXITs include
2469	* essential register state in the exit context information, potentially making
2470	* it possible to emulate the instruction causing the exit without involving
2471	* WHvGetVirtualProcessorRegisters.
2472	*
2473	*
2474	* @subsection subsec_nem_win_whv_cons Issues & Feedback
2475	*
2476	* Here are some observations (mostly against build 17101):
2477	*
2478	* - The VMEXIT performance is dismal (build 17134).
2479	*
2480	* Our proof of concept implementation with a kernel runloop (i.e. not using
2481	* WHvRunVirtualProcessor and friends, but calling VID.SYS fast I/O control
2482	* entry point directly) delivers 9-10% of the port I/O performance and only
2483	* 6-7% of the MMIO performance that we have with our own hypervisor.
2484	*
2485	* When using the offical WinHvPlatform API, the numbers are %3 for port I/O
2486	* and 5% for MMIO.
2487	*
2488	* While the tests we've done are using tight tight loops only doing port I/O
2489	* and MMIO, the problem is clearly visible when running regular guest OSes.
2490	* Anything that hammers the VGA device would be suffering, for example:
2491	*
2492	* - Windows 2000 boot screen animation overloads us with MMIO exits
2493	* and won't even boot because all the time is spent in interrupt
2494	* handlers and redrawin the screen.
2495	*
2496	* - DSL 4.4 and its bootmenu logo is slower than molasses in january.
2497	*
2498	* We have not found a workaround for this yet.
2499	*
2500	* Something that might improve the issue a little is to detect blocks with
2501	* excessive MMIO and port I/O exits and emulate instructions to cover
2502	* multiple exits before letting Hyper-V have a go at the guest execution
2503	* again. This will only improve the situation under some circumstances,
2504	* since emulating instructions without recompilation can be expensive, so
2505	* there will only be real gains if the exitting instructions are tightly
2506	* packed.
2507	*
2508	*
2509	* - Unable to access WHvX64RegisterMsrMtrrCap on AMD Ryzen (build 17134).
2510	*
2511	*
2512	* - On AMD Ryzen grub/debian 9.0 ends up with a unrecoverable exception
2513	* when IA32_MTRR_PHYSMASK0 is written.
2514	*
2515	*
2516	* - The IA32_APIC_BASE register does not work right:
2517	*
2518	* - Attempts by the guest to clear bit 11 (EN) are ignored, both the
2519	* guest and the VMM reads back the old value.
2520	*
2521	* - Attempts to modify the base address (bits NN:12) seems to be ignored
2522	* in the same way.
2523	*
2524	* - The VMM can modify both the base address as well as the the EN and
2525	* BSP bits, however this is useless if we cannot intercept the WRMSR.
2526	*
2527	* - Attempts by the guest to set the EXTD bit (X2APIC) result in \#GP(0),
2528	* while the VMM ends up with with ERROR_HV_INVALID_PARAMETER. Seems
2529	* there is no way to support X2APIC.
2530	*
2531	*
2532	* - The WHvCancelVirtualProcessor API schedules a dummy usermode APC callback
2533	* in order to cancel any current or future alertable wait in VID.SYS during
2534	* the VidMessageSlotHandleAndGetNext call.
2535	*
2536	* IIRC this will make the kernel schedule the specified callback thru
2537	* NTDLL!KiUserApcDispatcher by modifying the thread context and quite
2538	* possibly the userland thread stack. When the APC callback returns to
2539	* KiUserApcDispatcher, it will call NtContinue to restore the old thread
2540	* context and resume execution from there. This naturally adds up to some
2541	* CPU cycles, ring transitions aren't for free, especially after Spectre &
2542	* Meltdown mitigations.
2543	*
2544	* Using NtAltertThread call could do the same without the thread context
2545	* modifications and the extra kernel call.
2546	*
2547	*
2548	* - Not sure if this is a thing, but WHvCancelVirtualProcessor seems to cause
2549	* cause a lot more spurious WHvRunVirtualProcessor returns that what we get
2550	* with the replacement code. By spurious returns we mean that the
2551	* subsequent call to WHvRunVirtualProcessor would return immediately.
2552	*
2553	*
2554	* - When WHvRunVirtualProcessor returns without a message, or on a terse
2555	* VID message like HLT, it will make a kernel call to get some registers.
2556	* This is potentially inefficient if the caller decides he needs more
2557	* register state.
2558	*
2559	* It would be better to just return what's available and let the caller fetch
2560	* what is missing from his point of view in a single kernel call.
2561	*
2562	*
2563	* - The WHvRunVirtualProcessor implementation does lazy GPA range mappings when
2564	* a unmapped GPA message is received from hyper-V.
2565	*
2566	* Since MMIO is currently realized as unmapped GPA, this will slow down all
2567	* MMIO accesses a tiny little bit as WHvRunVirtualProcessor looks up the
2568	* guest physical address to check if it is a pending lazy mapping.
2569	*
2570	* The lazy mapping feature makes no sense to us. We as API user have all the
2571	* information and can do lazy mapping ourselves if we want/have to (see next
2572	* point).
2573	*
2574	*
2575	* - There is no API for modifying protection of a page within a GPA range.
2576	*
2577	* From what we can tell, the only way to modify the protection (like readonly
2578	* -> writable, or vice versa) is to first unmap the range and then remap it
2579	* with the new protection.
2580	*
2581	* We are for instance doing this quite a bit in order to track dirty VRAM
2582	* pages. VRAM pages starts out as readonly, when the guest writes to a page
2583	* we take an exit, notes down which page it is, makes it writable and restart
2584	* the instruction. After refreshing the display, we reset all the writable
2585	* pages to readonly again, bulk fashion.
2586	*
2587	* Now to work around this issue, we do page sized GPA ranges. In addition to
2588	* add a lot of tracking overhead to WinHvPlatform and VID.SYS, this also
2589	* causes us to exceed our quota before we've even mapped a default sized
2590	* (128MB) VRAM page-by-page. So, to work around this quota issue we have to
2591	* lazily map pages and actively restrict the number of mappings.
2592	*
2593	* Our best workaround thus far is bypassing WinHvPlatform and VID entirely
2594	* when in comes to guest memory management and instead use the underlying
2595	* hypercalls (HvCallMapGpaPages, HvCallUnmapGpaPages) to do it ourselves.
2596	* (This also maps a whole lot better into our own guest page management
2597	* infrastructure.)
2598	*
2599	*
2600	* - Observed problems doing WHvUnmapGpaRange immediately followed by
2601	* WHvMapGpaRange.
2602	*
2603	* As mentioned above, we've been forced to use this sequence when modifying
2604	* page protection. However, when transitioning from readonly to writable,
2605	* we've ended up looping forever with the same write to readonly memory
2606	* VMEXIT. We're wondering if this issue might be related to the lazy mapping
2607	* logic in WinHvPlatform.
2608	*
2609	* Workaround: Insert a WHvRunVirtualProcessor call and make sure to get a GPA
2610	* unmapped exit between the two calls. Not entirely great performance wise
2611	* (or the santity of our code).
2612	*
2613	*
2614	* - Implementing A20 gate behavior is tedious, where as correctly emulating the
2615	* A20M# pin (present on 486 and later) is near impossible for SMP setups
2616	* (e.g. possiblity of two CPUs with different A20 status).
2617	*
2618	* Workaround: Only do A20 on CPU 0, restricting the emulation to HMA. We
2619	* unmap all pages related to HMA (0x100000..0x10ffff) when the A20 state
2620	* changes, lazily syncing the right pages back when accessed.
2621	*
2622	*
2623	* - WHVRunVirtualProcessor wastes time converting VID/Hyper-V messages to its
2624	* own format (WHV_RUN_VP_EXIT_CONTEXT).
2625	*
2626	* We understand this might be because Microsoft wishes to remain free to
2627	* modify the VID/Hyper-V messages, but it's still rather silly and does slow
2628	* things down a little. We'd much rather just process the messages directly.
2629	*
2630	*
2631	* - WHVRunVirtualProcessor would've benefited from using a callback interface:
2632	*
2633	* - The potential size changes of the exit context structure wouldn't be
2634	* an issue, since the function could manage that itself.
2635	*
2636	* - State handling could probably be simplified (like cancelation).
2637	*
2638	*
2639	* - WHvGetVirtualProcessorRegisters and WHvSetVirtualProcessorRegisters
2640	* internally converts register names, probably using temporary heap buffers.
2641	*
2642	* From the looks of things, they are converting from WHV_REGISTER_NAME to
2643	* HV_REGISTER_NAME from in the "Virtual Processor Register Names" section in
2644	* the "Hypervisor Top-Level Functional Specification" document. This feels
2645	* like an awful waste of time.
2646	*
2647	* We simply cannot understand why HV_REGISTER_NAME isn't used directly here,
2648	* or at least the same values, making any conversion reduntant. Restricting
2649	* access to certain registers could easily be implement by scanning the
2650	* inputs.
2651	*
2652	* To avoid the heap + conversion overhead, we're currently using the
2653	* HvCallGetVpRegisters and HvCallSetVpRegisters calls directly.
2654	*
2655	*
2656	* - The YMM and XCR0 registers are not yet named (17083). This probably
2657	* wouldn't be a problem if HV_REGISTER_NAME was used, see previous point.
2658	*
2659	*
2660	* - Why does VID.SYS only query/set 32 registers at the time thru the
2661	* HvCallGetVpRegisters and HvCallSetVpRegisters hypercalls?
2662	*
2663	* We've not trouble getting/setting all the registers defined by
2664	* WHV_REGISTER_NAME in one hypercall (around 80). Some kind of stack
2665	* buffering or similar?
2666	*
2667	*
2668	* - To handle the VMMCALL / VMCALL instructions, it seems we need to intercept
2669	* \#UD exceptions and inspect the opcodes. A dedicated exit for hypercalls
2670	* would be more efficient, esp. for guests using \#UD for other purposes..
2671	*
2672	*
2673	* - Wrong instruction length in the VpContext with unmapped GPA memory exit
2674	* contexts on 17115/AMD.
2675	*
2676	* One byte "PUSH CS" was reported as 2 bytes, while a two byte
2677	* "MOV [EBX],EAX" was reported with a 1 byte instruction length. Problem
2678	* naturally present in untranslated hyper-v messages.
2679	*
2680	*
2681	* - The I/O port exit context information seems to be missing the address size
2682	* information needed for correct string I/O emulation.
2683	*
2684	* VT-x provides this information in bits 7:9 in the instruction information
2685	* field on newer CPUs. AMD-V in bits 7:9 in the EXITINFO1 field in the VMCB.
2686	*
2687	* We can probably work around this by scanning the instruction bytes for
2688	* address size prefixes. Haven't investigated it any further yet.
2689	*
2690	*
2691	* - Query WHvCapabilityCodeExceptionExitBitmap returns zero even when
2692	* intercepts demonstrably works (17134).
2693	*
2694	*
2695	* - The WHvGetCapability function has a weird design:
2696	* - The CapabilityCode parameter is pointlessly duplicated in the output
2697	* structure (WHV_CAPABILITY).
2698	*
2699	* - API takes void pointer, but everyone will probably be using
2700	* WHV_CAPABILITY due to WHV_CAPABILITY::CapabilityCode making it
2701	* impractical to use anything else.
2702	*
2703	* - No output size.
2704	*
2705	* - See GetFileAttributesEx, GetFileInformationByHandleEx,
2706	* FindFirstFileEx, and others for typical pattern for generic
2707	* information getters.
2708	*
2709	* Update: All concerns have been addressed in build 17110.
2710	*
2711	*
2712	* - The WHvGetPartitionProperty function uses the same weird design as
2713	* WHvGetCapability, see above.
2714	*
2715	* Update: All concerns have been addressed in build 17110.
2716	*
2717	*
2718	* - The WHvSetPartitionProperty function has a totally weird design too:
2719	* - In contrast to its partner WHvGetPartitionProperty, the property code
2720	* is not a separate input parameter here but part of the input
2721	* structure.
2722	*
2723	* - The input structure is a void pointer rather than a pointer to
2724	* WHV_PARTITION_PROPERTY which everyone probably will be using because
2725	* of the WHV_PARTITION_PROPERTY::PropertyCode field.
2726	*
2727	* - Really, why use PVOID for the input when the function isn't accepting
2728	* minimal sizes. E.g. WHVPartitionPropertyCodeProcessorClFlushSize only
2729	* requires a 9 byte input, but the function insists on 16 bytes (17083).
2730	*
2731	* - See GetFileAttributesEx, SetFileInformationByHandle, FindFirstFileEx,
2732	* and others for typical pattern for generic information setters and
2733	* getters.
2734	*
2735	* Update: All concerns have been addressed in build 17110.
2736	*
2737	*
2738	*
2739	* @section sec_nem_win_impl Our implementation.
2740	*
2741	* We set out with the goal of wanting to run as much as possible in ring-0,
2742	* reasoning that this would give use the best performance.
2743	*
2744	* This goal was approached gradually, starting out with a pure WinHvPlatform
2745	* implementation, gradually replacing parts: register access, guest memory
2746	* handling, running virtual processors. Then finally moving it all into
2747	* ring-0, while keeping most of it configurable so that we could make
2748	* comparisons (see NEMInternal.h and nemR3NativeRunGC()).
2749	*
2750	*
2751	* @subsection subsect_nem_win_impl_ioctl VID.SYS I/O control calls
2752	*
2753	* To run things in ring-0 we need to talk directly to VID.SYS thru its I/O
2754	* control interface. Looking at changes between like build 17083 and 17101 (if
2755	* memory serves) a set of the VID I/O control numbers shifted a little, which
2756	* means we need to determin them dynamically. We currently do this by hooking
2757	* the NtDeviceIoControlFile API call from VID.DLL and snooping up the
2758	* parameters when making dummy calls to relevant APIs. (We could also
2759	* disassemble the relevant APIs and try fish out the information from that, but
2760	* this is way simpler.)
2761	*
2762	* Issuing I/O control calls from ring-0 is facing a small challenge with
2763	* respect to direct buffering. When using direct buffering the device will
2764	* typically check that the buffer is actually in the user address space range
2765	* and reject kernel addresses. Fortunately, we've got the cross context VM
2766	* structure that is mapped into both kernel and user space, it's also locked
2767	* and safe to access from kernel space. So, we place the I/O control buffers
2768	* in the per-CPU part of it (NEMCPU::uIoCtlBuf) and give the driver the user
2769	* address if direct access buffering or kernel address if not.
2770	*
2771	* The I/O control calls are 'abstracted' in the support driver, see
2772	* SUPR0IoCtlSetupForHandle(), SUPR0IoCtlPerform() and SUPR0IoCtlCleanup().
2773	*
2774	*
2775	* @subsection subsect_nem_win_impl_cpumctx CPUMCTX
2776	*
2777	* Since the CPU state needs to live in Hyper-V when executing, we probably
2778	* should not transfer more than necessary when handling VMEXITs. To help us
2779	* manage this CPUMCTX got a new field CPUMCTX::fExtrn that to indicate which
2780	* part of the state is currently externalized (== in Hyper-V).
2781	*
2782	*
2783	*/
2784

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp@ 72526

Download in other formats: