VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h@ 72064

Last change on this file since 72064 was 71297, checked in by vboxsync, 7 years ago

NEM: Some more 17115 fixes and noted down an issue (possibly not at all new). bugref:9044
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 103.6 KB
Line 
1/* $Id: NEMAllNativeTemplate-win.cpp.h 71297 2018-03-10 06:02:02Z vboxsync $ */
2/** @file
3 * NEM - Native execution manager, Windows code template ring-0/3.
4 */
5
6/*
7 * Copyright (C) 2018 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Defined Constants And Macros *
21*********************************************************************************************************************************/
22/** Copy back a segment from hyper-V. */
23#define NEM_WIN_COPY_BACK_SEG(a_Dst, a_Src) \
24 do { \
25 (a_Dst).u64Base = (a_Src).Base; \
26 (a_Dst).u32Limit = (a_Src).Limit; \
27 (a_Dst).ValidSel = (a_Dst).Sel = (a_Src).Selector; \
28 (a_Dst).Attr.u = (a_Src).Attributes; \
29 (a_Dst).fFlags = CPUMSELREG_FLAGS_VALID; \
30 } while (0)
31
32
33/*********************************************************************************************************************************
34* Global Variables *
35*********************************************************************************************************************************/
36/** NEM_WIN_PAGE_STATE_XXX names. */
37NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
38
39/** HV_INTERCEPT_ACCESS_TYPE names. */
40static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" };
41
42
43/*********************************************************************************************************************************
44* Internal Functions *
45*********************************************************************************************************************************/
46NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
47 uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged);
48
49
50#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
51
52/**
53 * Wrapper around VMMR0_DO_NEM_MAP_PAGES for a single page.
54 *
55 * @returns VBox status code.
56 * @param pVM The cross context VM structure.
57 * @param pVCpu The cross context virtual CPU structure of the caller.
58 * @param GCPhysSrc The source page. Does not need to be page aligned.
59 * @param GCPhysDst The destination page. Same as @a GCPhysSrc except for
60 * when A20 is disabled.
61 * @param fFlags HV_MAP_GPA_XXX.
62 */
63DECLINLINE(int) nemHCWinHypercallMapPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fFlags)
64{
65#ifdef IN_RING0
66 /** @todo optimize further, caller generally has the physical address. */
67 PGVM pGVM = GVMMR0FastGetGVMByVM(pVM);
68 AssertReturn(pGVM, VERR_INVALID_VM_HANDLE);
69 return nemR0WinMapPages(pGVM, pVM, &pGVM->aCpus[pVCpu->idCpu],
70 GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
71 GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
72 1, fFlags);
73#else
74 pVCpu->nem.s.Hypercall.MapPages.GCPhysSrc = GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
75 pVCpu->nem.s.Hypercall.MapPages.GCPhysDst = GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
76 pVCpu->nem.s.Hypercall.MapPages.cPages = 1;
77 pVCpu->nem.s.Hypercall.MapPages.fFlags = fFlags;
78 return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_MAP_PAGES, 0, NULL);
79#endif
80}
81
82
83/**
84 * Wrapper around VMMR0_DO_NEM_UNMAP_PAGES for a single page.
85 *
86 * @returns VBox status code.
87 * @param pVM The cross context VM structure.
88 * @param pVCpu The cross context virtual CPU structure of the caller.
89 * @param GCPhys The page to unmap. Does not need to be page aligned.
90 */
91DECLINLINE(int) nemHCWinHypercallUnmapPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
92{
93# ifdef IN_RING0
94 PGVM pGVM = GVMMR0FastGetGVMByVM(pVM);
95 AssertReturn(pGVM, VERR_INVALID_VM_HANDLE);
96 return nemR0WinUnmapPages(pGVM, &pGVM->aCpus[pVCpu->idCpu], GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, 1);
97# else
98 pVCpu->nem.s.Hypercall.UnmapPages.GCPhys = GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
99 pVCpu->nem.s.Hypercall.UnmapPages.cPages = 1;
100 return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_UNMAP_PAGES, 0, NULL);
101# endif
102}
103
104#endif /* NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
105#ifndef IN_RING0
106
107NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
108{
109# ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
110 NOREF(pCtx);
111 int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPORT_STATE, 0, NULL);
112 AssertLogRelRCReturn(rc, rc);
113 return rc;
114
115# else
116 WHV_REGISTER_NAME aenmNames[128];
117 WHV_REGISTER_VALUE aValues[128];
118
119 /* GPRs */
120 aenmNames[0] = WHvX64RegisterRax;
121 aValues[0].Reg64 = pCtx->rax;
122 aenmNames[1] = WHvX64RegisterRcx;
123 aValues[1].Reg64 = pCtx->rcx;
124 aenmNames[2] = WHvX64RegisterRdx;
125 aValues[2].Reg64 = pCtx->rdx;
126 aenmNames[3] = WHvX64RegisterRbx;
127 aValues[3].Reg64 = pCtx->rbx;
128 aenmNames[4] = WHvX64RegisterRsp;
129 aValues[4].Reg64 = pCtx->rsp;
130 aenmNames[5] = WHvX64RegisterRbp;
131 aValues[5].Reg64 = pCtx->rbp;
132 aenmNames[6] = WHvX64RegisterRsi;
133 aValues[6].Reg64 = pCtx->rsi;
134 aenmNames[7] = WHvX64RegisterRdi;
135 aValues[7].Reg64 = pCtx->rdi;
136 aenmNames[8] = WHvX64RegisterR8;
137 aValues[8].Reg64 = pCtx->r8;
138 aenmNames[9] = WHvX64RegisterR9;
139 aValues[9].Reg64 = pCtx->r9;
140 aenmNames[10] = WHvX64RegisterR10;
141 aValues[10].Reg64 = pCtx->r10;
142 aenmNames[11] = WHvX64RegisterR11;
143 aValues[11].Reg64 = pCtx->r11;
144 aenmNames[12] = WHvX64RegisterR12;
145 aValues[12].Reg64 = pCtx->r12;
146 aenmNames[13] = WHvX64RegisterR13;
147 aValues[13].Reg64 = pCtx->r13;
148 aenmNames[14] = WHvX64RegisterR14;
149 aValues[14].Reg64 = pCtx->r14;
150 aenmNames[15] = WHvX64RegisterR15;
151 aValues[15].Reg64 = pCtx->r15;
152
153 /* RIP & Flags */
154 aenmNames[16] = WHvX64RegisterRip;
155 aValues[16].Reg64 = pCtx->rip;
156 aenmNames[17] = WHvX64RegisterRflags;
157 aValues[17].Reg64 = pCtx->rflags.u;
158
159 /* Segments */
160# define COPY_OUT_SEG(a_idx, a_enmName, a_SReg) \
161 do { \
162 aenmNames[a_idx] = a_enmName; \
163 aValues[a_idx].Segment.Base = (a_SReg).u64Base; \
164 aValues[a_idx].Segment.Limit = (a_SReg).u32Limit; \
165 aValues[a_idx].Segment.Selector = (a_SReg).Sel; \
166 aValues[a_idx].Segment.Attributes = (a_SReg).Attr.u; \
167 } while (0)
168 COPY_OUT_SEG(18, WHvX64RegisterEs, pCtx->es);
169 COPY_OUT_SEG(19, WHvX64RegisterCs, pCtx->cs);
170 COPY_OUT_SEG(20, WHvX64RegisterSs, pCtx->ss);
171 COPY_OUT_SEG(21, WHvX64RegisterDs, pCtx->ds);
172 COPY_OUT_SEG(22, WHvX64RegisterFs, pCtx->fs);
173 COPY_OUT_SEG(23, WHvX64RegisterGs, pCtx->gs);
174 COPY_OUT_SEG(24, WHvX64RegisterLdtr, pCtx->ldtr);
175 COPY_OUT_SEG(25, WHvX64RegisterTr, pCtx->tr);
176
177 uintptr_t iReg = 26;
178 /* Descriptor tables. */
179 aenmNames[iReg] = WHvX64RegisterIdtr;
180 aValues[iReg].Table.Limit = pCtx->idtr.cbIdt;
181 aValues[iReg].Table.Base = pCtx->idtr.pIdt;
182 iReg++;
183 aenmNames[iReg] = WHvX64RegisterGdtr;
184 aValues[iReg].Table.Limit = pCtx->gdtr.cbGdt;
185 aValues[iReg].Table.Base = pCtx->gdtr.pGdt;
186 iReg++;
187
188 /* Control registers. */
189 aenmNames[iReg] = WHvX64RegisterCr0;
190 aValues[iReg].Reg64 = pCtx->cr0;
191 iReg++;
192 aenmNames[iReg] = WHvX64RegisterCr2;
193 aValues[iReg].Reg64 = pCtx->cr2;
194 iReg++;
195 aenmNames[iReg] = WHvX64RegisterCr3;
196 aValues[iReg].Reg64 = pCtx->cr3;
197 iReg++;
198 aenmNames[iReg] = WHvX64RegisterCr4;
199 aValues[iReg].Reg64 = pCtx->cr4;
200 iReg++;
201 aenmNames[iReg] = WHvX64RegisterCr8;
202 aValues[iReg].Reg64 = CPUMGetGuestCR8(pVCpu);
203 iReg++;
204
205 /* Debug registers. */
206/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
207 aenmNames[iReg] = WHvX64RegisterDr0;
208 //aValues[iReg].Reg64 = CPUMGetHyperDR0(pVCpu);
209 aValues[iReg].Reg64 = pCtx->dr[0];
210 iReg++;
211 aenmNames[iReg] = WHvX64RegisterDr1;
212 //aValues[iReg].Reg64 = CPUMGetHyperDR1(pVCpu);
213 aValues[iReg].Reg64 = pCtx->dr[1];
214 iReg++;
215 aenmNames[iReg] = WHvX64RegisterDr2;
216 //aValues[iReg].Reg64 = CPUMGetHyperDR2(pVCpu);
217 aValues[iReg].Reg64 = pCtx->dr[2];
218 iReg++;
219 aenmNames[iReg] = WHvX64RegisterDr3;
220 //aValues[iReg].Reg64 = CPUMGetHyperDR3(pVCpu);
221 aValues[iReg].Reg64 = pCtx->dr[3];
222 iReg++;
223 aenmNames[iReg] = WHvX64RegisterDr6;
224 //aValues[iReg].Reg64 = CPUMGetHyperDR6(pVCpu);
225 aValues[iReg].Reg64 = pCtx->dr[6];
226 iReg++;
227 aenmNames[iReg] = WHvX64RegisterDr7;
228 //aValues[iReg].Reg64 = CPUMGetHyperDR7(pVCpu);
229 aValues[iReg].Reg64 = pCtx->dr[7];
230 iReg++;
231
232 /* Vector state. */
233 aenmNames[iReg] = WHvX64RegisterXmm0;
234 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Lo;
235 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Hi;
236 iReg++;
237 aenmNames[iReg] = WHvX64RegisterXmm1;
238 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Lo;
239 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Hi;
240 iReg++;
241 aenmNames[iReg] = WHvX64RegisterXmm2;
242 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Lo;
243 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Hi;
244 iReg++;
245 aenmNames[iReg] = WHvX64RegisterXmm3;
246 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Lo;
247 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Hi;
248 iReg++;
249 aenmNames[iReg] = WHvX64RegisterXmm4;
250 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Lo;
251 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Hi;
252 iReg++;
253 aenmNames[iReg] = WHvX64RegisterXmm5;
254 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Lo;
255 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Hi;
256 iReg++;
257 aenmNames[iReg] = WHvX64RegisterXmm6;
258 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Lo;
259 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Hi;
260 iReg++;
261 aenmNames[iReg] = WHvX64RegisterXmm7;
262 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Lo;
263 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Hi;
264 iReg++;
265 aenmNames[iReg] = WHvX64RegisterXmm8;
266 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Lo;
267 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Hi;
268 iReg++;
269 aenmNames[iReg] = WHvX64RegisterXmm9;
270 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Lo;
271 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Hi;
272 iReg++;
273 aenmNames[iReg] = WHvX64RegisterXmm10;
274 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Lo;
275 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Hi;
276 iReg++;
277 aenmNames[iReg] = WHvX64RegisterXmm11;
278 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Lo;
279 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Hi;
280 iReg++;
281 aenmNames[iReg] = WHvX64RegisterXmm12;
282 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Lo;
283 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Hi;
284 iReg++;
285 aenmNames[iReg] = WHvX64RegisterXmm13;
286 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Lo;
287 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Hi;
288 iReg++;
289 aenmNames[iReg] = WHvX64RegisterXmm14;
290 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Lo;
291 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Hi;
292 iReg++;
293 aenmNames[iReg] = WHvX64RegisterXmm15;
294 aValues[iReg].Reg128.Low64 = pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Lo;
295 aValues[iReg].Reg128.High64 = pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Hi;
296 iReg++;
297
298 /* Floating point state. */
299 aenmNames[iReg] = WHvX64RegisterFpMmx0;
300 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[0].au64[0];
301 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[0].au64[1];
302 iReg++;
303 aenmNames[iReg] = WHvX64RegisterFpMmx1;
304 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[1].au64[0];
305 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[1].au64[1];
306 iReg++;
307 aenmNames[iReg] = WHvX64RegisterFpMmx2;
308 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[2].au64[0];
309 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[2].au64[1];
310 iReg++;
311 aenmNames[iReg] = WHvX64RegisterFpMmx3;
312 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[3].au64[0];
313 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[3].au64[1];
314 iReg++;
315 aenmNames[iReg] = WHvX64RegisterFpMmx4;
316 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[4].au64[0];
317 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[4].au64[1];
318 iReg++;
319 aenmNames[iReg] = WHvX64RegisterFpMmx5;
320 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[5].au64[0];
321 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[5].au64[1];
322 iReg++;
323 aenmNames[iReg] = WHvX64RegisterFpMmx6;
324 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[6].au64[0];
325 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[6].au64[1];
326 iReg++;
327 aenmNames[iReg] = WHvX64RegisterFpMmx7;
328 aValues[iReg].Fp.AsUINT128.Low64 = pCtx->pXStateR3->x87.aRegs[7].au64[0];
329 aValues[iReg].Fp.AsUINT128.High64 = pCtx->pXStateR3->x87.aRegs[7].au64[1];
330 iReg++;
331
332 aenmNames[iReg] = WHvX64RegisterFpControlStatus;
333 aValues[iReg].FpControlStatus.FpControl = pCtx->pXStateR3->x87.FCW;
334 aValues[iReg].FpControlStatus.FpStatus = pCtx->pXStateR3->x87.FSW;
335 aValues[iReg].FpControlStatus.FpTag = pCtx->pXStateR3->x87.FTW;
336 aValues[iReg].FpControlStatus.Reserved = pCtx->pXStateR3->x87.FTW >> 8;
337 aValues[iReg].FpControlStatus.LastFpOp = pCtx->pXStateR3->x87.FOP;
338 aValues[iReg].FpControlStatus.LastFpRip = (pCtx->pXStateR3->x87.FPUIP)
339 | ((uint64_t)pCtx->pXStateR3->x87.CS << 32)
340 | ((uint64_t)pCtx->pXStateR3->x87.Rsrvd1 << 48);
341 iReg++;
342
343 aenmNames[iReg] = WHvX64RegisterXmmControlStatus;
344 aValues[iReg].XmmControlStatus.LastFpRdp = (pCtx->pXStateR3->x87.FPUDP)
345 | ((uint64_t)pCtx->pXStateR3->x87.DS << 32)
346 | ((uint64_t)pCtx->pXStateR3->x87.Rsrvd2 << 48);
347 aValues[iReg].XmmControlStatus.XmmStatusControl = pCtx->pXStateR3->x87.MXCSR;
348 aValues[iReg].XmmControlStatus.XmmStatusControlMask = pCtx->pXStateR3->x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
349 iReg++;
350
351 /* MSRs */
352 // WHvX64RegisterTsc - don't touch
353 aenmNames[iReg] = WHvX64RegisterEfer;
354 aValues[iReg].Reg64 = pCtx->msrEFER;
355 iReg++;
356 aenmNames[iReg] = WHvX64RegisterKernelGsBase;
357 aValues[iReg].Reg64 = pCtx->msrKERNELGSBASE;
358 iReg++;
359 aenmNames[iReg] = WHvX64RegisterApicBase;
360 aValues[iReg].Reg64 = APICGetBaseMsrNoCheck(pVCpu);
361 iReg++;
362 aenmNames[iReg] = WHvX64RegisterPat;
363 aValues[iReg].Reg64 = pCtx->msrPAT;
364 iReg++;
365 /// @todo WHvX64RegisterSysenterCs
366 /// @todo WHvX64RegisterSysenterEip
367 /// @todo WHvX64RegisterSysenterEsp
368 aenmNames[iReg] = WHvX64RegisterStar;
369 aValues[iReg].Reg64 = pCtx->msrSTAR;
370 iReg++;
371 aenmNames[iReg] = WHvX64RegisterLstar;
372 aValues[iReg].Reg64 = pCtx->msrLSTAR;
373 iReg++;
374 aenmNames[iReg] = WHvX64RegisterCstar;
375 aValues[iReg].Reg64 = pCtx->msrCSTAR;
376 iReg++;
377 aenmNames[iReg] = WHvX64RegisterSfmask;
378 aValues[iReg].Reg64 = pCtx->msrSFMASK;
379 iReg++;
380
381 /* event injection (always clear it). */
382 aenmNames[iReg] = WHvRegisterPendingInterruption;
383 aValues[iReg].Reg64 = 0;
384 iReg++;
385 /// @todo WHvRegisterInterruptState
386 /// @todo WHvRegisterPendingEvent0
387 /// @todo WHvRegisterPendingEvent1
388
389 /*
390 * Set the registers.
391 */
392 Assert(iReg < RT_ELEMENTS(aValues));
393 Assert(iReg < RT_ELEMENTS(aenmNames));
394# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
395 Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
396 pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
397# endif
398 HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
399 if (SUCCEEDED(hrc))
400 return VINF_SUCCESS;
401 AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
402 pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
403 hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
404 return VERR_INTERNAL_ERROR;
405# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
406}
407
408
409NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint64_t fWhat)
410{
411# ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
412 /* See NEMR0ImportState */
413 NOREF(pCtx);
414 int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_IMPORT_STATE, fWhat, NULL);
415 if (RT_SUCCESS(rc))
416 return rc;
417 if (rc == VERR_NEM_FLUSH_TLB)
418 return PGMFlushTLB(pVCpu, pCtx->cr3, true /*fGlobal*/);
419 if (rc == VERR_NEM_CHANGE_PGM_MODE)
420 return PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
421 AssertLogRelRCReturn(rc, rc);
422 return rc;
423
424# else
425 WHV_REGISTER_NAME aenmNames[128];
426
427 /* GPRs */
428 aenmNames[0] = WHvX64RegisterRax;
429 aenmNames[1] = WHvX64RegisterRcx;
430 aenmNames[2] = WHvX64RegisterRdx;
431 aenmNames[3] = WHvX64RegisterRbx;
432 aenmNames[4] = WHvX64RegisterRsp;
433 aenmNames[5] = WHvX64RegisterRbp;
434 aenmNames[6] = WHvX64RegisterRsi;
435 aenmNames[7] = WHvX64RegisterRdi;
436 aenmNames[8] = WHvX64RegisterR8;
437 aenmNames[9] = WHvX64RegisterR9;
438 aenmNames[10] = WHvX64RegisterR10;
439 aenmNames[11] = WHvX64RegisterR11;
440 aenmNames[12] = WHvX64RegisterR12;
441 aenmNames[13] = WHvX64RegisterR13;
442 aenmNames[14] = WHvX64RegisterR14;
443 aenmNames[15] = WHvX64RegisterR15;
444
445 /* RIP & Flags */
446 aenmNames[16] = WHvX64RegisterRip;
447 aenmNames[17] = WHvX64RegisterRflags;
448
449 /* Segments */
450 aenmNames[18] = WHvX64RegisterEs;
451 aenmNames[19] = WHvX64RegisterCs;
452 aenmNames[20] = WHvX64RegisterSs;
453 aenmNames[21] = WHvX64RegisterDs;
454 aenmNames[22] = WHvX64RegisterFs;
455 aenmNames[23] = WHvX64RegisterGs;
456 aenmNames[24] = WHvX64RegisterLdtr;
457 aenmNames[25] = WHvX64RegisterTr;
458
459 /* Descriptor tables. */
460 aenmNames[26] = WHvX64RegisterIdtr;
461 aenmNames[27] = WHvX64RegisterGdtr;
462
463 /* Control registers. */
464 aenmNames[28] = WHvX64RegisterCr0;
465 aenmNames[29] = WHvX64RegisterCr2;
466 aenmNames[30] = WHvX64RegisterCr3;
467 aenmNames[31] = WHvX64RegisterCr4;
468 aenmNames[32] = WHvX64RegisterCr8;
469
470 /* Debug registers. */
471 aenmNames[33] = WHvX64RegisterDr0;
472 aenmNames[34] = WHvX64RegisterDr1;
473 aenmNames[35] = WHvX64RegisterDr2;
474 aenmNames[36] = WHvX64RegisterDr3;
475 aenmNames[37] = WHvX64RegisterDr6;
476 aenmNames[38] = WHvX64RegisterDr7;
477
478 /* Vector state. */
479 aenmNames[39] = WHvX64RegisterXmm0;
480 aenmNames[40] = WHvX64RegisterXmm1;
481 aenmNames[41] = WHvX64RegisterXmm2;
482 aenmNames[42] = WHvX64RegisterXmm3;
483 aenmNames[43] = WHvX64RegisterXmm4;
484 aenmNames[44] = WHvX64RegisterXmm5;
485 aenmNames[45] = WHvX64RegisterXmm6;
486 aenmNames[46] = WHvX64RegisterXmm7;
487 aenmNames[47] = WHvX64RegisterXmm8;
488 aenmNames[48] = WHvX64RegisterXmm9;
489 aenmNames[49] = WHvX64RegisterXmm10;
490 aenmNames[50] = WHvX64RegisterXmm11;
491 aenmNames[51] = WHvX64RegisterXmm12;
492 aenmNames[52] = WHvX64RegisterXmm13;
493 aenmNames[53] = WHvX64RegisterXmm14;
494 aenmNames[54] = WHvX64RegisterXmm15;
495
496 /* Floating point state. */
497 aenmNames[55] = WHvX64RegisterFpMmx0;
498 aenmNames[56] = WHvX64RegisterFpMmx1;
499 aenmNames[57] = WHvX64RegisterFpMmx2;
500 aenmNames[58] = WHvX64RegisterFpMmx3;
501 aenmNames[59] = WHvX64RegisterFpMmx4;
502 aenmNames[60] = WHvX64RegisterFpMmx5;
503 aenmNames[61] = WHvX64RegisterFpMmx6;
504 aenmNames[62] = WHvX64RegisterFpMmx7;
505 aenmNames[63] = WHvX64RegisterFpControlStatus;
506 aenmNames[64] = WHvX64RegisterXmmControlStatus;
507
508 /* MSRs */
509 // WHvX64RegisterTsc - don't touch
510 aenmNames[65] = WHvX64RegisterEfer;
511 aenmNames[66] = WHvX64RegisterKernelGsBase;
512 aenmNames[67] = WHvX64RegisterApicBase;
513 aenmNames[68] = WHvX64RegisterPat;
514 aenmNames[69] = WHvX64RegisterSysenterCs;
515 aenmNames[70] = WHvX64RegisterSysenterEip;
516 aenmNames[71] = WHvX64RegisterSysenterEsp;
517 aenmNames[72] = WHvX64RegisterStar;
518 aenmNames[73] = WHvX64RegisterLstar;
519 aenmNames[74] = WHvX64RegisterCstar;
520 aenmNames[75] = WHvX64RegisterSfmask;
521
522 /* event injection */
523 aenmNames[76] = WHvRegisterPendingInterruption;
524 aenmNames[77] = WHvRegisterInterruptState;
525 aenmNames[78] = WHvRegisterInterruptState;
526 aenmNames[79] = WHvRegisterPendingEvent0;
527 aenmNames[80] = WHvRegisterPendingEvent1;
528 unsigned const cRegs = 81;
529
530 /*
531 * Get the registers.
532 */
533 WHV_REGISTER_VALUE aValues[cRegs];
534 RT_ZERO(aValues);
535 Assert(RT_ELEMENTS(aValues) >= cRegs);
536 Assert(RT_ELEMENTS(aenmNames) >= cRegs);
537# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
538 Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
539 pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
540# endif
541 HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues);
542 if (SUCCEEDED(hrc))
543 {
544 /* GPRs */
545 Assert(aenmNames[0] == WHvX64RegisterRax);
546 Assert(aenmNames[15] == WHvX64RegisterR15);
547 pCtx->rax = aValues[0].Reg64;
548 pCtx->rcx = aValues[1].Reg64;
549 pCtx->rdx = aValues[2].Reg64;
550 pCtx->rbx = aValues[3].Reg64;
551 pCtx->rsp = aValues[4].Reg64;
552 pCtx->rbp = aValues[5].Reg64;
553 pCtx->rsi = aValues[6].Reg64;
554 pCtx->rdi = aValues[7].Reg64;
555 pCtx->r8 = aValues[8].Reg64;
556 pCtx->r9 = aValues[9].Reg64;
557 pCtx->r10 = aValues[10].Reg64;
558 pCtx->r11 = aValues[11].Reg64;
559 pCtx->r12 = aValues[12].Reg64;
560 pCtx->r13 = aValues[13].Reg64;
561 pCtx->r14 = aValues[14].Reg64;
562 pCtx->r15 = aValues[15].Reg64;
563
564 /* RIP & Flags */
565 Assert(aenmNames[16] == WHvX64RegisterRip);
566 pCtx->rip = aValues[16].Reg64;
567 pCtx->rflags.u = aValues[17].Reg64;
568
569 /* Segments */
570# define COPY_BACK_SEG(a_idx, a_enmName, a_SReg) \
571 do { \
572 Assert(aenmNames[a_idx] == a_enmName); \
573 NEM_WIN_COPY_BACK_SEG(a_SReg, aValues[a_idx]); \
574 } while (0)
575 COPY_BACK_SEG(18, WHvX64RegisterEs, pCtx->es);
576 COPY_BACK_SEG(19, WHvX64RegisterCs, pCtx->cs);
577 COPY_BACK_SEG(20, WHvX64RegisterSs, pCtx->ss);
578 COPY_BACK_SEG(21, WHvX64RegisterDs, pCtx->ds);
579 COPY_BACK_SEG(22, WHvX64RegisterFs, pCtx->fs);
580 COPY_BACK_SEG(23, WHvX64RegisterGs, pCtx->gs);
581 COPY_BACK_SEG(24, WHvX64RegisterLdtr, pCtx->ldtr);
582 /* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
583 avoid to trigger sanity assertions around the code, always fix this. */
584 COPY_BACK_SEG(25, WHvX64RegisterTr, pCtx->tr);
585 switch (pCtx->tr.Attr.n.u4Type)
586 {
587 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
588 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
589 break;
590 case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
591 pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
592 break;
593 case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
594 pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
595 break;
596 }
597
598 /* Descriptor tables. */
599 Assert(aenmNames[26] == WHvX64RegisterIdtr);
600 pCtx->idtr.cbIdt = aValues[26].Table.Limit;
601 pCtx->idtr.pIdt = aValues[26].Table.Base;
602 Assert(aenmNames[27] == WHvX64RegisterGdtr);
603 pCtx->gdtr.cbGdt = aValues[27].Table.Limit;
604 pCtx->gdtr.pGdt = aValues[27].Table.Base;
605
606 /* Control registers. */
607 Assert(aenmNames[28] == WHvX64RegisterCr0);
608 bool fMaybeChangedMode = false;
609 bool fFlushTlb = false;
610 bool fFlushGlobalTlb = false;
611 if (pCtx->cr0 != aValues[28].Reg64)
612 {
613 CPUMSetGuestCR0(pVCpu, aValues[28].Reg64);
614 fMaybeChangedMode = true;
615 fFlushTlb = fFlushGlobalTlb = true; /// @todo fix this
616 }
617 Assert(aenmNames[29] == WHvX64RegisterCr2);
618 pCtx->cr2 = aValues[29].Reg64;
619 if (pCtx->cr3 != aValues[30].Reg64)
620 {
621 CPUMSetGuestCR3(pVCpu, aValues[30].Reg64);
622 fFlushTlb = true;
623 }
624 if (pCtx->cr4 != aValues[31].Reg64)
625 {
626 CPUMSetGuestCR4(pVCpu, aValues[31].Reg64);
627 fMaybeChangedMode = true;
628 fFlushTlb = fFlushGlobalTlb = true; /// @todo fix this
629 }
630 APICSetTpr(pVCpu, (uint8_t)aValues[32].Reg64 << 4);
631
632 /* Debug registers. */
633 Assert(aenmNames[33] == WHvX64RegisterDr0);
634 /** @todo fixme */
635 if (pCtx->dr[0] != aValues[33].Reg64)
636 CPUMSetGuestDR0(pVCpu, aValues[33].Reg64);
637 if (pCtx->dr[1] != aValues[34].Reg64)
638 CPUMSetGuestDR1(pVCpu, aValues[34].Reg64);
639 if (pCtx->dr[2] != aValues[35].Reg64)
640 CPUMSetGuestDR2(pVCpu, aValues[35].Reg64);
641 if (pCtx->dr[3] != aValues[36].Reg64)
642 CPUMSetGuestDR3(pVCpu, aValues[36].Reg64);
643 Assert(aenmNames[37] == WHvX64RegisterDr6);
644 Assert(aenmNames[38] == WHvX64RegisterDr7);
645 if (pCtx->dr[6] != aValues[37].Reg64)
646 CPUMSetGuestDR6(pVCpu, aValues[37].Reg64);
647 if (pCtx->dr[7] != aValues[38].Reg64)
648 CPUMSetGuestDR6(pVCpu, aValues[38].Reg64);
649
650 /* Vector state. */
651 Assert(aenmNames[39] == WHvX64RegisterXmm0);
652 Assert(aenmNames[54] == WHvX64RegisterXmm15);
653 pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Lo = aValues[39].Reg128.Low64;
654 pCtx->pXStateR3->x87.aXMM[0].uXmm.s.Hi = aValues[39].Reg128.High64;
655 pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Lo = aValues[40].Reg128.Low64;
656 pCtx->pXStateR3->x87.aXMM[1].uXmm.s.Hi = aValues[40].Reg128.High64;
657 pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Lo = aValues[41].Reg128.Low64;
658 pCtx->pXStateR3->x87.aXMM[2].uXmm.s.Hi = aValues[41].Reg128.High64;
659 pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Lo = aValues[42].Reg128.Low64;
660 pCtx->pXStateR3->x87.aXMM[3].uXmm.s.Hi = aValues[42].Reg128.High64;
661 pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Lo = aValues[43].Reg128.Low64;
662 pCtx->pXStateR3->x87.aXMM[4].uXmm.s.Hi = aValues[43].Reg128.High64;
663 pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Lo = aValues[44].Reg128.Low64;
664 pCtx->pXStateR3->x87.aXMM[5].uXmm.s.Hi = aValues[44].Reg128.High64;
665 pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Lo = aValues[45].Reg128.Low64;
666 pCtx->pXStateR3->x87.aXMM[6].uXmm.s.Hi = aValues[45].Reg128.High64;
667 pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Lo = aValues[46].Reg128.Low64;
668 pCtx->pXStateR3->x87.aXMM[7].uXmm.s.Hi = aValues[46].Reg128.High64;
669 pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Lo = aValues[47].Reg128.Low64;
670 pCtx->pXStateR3->x87.aXMM[8].uXmm.s.Hi = aValues[47].Reg128.High64;
671 pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Lo = aValues[48].Reg128.Low64;
672 pCtx->pXStateR3->x87.aXMM[9].uXmm.s.Hi = aValues[48].Reg128.High64;
673 pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Lo = aValues[49].Reg128.Low64;
674 pCtx->pXStateR3->x87.aXMM[10].uXmm.s.Hi = aValues[49].Reg128.High64;
675 pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Lo = aValues[50].Reg128.Low64;
676 pCtx->pXStateR3->x87.aXMM[11].uXmm.s.Hi = aValues[50].Reg128.High64;
677 pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Lo = aValues[51].Reg128.Low64;
678 pCtx->pXStateR3->x87.aXMM[12].uXmm.s.Hi = aValues[51].Reg128.High64;
679 pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Lo = aValues[52].Reg128.Low64;
680 pCtx->pXStateR3->x87.aXMM[13].uXmm.s.Hi = aValues[52].Reg128.High64;
681 pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Lo = aValues[53].Reg128.Low64;
682 pCtx->pXStateR3->x87.aXMM[14].uXmm.s.Hi = aValues[53].Reg128.High64;
683 pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Lo = aValues[54].Reg128.Low64;
684 pCtx->pXStateR3->x87.aXMM[15].uXmm.s.Hi = aValues[54].Reg128.High64;
685
686 /* Floating point state. */
687 Assert(aenmNames[55] == WHvX64RegisterFpMmx0);
688 Assert(aenmNames[62] == WHvX64RegisterFpMmx7);
689 pCtx->pXStateR3->x87.aRegs[0].au64[0] = aValues[55].Fp.AsUINT128.Low64;
690 pCtx->pXStateR3->x87.aRegs[0].au64[1] = aValues[55].Fp.AsUINT128.High64;
691 pCtx->pXStateR3->x87.aRegs[1].au64[0] = aValues[56].Fp.AsUINT128.Low64;
692 pCtx->pXStateR3->x87.aRegs[1].au64[1] = aValues[56].Fp.AsUINT128.High64;
693 pCtx->pXStateR3->x87.aRegs[2].au64[0] = aValues[57].Fp.AsUINT128.Low64;
694 pCtx->pXStateR3->x87.aRegs[2].au64[1] = aValues[57].Fp.AsUINT128.High64;
695 pCtx->pXStateR3->x87.aRegs[3].au64[0] = aValues[58].Fp.AsUINT128.Low64;
696 pCtx->pXStateR3->x87.aRegs[3].au64[1] = aValues[58].Fp.AsUINT128.High64;
697 pCtx->pXStateR3->x87.aRegs[4].au64[0] = aValues[59].Fp.AsUINT128.Low64;
698 pCtx->pXStateR3->x87.aRegs[4].au64[1] = aValues[59].Fp.AsUINT128.High64;
699 pCtx->pXStateR3->x87.aRegs[5].au64[0] = aValues[60].Fp.AsUINT128.Low64;
700 pCtx->pXStateR3->x87.aRegs[5].au64[1] = aValues[60].Fp.AsUINT128.High64;
701 pCtx->pXStateR3->x87.aRegs[6].au64[0] = aValues[61].Fp.AsUINT128.Low64;
702 pCtx->pXStateR3->x87.aRegs[6].au64[1] = aValues[61].Fp.AsUINT128.High64;
703 pCtx->pXStateR3->x87.aRegs[7].au64[0] = aValues[62].Fp.AsUINT128.Low64;
704 pCtx->pXStateR3->x87.aRegs[7].au64[1] = aValues[62].Fp.AsUINT128.High64;
705
706 Assert(aenmNames[63] == WHvX64RegisterFpControlStatus);
707 pCtx->pXStateR3->x87.FCW = aValues[63].FpControlStatus.FpControl;
708 pCtx->pXStateR3->x87.FSW = aValues[63].FpControlStatus.FpStatus;
709 pCtx->pXStateR3->x87.FTW = aValues[63].FpControlStatus.FpTag
710 /*| (aValues[63].FpControlStatus.Reserved << 8)*/;
711 pCtx->pXStateR3->x87.FOP = aValues[63].FpControlStatus.LastFpOp;
712 pCtx->pXStateR3->x87.FPUIP = (uint32_t)aValues[63].FpControlStatus.LastFpRip;
713 pCtx->pXStateR3->x87.CS = (uint16_t)(aValues[63].FpControlStatus.LastFpRip >> 32);
714 pCtx->pXStateR3->x87.Rsrvd1 = (uint16_t)(aValues[63].FpControlStatus.LastFpRip >> 48);
715
716 Assert(aenmNames[64] == WHvX64RegisterXmmControlStatus);
717 pCtx->pXStateR3->x87.FPUDP = (uint32_t)aValues[64].XmmControlStatus.LastFpRdp;
718 pCtx->pXStateR3->x87.DS = (uint16_t)(aValues[64].XmmControlStatus.LastFpRdp >> 32);
719 pCtx->pXStateR3->x87.Rsrvd2 = (uint16_t)(aValues[64].XmmControlStatus.LastFpRdp >> 48);
720 pCtx->pXStateR3->x87.MXCSR = aValues[64].XmmControlStatus.XmmStatusControl;
721 pCtx->pXStateR3->x87.MXCSR_MASK = aValues[64].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
722
723 /* MSRs */
724 // WHvX64RegisterTsc - don't touch
725 Assert(aenmNames[65] == WHvX64RegisterEfer);
726 if (aValues[65].Reg64 != pCtx->msrEFER)
727 {
728 pCtx->msrEFER = aValues[65].Reg64;
729 fMaybeChangedMode = true;
730 }
731
732 Assert(aenmNames[66] == WHvX64RegisterKernelGsBase);
733 pCtx->msrKERNELGSBASE = aValues[66].Reg64;
734
735 Assert(aenmNames[67] == WHvX64RegisterApicBase);
736 if (aValues[67].Reg64 != APICGetBaseMsrNoCheck(pVCpu))
737 {
738 VBOXSTRICTRC rc2 = APICSetBaseMsr(pVCpu, aValues[67].Reg64);
739 Assert(rc2 == VINF_SUCCESS); NOREF(rc2);
740 }
741
742 Assert(aenmNames[68] == WHvX64RegisterPat);
743 pCtx->msrPAT = aValues[68].Reg64;
744 /// @todo WHvX64RegisterSysenterCs
745 /// @todo WHvX64RegisterSysenterEip
746 /// @todo WHvX64RegisterSysenterEsp
747 Assert(aenmNames[72] == WHvX64RegisterStar);
748 pCtx->msrSTAR = aValues[72].Reg64;
749 Assert(aenmNames[73] == WHvX64RegisterLstar);
750 pCtx->msrLSTAR = aValues[73].Reg64;
751 Assert(aenmNames[74] == WHvX64RegisterCstar);
752 pCtx->msrCSTAR = aValues[74].Reg64;
753 Assert(aenmNames[75] == WHvX64RegisterSfmask);
754 pCtx->msrSFMASK = aValues[75].Reg64;
755
756 /// @todo WHvRegisterPendingInterruption
757 Assert(aenmNames[76] == WHvRegisterPendingInterruption);
758 WHV_X64_PENDING_INTERRUPTION_REGISTER const * pPendingInt = (WHV_X64_PENDING_INTERRUPTION_REGISTER const *)&aValues[76];
759 if (pPendingInt->InterruptionPending)
760 {
761 Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
762 pPendingInt->InterruptionType, pPendingInt->InterruptionVector, pPendingInt->DeliverErrorCode,
763 pPendingInt->ErrorCode, pPendingInt->InstructionLength, pPendingInt->NestedEvent));
764 AssertMsg((pPendingInt->AsUINT64 & UINT64_C(0xfc00)) == 0, ("%#RX64\n", pPendingInt->AsUINT64));
765 }
766
767 /// @todo WHvRegisterInterruptState
768 /// @todo WHvRegisterPendingEvent0
769 /// @todo WHvRegisterPendingEvent1
770
771 pCtx->fExtrn = 0;
772
773 if (fMaybeChangedMode)
774 {
775 int rc = PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
776 AssertRC(rc);
777 }
778 if (fFlushTlb)
779 {
780 int rc = PGMFlushTLB(pVCpu, pCtx->cr3, fFlushGlobalTlb);
781 AssertRC(rc);
782 }
783
784 return VINF_SUCCESS;
785 }
786
787 AssertLogRelMsgFailed(("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
788 pVM->nem.s.hPartition, pVCpu->idCpu, cRegs,
789 hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
790 return VERR_INTERNAL_ERROR;
791# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
792}
793
794#endif /* !IN_RING0 */
795
796
797#ifdef LOG_ENABLED
798/**
799 * Get the virtual processor running status.
800 */
801DECLINLINE(VID_PROCESSOR_STATUS) nemHCWinCpuGetRunningStatus(PVMCPU pVCpu)
802{
803# ifdef IN_RING0
804 NOREF(pVCpu);
805 return VidProcessorStatusUndefined;
806# else
807 RTERRVARS Saved;
808 RTErrVarsSave(&Saved);
809
810 /*
811 * This API is disabled in release builds, it seems. On build 17101 it requires
812 * the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00
813 */
814 VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined;
815 NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus);
816 AssertRC(rcNt);
817
818 RTErrVarsRestore(&Saved);
819 return enmCpuStatus;
820# endif
821}
822#endif
823
824
825#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
826# ifdef IN_RING3 /* hopefully not needed in ring-0, as we'd need KTHREADs and KeAlertThread. */
827/**
828 * Our own WHvCancelRunVirtualProcessor that can later be moved to ring-0.
829 *
830 * This is an experiment only.
831 *
832 * @returns VBox status code.
833 * @param pVM The cross context VM structure.
834 * @param pVCpu The cross context virtual CPU structure of the
835 * calling EMT.
836 */
837NEM_TMPL_STATIC int nemHCWinCancelRunVirtualProcessor(PVM pVM, PVMCPU pVCpu)
838{
839 /*
840 * Work the state.
841 *
842 * From the looks of things, we should let the EMT call VidStopVirtualProcessor.
843 * So, we just need to modify the state and kick the EMT if it's waiting on
844 * messages. For the latter we use QueueUserAPC / KeAlterThread.
845 */
846 for (;;)
847 {
848 VMCPUSTATE enmState = VMCPU_GET_STATE(pVCpu);
849 switch (enmState)
850 {
851 case VMCPUSTATE_STARTED_EXEC_NEM:
852 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM))
853 {
854 Log8(("nemHCWinCancelRunVirtualProcessor: Switched %u to canceled state\n", pVCpu->idCpu));
855 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelChangedState);
856 return VINF_SUCCESS;
857 }
858 break;
859
860 case VMCPUSTATE_STARTED_EXEC_NEM_WAIT:
861 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT))
862 {
863# ifdef IN_RING0
864 NTSTATUS rcNt = KeAlertThread(??);
865# else
866 NTSTATUS rcNt = NtAlertThread(pVCpu->nem.s.hNativeThreadHandle);
867# endif
868 Log8(("nemHCWinCancelRunVirtualProcessor: Alerted %u: %#x\n", pVCpu->idCpu, rcNt));
869 Assert(rcNt == STATUS_SUCCESS);
870 if (NT_SUCCESS(rcNt))
871 {
872 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelAlertedThread);
873 return VINF_SUCCESS;
874 }
875 AssertLogRelMsgFailedReturn(("NtAlertThread failed: %#x\n", rcNt), RTErrConvertFromNtStatus(rcNt));
876 }
877 break;
878
879 default:
880 return VINF_SUCCESS;
881 }
882
883 ASMNopPause();
884 RT_NOREF(pVM);
885 }
886}
887# endif /* IN_RING3 */
888#endif /* NEM_WIN_USE_OUR_OWN_RUN_API */
889
890
891#ifdef LOG_ENABLED
892/**
893 * Logs the current CPU state.
894 */
895NEM_TMPL_STATIC void nemHCWinLogState(PVM pVM, PVMCPU pVCpu)
896{
897 if (LogIs3Enabled())
898 {
899# ifdef IN_RING3
900 char szRegs[4096];
901 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
902 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
903 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
904 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
905 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
906 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
907 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
908 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
909 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
910 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
911 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
912 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
913 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
914 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
915 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
916 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
917 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
918 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
919 " efer=%016VR{efer}\n"
920 " pat=%016VR{pat}\n"
921 " sf_mask=%016VR{sf_mask}\n"
922 "krnl_gs_base=%016VR{krnl_gs_base}\n"
923 " lstar=%016VR{lstar}\n"
924 " star=%016VR{star} cstar=%016VR{cstar}\n"
925 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
926 );
927
928 char szInstr[256];
929 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
930 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
931 szInstr, sizeof(szInstr), NULL);
932 Log3(("%s%s\n", szRegs, szInstr));
933# else
934 /** @todo stat logging in ring-0 */
935 RT_NOREF(pVM, pVCpu);
936# endif
937 }
938}
939#endif /* LOG_ENABLED */
940
941
942/**
943 * Advances the guest RIP and clear EFLAGS.RF.
944 *
945 * This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
946 *
947 * @param pVCpu The cross context virtual CPU structure.
948 * @param pCtx The CPU context to update.
949 * @param pExitCtx The exit context.
950 */
951DECLINLINE(void) nemHCWinAdvanceGuestRipAndClearRF(PVMCPU pVCpu, PCPUMCTX pCtx, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr)
952{
953 Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)));
954
955 /* Advance the RIP. */
956 Assert(pMsgHdr->InstructionLength > 0 && pMsgHdr->InstructionLength < 16);
957 pCtx->rip += pMsgHdr->InstructionLength;
958 pCtx->rflags.Bits.u1RF = 0;
959
960 /* Update interrupt inhibition. */
961 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
962 { /* likely */ }
963 else if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
964 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
965}
966
967
968NEM_TMPL_STATIC VBOXSTRICTRC
969nemHCWinHandleHalt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
970{
971 NOREF(pVM); NOREF(pVCpu); NOREF(pCtx);
972 LogFlow(("nemHCWinHandleHalt\n"));
973 return VINF_EM_HALT;
974}
975
976
977NEM_TMPL_STATIC DECLCALLBACK(int)
978nemHCWinUnmapOnePageCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, uint8_t *pu2NemState, void *pvUser)
979{
980 RT_NOREF_PV(pvUser);
981#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
982 int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
983 AssertRC(rc);
984 if (RT_SUCCESS(rc))
985#else
986 RT_NOREF_PV(pVCpu);
987 HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
988 if (SUCCEEDED(hrc))
989#endif
990 {
991 Log5(("NEM GPA unmap all: %RGp (cMappedPages=%u)\n", GCPhys, pVM->nem.s.cMappedPages - 1));
992 *pu2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
993 }
994 else
995 {
996#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
997 LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
998#else
999 LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1000 GCPhys, g_apszPageStates[*pu2NemState], hrc, hrc, RTNtLastStatusValue(),
1001 RTNtLastErrorValue(), pVM->nem.s.cMappedPages));
1002#endif
1003 *pu2NemState = NEM_WIN_PAGE_STATE_NOT_SET;
1004 }
1005 if (pVM->nem.s.cMappedPages > 0)
1006 ASMAtomicDecU32(&pVM->nem.s.cMappedPages);
1007 return VINF_SUCCESS;
1008}
1009
1010
1011/**
1012 * State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
1013 * and nemHCWinHandleMemoryAccessPageCheckerCallback.
1014 */
1015typedef struct NEMHCWINHMACPCCSTATE
1016{
1017 /** Input: Write access. */
1018 bool fWriteAccess;
1019 /** Output: Set if we did something. */
1020 bool fDidSomething;
1021 /** Output: Set it we should resume. */
1022 bool fCanResume;
1023} NEMHCWINHMACPCCSTATE;
1024
1025/**
1026 * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1027 * Worker for nemR3WinHandleMemoryAccess; pvUser points to a
1028 * NEMHCWINHMACPCCSTATE structure. }
1029 */
1030NEM_TMPL_STATIC DECLCALLBACK(int)
1031nemHCWinHandleMemoryAccessPageCheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1032{
1033 NEMHCWINHMACPCCSTATE *pState = (NEMHCWINHMACPCCSTATE *)pvUser;
1034 pState->fDidSomething = false;
1035 pState->fCanResume = false;
1036
1037 /* If A20 is disabled, we may need to make another query on the masked
1038 page to get the correct protection information. */
1039 uint8_t u2State = pInfo->u2NemState;
1040 RTGCPHYS GCPhysSrc;
1041 if ( pVM->nem.s.fA20Enabled
1042 || !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
1043 GCPhysSrc = GCPhys;
1044 else
1045 {
1046 GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20);
1047 PGMPHYSNEMPAGEINFO Info2;
1048 int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL);
1049 AssertRCReturn(rc, rc);
1050
1051 *pInfo = Info2;
1052 pInfo->u2NemState = u2State;
1053 }
1054
1055 /*
1056 * Consolidate current page state with actual page protection and access type.
1057 * We don't really consider downgrades here, as they shouldn't happen.
1058 */
1059#ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1060 /** @todo Someone at microsoft please explain:
1061 * I'm not sure WTF was going on, but I ended up in a loop if I remapped a
1062 * readonly page as writable (unmap, then map again). Specifically, this was an
1063 * issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1. So, in
1064 * a hope to work around that we no longer pre-map anything, just unmap stuff
1065 * and do it lazily here. And here we will first unmap, restart, and then remap
1066 * with new protection or backing.
1067 */
1068#endif
1069 int rc;
1070 switch (u2State)
1071 {
1072 case NEM_WIN_PAGE_STATE_UNMAPPED:
1073 case NEM_WIN_PAGE_STATE_NOT_SET:
1074 if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1075 {
1076 Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1077 return VINF_SUCCESS;
1078 }
1079
1080 /* Don't bother remapping it if it's a write request to a non-writable page. */
1081 if ( pState->fWriteAccess
1082 && !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1083 {
1084 Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1085 return VINF_SUCCESS;
1086 }
1087
1088 /* Map the page. */
1089 rc = nemHCNativeSetPhysPage(pVM,
1090 pVCpu,
1091 GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1092 GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1093 pInfo->fNemProt,
1094 &u2State,
1095 true /*fBackingState*/);
1096 pInfo->u2NemState = u2State;
1097 Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1098 GCPhys, g_apszPageStates[u2State], rc));
1099 pState->fDidSomething = true;
1100 pState->fCanResume = true;
1101 return rc;
1102
1103 case NEM_WIN_PAGE_STATE_READABLE:
1104 if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1105 && (pInfo->fNemProt & (NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE)))
1106 {
1107 Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1108 return VINF_SUCCESS;
1109 }
1110
1111#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1112 /* Upgrade page to writable. */
1113/** @todo test this*/
1114 if ( (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1115 && pState->fWriteAccess)
1116 {
1117 rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhys,
1118 HV_MAP_GPA_READABLE | HV_MAP_GPA_WRITABLE
1119 | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
1120 AssertRC(rc);
1121 if (RT_SUCCESS(rc))
1122 {
1123 pInfo->u2NemState = NEM_WIN_PAGE_STATE_WRITABLE;
1124 pState->fDidSomething = true;
1125 pState->fCanResume = true;
1126 Log5(("NEM GPA write-upgrade/exit: %RGp (was %s, cMappedPages=%u)\n",
1127 GCPhys, g_apszPageStates[u2State], pVM->nem.s.cMappedPages));
1128 }
1129 }
1130 else
1131 {
1132 /* Need to emulate the acces. */
1133 AssertBreak(pInfo->fNemProt != NEM_PAGE_PROT_NONE); /* There should be no downgrades. */
1134 rc = VINF_SUCCESS;
1135 }
1136 return rc;
1137#else
1138 break;
1139#endif
1140
1141 case NEM_WIN_PAGE_STATE_WRITABLE:
1142 if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1143 {
1144 Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3\n", GCPhys));
1145 return VINF_SUCCESS;
1146 }
1147#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1148 AssertFailed(); /* There should be no downgrades. */
1149#endif
1150 break;
1151
1152 default:
1153 AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_INTERNAL_ERROR_3);
1154 }
1155
1156 /*
1157 * Unmap and restart the instruction.
1158 * If this fails, which it does every so often, just unmap everything for now.
1159 */
1160#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1161 rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1162 AssertRC(rc);
1163 if (RT_SUCCESS(rc))
1164#else
1165 /** @todo figure out whether we mess up the state or if it's WHv. */
1166 HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1167 if (SUCCEEDED(hrc))
1168#endif
1169 {
1170 pState->fDidSomething = true;
1171 pState->fCanResume = true;
1172 pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1173 uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
1174 Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
1175 return VINF_SUCCESS;
1176 }
1177#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1178 LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhys, rc));
1179 return rc;
1180#else
1181 LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1182 GCPhys, g_apszPageStates[u2State], hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue(),
1183 pVM->nem.s.cMappedPages));
1184
1185 PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinUnmapOnePageCallback, NULL);
1186 Log(("nemHCWinHandleMemoryAccessPageCheckerCallback: Unmapped all (cMappedPages=%u)\n", pVM->nem.s.cMappedPages));
1187
1188 pState->fDidSomething = true;
1189 pState->fCanResume = true;
1190 pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1191 return VINF_SUCCESS;
1192#endif
1193}
1194
1195
1196#ifdef NEM_WIN_USE_OUR_OWN_RUN_API
1197
1198# ifdef IN_RING0
1199/**
1200 * Wrapper around nemR0WinImportState that converts VERR_NEM_CHANGE_PGM_MODE and
1201 * VERR_NEM_FLUSH_TBL into informational status codes and logs+asserts statuses.
1202 *
1203 * @returns VBox strict status code.
1204 * @param pGVM The global (ring-0) VM structure.
1205 * @param pGVCpu The global (ring-0) per CPU structure.
1206 * @param pCtx The CPU context to import into.
1207 * @param fWhat What to import.
1208 * @param pszCaller Whoe is doing the importing.
1209 */
1210DECLINLINE(VBOXSTRICTRC) nemR0WinImportStateStrict(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx, uint64_t fWhat, const char *pszCaller)
1211{
1212 int rc = nemR0WinImportState(pGVM, pGVCpu, pCtx, fWhat);
1213 if (RT_SUCCESS(rc))
1214 {
1215 Assert(rc == VINF_SUCCESS);
1216 return VINF_SUCCESS;
1217 }
1218
1219 if (rc == VERR_NEM_CHANGE_PGM_MODE || rc == VERR_NEM_FLUSH_TLB)
1220 {
1221 Log4(("%s/%u: nemR0WinImportState -> %Rrc\n", pszCaller, pGVCpu->idCpu, -rc));
1222 return -rc;
1223 }
1224 RT_NOREF(pszCaller);
1225 AssertMsgFailedReturn(("%s/%u: nemR0WinImportState failed: %Rrc\n", pszCaller, pGVCpu->idCpu, rc), rc);
1226}
1227# endif /* IN_RING0 */
1228
1229/**
1230 * Copies register state from the X64 intercept message header.
1231 *
1232 * ASSUMES no state copied yet.
1233 *
1234 * @param pCtx The registe rcontext.
1235 * @param pHdr The X64 intercept message header.
1236 */
1237DECLINLINE(void) nemHCWinCopyStateFromX64Header(PCPUMCTX pCtx, HV_X64_INTERCEPT_MESSAGE_HEADER const *pHdr)
1238{
1239 Assert( (pCtx->fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS))
1240 == (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS));
1241 NEM_WIN_COPY_BACK_SEG(pCtx->cs, pHdr->CsSegment);
1242 pCtx->rip = pHdr->Rip;
1243 pCtx->rflags.u = pHdr->Rflags;
1244 pCtx->fExtrn &= ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS);
1245}
1246
1247
1248/**
1249 * Deals with memory intercept message.
1250 *
1251 * @returns Strict VBox status code.
1252 * @param pVM The cross context VM structure.
1253 * @param pVCpu The cross context per CPU structure.
1254 * @param pMsg The message.
1255 * @param pCtx The register context.
1256 * @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
1257 */
1258NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageMemory(PVM pVM, PVMCPU pVCpu, HV_X64_MEMORY_INTERCEPT_MESSAGE const *pMsg,
1259 PCPUMCTX pCtx, PGVMCPU pGVCpu)
1260{
1261 Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
1262 || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
1263 || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
1264 AssertMsg(pMsg->Header.InstructionLength < 0x10, ("%#x\n", pMsg->Header.InstructionLength));
1265
1266 /*
1267 * Whatever we do, we must clear pending event ejection upon resume.
1268 */
1269 if (pMsg->Header.ExecutionState.InterruptionPending)
1270 pCtx->fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_MASK;
1271
1272#if 0 /* Experiment: 20K -> 34K exit/s. */
1273 if ( pMsg->Header.ExecutionState.EferLma
1274 && pMsg->Header.CsSegment.Long
1275 && pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
1276 {
1277 if ( pMsg->Header.Rip - (uint64_t)0xf65a < (uint64_t)(0xf662 - 0xf65a)
1278 && pMsg->InstructionBytes[0] == 0x89
1279 && pMsg->InstructionBytes[1] == 0x03)
1280 {
1281 pCtx->rip = pMsg->Header.Rip + 2;
1282 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
1283 AssertMsg(pMsg->Header.InstructionLength == 2, ("%#x\n", pMsg->Header.InstructionLength));
1284 //Log(("%RX64 msg:\n%.80Rhxd\n", pCtx->rip, pMsg));
1285 return VINF_SUCCESS;
1286 }
1287 }
1288#endif
1289
1290 /*
1291 * Ask PGM for information about the given GCPhys. We need to check if we're
1292 * out of sync first.
1293 */
1294 NEMHCWINHMACPCCSTATE State = { pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE, false, false };
1295 PGMPHYSNEMPAGEINFO Info;
1296 int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pMsg->GuestPhysicalAddress, State.fWriteAccess, &Info,
1297 nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
1298 if (RT_SUCCESS(rc))
1299 {
1300 if (Info.fNemProt & ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
1301 ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
1302 {
1303 if (State.fCanResume)
1304 {
1305 Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
1306 pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
1307 pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
1308 Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
1309 State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
1310 return VINF_SUCCESS;
1311 }
1312 }
1313 Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
1314 pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
1315 pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
1316 Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
1317 State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
1318 }
1319 else
1320 Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating (%s)\n",
1321 pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, pMsg->GuestPhysicalAddress, rc,
1322 State.fDidSomething ? " modified-backing" : "", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
1323
1324 /*
1325 * Emulate the memory access, either access handler or special memory.
1326 */
1327 nemHCWinCopyStateFromX64Header(pCtx, &pMsg->Header);
1328 VBOXSTRICTRC rcStrict;
1329# ifdef IN_RING0
1330 rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "MemExit");
1331 if (rcStrict != VINF_SUCCESS)
1332 return rcStrict;
1333# else
1334 rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
1335 AssertRCReturn(rc, rc);
1336 NOREF(pGVCpu);
1337# endif
1338
1339 if (pMsg->Reserved1)
1340 Log(("MemExit/Reserved1=%#x\n", pMsg->Reserved1));
1341 if (pMsg->Header.ExecutionState.Reserved0 || pMsg->Header.ExecutionState.Reserved1)
1342 Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pMsg->Header.ExecutionState.Reserved0, pMsg->Header.ExecutionState.Reserved1));
1343 //if (pMsg->InstructionByteCount > 0)
1344 // Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
1345
1346 if (pMsg->InstructionByteCount > 0)
1347 rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(pCtx), pMsg->Header.Rip,
1348 pMsg->InstructionBytes, pMsg->InstructionByteCount);
1349 else
1350 rcStrict = IEMExecOne(pVCpu);
1351 /** @todo do we need to do anything wrt debugging here? */
1352 return rcStrict;
1353
1354}
1355
1356
1357/**
1358 * Deals with I/O port intercept message.
1359 *
1360 * @returns Strict VBox status code.
1361 * @param pVM The cross context VM structure.
1362 * @param pVCpu The cross context per CPU structure.
1363 * @param pMsg The message.
1364 * @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
1365 */
1366NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageIoPort(PVM pVM, PVMCPU pVCpu, HV_X64_IO_PORT_INTERCEPT_MESSAGE const *pMsg,
1367 PCPUMCTX pCtx, PGVMCPU pGVCpu)
1368{
1369 Assert( pMsg->AccessInfo.AccessSize == 1
1370 || pMsg->AccessInfo.AccessSize == 2
1371 || pMsg->AccessInfo.AccessSize == 4);
1372 Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
1373 || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
1374 AssertMsg(pMsg->Header.InstructionLength < 0x10, ("%#x\n", pMsg->Header.InstructionLength));
1375
1376 /*
1377 * Whatever we do, we must clear pending event ejection upon resume.
1378 */
1379 if (pMsg->Header.ExecutionState.InterruptionPending)
1380 pCtx->fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_MASK;
1381
1382 VBOXSTRICTRC rcStrict;
1383 if (!pMsg->AccessInfo.StringOp)
1384 {
1385 /*
1386 * Simple port I/O.
1387 */
1388 static uint32_t const s_fAndMask[8] =
1389 { UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
1390 uint32_t const fAndMask = s_fAndMask[pMsg->AccessInfo.AccessSize];
1391 if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
1392 {
1393 rcStrict = IOMIOPortWrite(pVM, pVCpu, pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize);
1394 Log4(("IOExit/%u: %04x:%08RX64: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
1395 pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, pMsg->PortNumber,
1396 (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
1397 if (IOM_SUCCESS(rcStrict))
1398 {
1399 nemHCWinCopyStateFromX64Header(pCtx, &pMsg->Header);
1400 nemHCWinAdvanceGuestRipAndClearRF(pVCpu, pCtx, &pMsg->Header);
1401 }
1402 }
1403 else
1404 {
1405 uint32_t uValue = 0;
1406 rcStrict = IOMIOPortRead(pVM, pVCpu, pMsg->PortNumber, &uValue, pMsg->AccessInfo.AccessSize);
1407 Log4(("IOExit/%u: %04x:%08RX64: IN %#x LB %u -> %#x, rcStrict=%Rrc\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
1408 pMsg->Header.Rip, pMsg->PortNumber, pMsg->AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
1409 if (IOM_SUCCESS(rcStrict))
1410 {
1411 if (pMsg->AccessInfo.AccessSize != 4)
1412 pCtx->rax = (pMsg->Rax & ~(uint64_t)fAndMask) | (uValue & fAndMask);
1413 else
1414 pCtx->rax = uValue;
1415 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RAX;
1416 Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pMsg->Rax, pCtx->rax));
1417 nemHCWinCopyStateFromX64Header(pCtx, &pMsg->Header);
1418 nemHCWinAdvanceGuestRipAndClearRF(pVCpu, pCtx, &pMsg->Header);
1419 }
1420 }
1421 }
1422 else
1423 {
1424 /*
1425 * String port I/O.
1426 */
1427 /** @todo Someone at Microsoft please explain how we can get the address mode
1428 * from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
1429 * getting the default mode, it can always be overridden by a prefix. This
1430 * forces us to interpret the instruction from opcodes, which is suboptimal.
1431 * Both AMD-V and VT-x includes the address size in the exit info, at least on
1432 * CPUs that are reasonably new.
1433 *
1434 * Of course, it's possible this is an undocumented and we just need to do some
1435 * experiments to figure out how it's communicated. Alternatively, we can scan
1436 * the opcode bytes for possible evil prefixes.
1437 */
1438 nemHCWinCopyStateFromX64Header(pCtx, &pMsg->Header);
1439 pCtx->fExtrn &= ~( CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI
1440 | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES);
1441 NEM_WIN_COPY_BACK_SEG(pCtx->ds, pMsg->DsSegment);
1442 NEM_WIN_COPY_BACK_SEG(pCtx->es, pMsg->EsSegment);
1443 pCtx->rax = pMsg->Rax;
1444 pCtx->rcx = pMsg->Rcx;
1445 pCtx->rdi = pMsg->Rdi;
1446 pCtx->rsi = pMsg->Rsi;
1447# ifdef IN_RING0
1448 rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
1449 if (rcStrict != VINF_SUCCESS)
1450 return rcStrict;
1451# else
1452 int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
1453 AssertRCReturn(rc, rc);
1454 RT_NOREF(pGVCpu);
1455# endif
1456
1457 Log4(("IOExit/%u: %04x:%08RX64: %s%s %#x LB %u (emulating)\n",
1458 pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
1459 pMsg->AccessInfo.RepPrefix ? "REP " : "",
1460 pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUTS" : "INS",
1461 pMsg->PortNumber, pMsg->AccessInfo.AccessSize ));
1462 rcStrict = IEMExecOne(pVCpu);
1463 }
1464 if (IOM_SUCCESS(rcStrict))
1465 {
1466 /*
1467 * Do debug checks.
1468 */
1469 if ( pMsg->Header.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflext DR7? */
1470 || (pMsg->Header.Rflags & X86_EFL_TF)
1471 || DBGFBpIsHwIoArmed(pVM) )
1472 {
1473 /** @todo Debugging. */
1474 }
1475 }
1476 return rcStrict;
1477}
1478
1479
1480/**
1481 * Handles messages (VM exits).
1482 *
1483 * @returns Strict VBox status code.
1484 * @param pVM The cross context VM structure.
1485 * @param pVCpu The cross context per CPU structure.
1486 * @param pMappingHeader The message slot mapping.
1487 * @param pCtx The register context.
1488 * @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
1489 */
1490NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessage(PVM pVM, PVMCPU pVCpu, VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader,
1491 PCPUMCTX pCtx, PGVMCPU pGVCpu)
1492{
1493 if (pMappingHeader->enmVidMsgType == VidMessageHypervisorMessage)
1494 {
1495 AssertMsg(pMappingHeader->cbMessage == HV_MESSAGE_SIZE, ("%#x\n", pMappingHeader->cbMessage));
1496 HV_MESSAGE const *pMsg = (HV_MESSAGE const *)(pMappingHeader + 1);
1497 switch (pMsg->Header.MessageType)
1498 {
1499 case HvMessageTypeUnmappedGpa:
1500 Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
1501 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
1502 return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pCtx, pGVCpu);
1503
1504 case HvMessageTypeGpaIntercept:
1505 Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
1506 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemIntercept);
1507 return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pCtx, pGVCpu);
1508
1509 case HvMessageTypeX64IoPortIntercept:
1510 Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64IoPortIntercept));
1511 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
1512 return nemHCWinHandleMessageIoPort(pVM, pVCpu, &pMsg->X64IoPortIntercept, pCtx, pGVCpu);
1513
1514 case HvMessageTypeX64Halt:
1515 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
1516 return VINF_EM_HALT;
1517
1518 case HvMessageTypeX64InterruptWindow:
1519 AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n", pMsg->Header.MessageType),
1520 VERR_INTERNAL_ERROR_2);
1521
1522 case HvMessageTypeInvalidVpRegisterValue:
1523 case HvMessageTypeUnrecoverableException:
1524 case HvMessageTypeUnsupportedFeature:
1525 case HvMessageTypeTlbPageSizeMismatch:
1526 AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n", pMsg->Header.MessageType),
1527 VERR_INTERNAL_ERROR_2);
1528
1529 case HvMessageTypeX64MsrIntercept:
1530 case HvMessageTypeX64CpuidIntercept:
1531 case HvMessageTypeX64ExceptionIntercept:
1532 case HvMessageTypeX64ApicEoi:
1533 case HvMessageTypeX64LegacyFpError:
1534 case HvMessageTypeX64RegisterIntercept:
1535 case HvMessageTypeApicEoi:
1536 case HvMessageTypeFerrAsserted:
1537 case HvMessageTypeEventLogBufferComplete:
1538 case HvMessageTimerExpired:
1539 AssertLogRelMsgFailedReturn(("Unexpected message on CPU #%u: #x\n", pVCpu->idCpu, pMsg->Header.MessageType),
1540 VERR_INTERNAL_ERROR_2);
1541
1542 default:
1543 AssertLogRelMsgFailedReturn(("Unknown message on CPU #%u: #x\n", pVCpu->idCpu, pMsg->Header.MessageType),
1544 VERR_INTERNAL_ERROR_2);
1545 }
1546 }
1547 else
1548 AssertLogRelMsgFailedReturn(("Unexpected VID message type on CPU #%u: %#x LB %u\n",
1549 pVCpu->idCpu, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage),
1550 VERR_INTERNAL_ERROR_3);
1551}
1552
1553
1554/**
1555 * Worker for nemHCWinRunGC that stops the execution on the way out.
1556 *
1557 * The CPU was running the last time we checked, no there are no messages that
1558 * needs being marked handled/whatever. Caller checks this.
1559 *
1560 * @returns rcStrict on success, error status on failure.
1561 * @param pVM The cross context VM structure.
1562 * @param pVCpu The cross context per CPU structure.
1563 * @param rcStrict The nemHCWinRunGC return status. This is a little
1564 * bit unnecessary, except in internal error cases,
1565 * since we won't need to stop the CPU if we took an
1566 * exit.
1567 * @param pMappingHeader The message slot mapping.
1568 * @param pGVM The global (ring-0) VM structure (NULL in r3).
1569 * @param pGVCpu The global (ring-0) per CPU structure (NULL in r3).
1570 */
1571NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinStopCpu(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict,
1572 VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader,
1573 PGVM pGVM, PGVMCPU pGVCpu)
1574{
1575 /*
1576 * Try stopping the processor. If we're lucky we manage to do this before it
1577 * does another VM exit.
1578 */
1579# ifdef IN_RING0
1580 pVCpu->nem.s.uIoCtlBuf.idCpu = pGVCpu->idCpu;
1581 NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlStopVirtualProcessor.uFunction,
1582 &pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
1583 NULL, 0);
1584 if (NT_SUCCESS(rcNt))
1585 {
1586 Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
1587 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
1588 return rcStrict;
1589 }
1590# else
1591 BOOL fRet = VidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu);
1592 if (fRet)
1593 {
1594 Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
1595 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
1596 return rcStrict;
1597 }
1598 RT_NOREF(pGVM, pGVCpu);
1599# endif
1600
1601 /*
1602 * Dang. The CPU stopped by itself and we got a couple of message to deal with.
1603 */
1604# ifdef IN_RING0
1605 AssertLogRelMsgReturn(rcNt == ERROR_VID_STOP_PENDING, ("rcNt=%#x\n", rcNt),
1606 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1607# else
1608 DWORD dwErr = RTNtLastErrorValue();
1609 AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u (%#x)\n", dwErr, dwErr),
1610 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1611# endif
1612 Log8(("nemHCWinStopCpu: Stopping CPU pending...\n"));
1613 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPending);
1614
1615 /*
1616 * First message: Exit or similar.
1617 * Note! We can safely ASSUME that rcStrict isn't an important information one.
1618 */
1619# ifdef IN_RING0
1620 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
1621 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
1622 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = 30000; /*ms*/
1623 rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
1624 &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
1625 sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
1626 NULL, 0);
1627 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
1628 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1629# else
1630 BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
1631 VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /*ms*/);
1632 AssertLogRelMsgReturn(fWait, ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
1633 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1634# endif
1635
1636 /* It should be a hypervisor message and definitely not a stop request completed message. */
1637 VID_MESSAGE_TYPE enmVidMsgType = pMappingHeader->enmVidMsgType;
1638 AssertLogRelMsgReturn(enmVidMsgType != VidMessageStopRequestComplete,
1639 ("Unexpected 1st message following ERROR_VID_STOP_PENDING: %#x LB %#x\n",
1640 enmVidMsgType, pMappingHeader->cbMessage),
1641 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1642
1643 VBOXSTRICTRC rcStrict2 = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, CPUMQueryGuestCtxPtr(pVCpu), pGVCpu);
1644 if (rcStrict2 != VINF_SUCCESS && RT_SUCCESS(rcStrict))
1645 rcStrict = rcStrict2;
1646
1647 /*
1648 * Mark it as handled and get the stop request completed message, then mark
1649 * that as handled too. CPU is back into fully stopped stated then.
1650 */
1651# ifdef IN_RING0
1652 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
1653 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE;
1654 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = 30000; /*ms*/
1655 rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
1656 &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
1657 sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
1658 NULL, 0);
1659 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("2st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
1660 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1661# else
1662 fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
1663 VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /*ms*/);
1664 AssertLogRelMsgReturn(fWait, ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
1665 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1666# endif
1667
1668 /* It should be a stop request completed message. */
1669 enmVidMsgType = pMappingHeader->enmVidMsgType;
1670 AssertLogRelMsgReturn(enmVidMsgType == VidMessageStopRequestComplete,
1671 ("Unexpected 2nd message following ERROR_VID_STOP_PENDING: %#x LB %#x\n",
1672 enmVidMsgType, pMappingHeader->cbMessage),
1673 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1674
1675 /* Mark this as handled. */
1676# ifdef IN_RING0
1677 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
1678 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = VID_MSHAGN_F_HANDLE_MESSAGE;
1679 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = 30000; /*ms*/
1680 rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
1681 &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
1682 sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
1683 NULL, 0);
1684 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
1685 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1686# else
1687 fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /*ms*/);
1688 AssertLogRelMsgReturn(fWait, ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
1689 RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
1690# endif
1691 Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict) ));
1692 return rcStrict;
1693}
1694
1695
1696NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVM pVM, PVMCPU pVCpu, PGVM pGVM, PGVMCPU pGVCpu)
1697{
1698 PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1699 LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pCtx->cs.Sel, pCtx->rip, pCtx->rflags));
1700# ifdef LOG_ENABLED
1701 if (LogIs3Enabled())
1702 nemHCWinLogState(pVM, pVCpu);
1703# endif
1704
1705 /*
1706 * Try switch to NEM runloop state.
1707 */
1708 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
1709 { /* likely */ }
1710 else
1711 {
1712 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1713 LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
1714 return VINF_SUCCESS;
1715 }
1716
1717 /*
1718 * The run loop.
1719 *
1720 * Current approach to state updating to use the sledgehammer and sync
1721 * everything every time. This will be optimized later.
1722 */
1723 VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader = (VID_MESSAGE_MAPPING_HEADER volatile *)pVCpu->nem.s.pvMsgSlotMapping;
1724 uint32_t cMillies = 5000; /** @todo lower this later... */
1725 const bool fSingleStepping = false; /** @todo get this from somewhere. */
1726 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1727 for (unsigned iLoop = 0;;iLoop++)
1728 {
1729 /*
1730 * Ensure that hyper-V has the whole state.
1731 */
1732 if ((pCtx->fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK)) != (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK))
1733 {
1734# ifdef IN_RING0
1735 int rc2 = nemR0WinExportState(pGVM, pGVCpu, pCtx);
1736# else
1737 int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu, pCtx);
1738 RT_NOREF(pGVM, pGVCpu);
1739# endif
1740 AssertRCReturn(rc2, rc2);
1741 }
1742
1743 /*
1744 * Run a bit.
1745 */
1746 if ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
1747 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
1748 {
1749 if (pVCpu->nem.s.fHandleAndGetFlags)
1750 { /* Very likely that the CPU does NOT need starting (pending msg, running). */ }
1751 else
1752 {
1753# ifdef IN_RING0
1754 pVCpu->nem.s.uIoCtlBuf.idCpu = pGVCpu->idCpu;
1755 NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlStartVirtualProcessor.uFunction,
1756 &pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
1757 NULL, 0);
1758 LogFlow(("NEM/%u: IoCtlStartVirtualProcessor -> %#x\n", pVCpu->idCpu, rcNt));
1759 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("VidStartVirtualProcessor failed for CPU #%u: %#x\n", pGVCpu->idCpu, rcNt),
1760 VERR_INTERNAL_ERROR_3);
1761# else
1762 AssertLogRelMsgReturn(g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
1763 ("VidStartVirtualProcessor failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
1764 pVCpu->idCpu, RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()),
1765 VERR_INTERNAL_ERROR_3);
1766# endif
1767 pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
1768 }
1769
1770 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
1771 {
1772# ifdef IN_RING0
1773 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
1774 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = pVCpu->nem.s.fHandleAndGetFlags;
1775 pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
1776 NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
1777 &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
1778 sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
1779 NULL, 0);
1780 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
1781 if (rcNt == STATUS_SUCCESS)
1782# else
1783 BOOL fRet = VidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
1784 pVCpu->nem.s.fHandleAndGetFlags, cMillies);
1785 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
1786 if (fRet)
1787# endif
1788 {
1789 /*
1790 * Deal with the message.
1791 */
1792 rcStrict = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, pCtx, pGVCpu);
1793 pVCpu->nem.s.fHandleAndGetFlags |= VID_MSHAGN_F_HANDLE_MESSAGE;
1794 if (rcStrict == VINF_SUCCESS)
1795 { /* hopefully likely */ }
1796 else
1797 {
1798 LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1799 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1800 break;
1801 }
1802 }
1803 else
1804 {
1805 /* VID.SYS merges STATUS_ALERTED and STATUS_USER_APC into STATUS_TIMEOUT,
1806 so after NtAlertThread we end up here with a STATUS_TIMEOUT. And yeah,
1807 the error code conversion is into WAIT_XXX, i.e. NT status codes. */
1808# ifndef IN_RING0
1809 DWORD rcNt = GetLastError();
1810# endif
1811 LogFlow(("NEM/%u: VidMessageSlotHandleAndGetNext -> %#x\n", pVCpu->idCpu, rcNt));
1812 AssertLogRelMsgReturn( rcNt == STATUS_TIMEOUT
1813 || rcNt == STATUS_ALERTED /* just in case */
1814 || rcNt == STATUS_USER_APC /* ditto */
1815 , ("VidMessageSlotHandleAndGetNext failed for CPU #%u: %#x (%u)\n", pVCpu->idCpu, rcNt, rcNt),
1816 VERR_INTERNAL_ERROR_3);
1817 pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
1818 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatGetMsgTimeout);
1819 }
1820
1821 /*
1822 * If no relevant FFs are pending, loop.
1823 */
1824 if ( !VM_FF_IS_PENDING( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
1825 && !VMCPU_FF_IS_PENDING(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
1826 continue;
1827
1828 /** @todo Try handle pending flags, not just return to EM loops. Take care
1829 * not to set important RCs here unless we've handled a message. */
1830 LogFlow(("NEM/%u: breaking: pending FF (%#x / %#x)\n",
1831 pVCpu->idCpu, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
1832 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
1833 }
1834 else
1835 {
1836 LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
1837 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
1838 }
1839 }
1840 else
1841 {
1842 LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
1843 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
1844 }
1845 break;
1846 } /* the run loop */
1847
1848
1849 /*
1850 * If the CPU is running, make sure to stop it before we try sync back the
1851 * state and return to EM.
1852 */
1853 if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
1854 {
1855 pVCpu->nem.s.fHandleAndGetFlags = 0;
1856 rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader, pGVM, pGVCpu);
1857 }
1858
1859 if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
1860 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1861
1862 if (pCtx->fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
1863 {
1864# ifdef IN_RING0
1865 int rc2 = nemR0WinImportState(pGVM, pGVCpu, pCtx, CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK);
1866 if (RT_SUCCESS(rc2))
1867 pCtx->fExtrn = 0;
1868 else if (rc2 == VERR_NEM_CHANGE_PGM_MODE || rc2 == VERR_NEM_FLUSH_TLB)
1869 {
1870 pCtx->fExtrn = 0;
1871 if (rcStrict == VINF_SUCCESS || rcStrict == -rc2)
1872 rcStrict = -rc2;
1873 else
1874 {
1875 pVCpu->nem.s.rcPgmPending = -rc2;
1876 LogFlow(("NEM/%u: rcPgmPending=%Rrc (rcStrict=%Rrc)\n", pVCpu->idCpu, rc2, VBOXSTRICTRC_VAL(rcStrict) ));
1877 }
1878 }
1879# else
1880 int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK);
1881 if (RT_SUCCESS(rc2))
1882 pCtx->fExtrn = 0;
1883# endif
1884 else if (RT_SUCCESS(rcStrict))
1885 rcStrict = rc2;
1886 }
1887 else
1888 pCtx->fExtrn = 0;
1889
1890 LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
1891 pVCpu->idCpu, pCtx->cs.Sel, pCtx->rip, pCtx->rflags, VBOXSTRICTRC_VAL(rcStrict) ));
1892 return rcStrict;
1893}
1894
1895#endif /* NEM_WIN_USE_OUR_OWN_RUN_API */
1896
1897
1898/**
1899 * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
1900 */
1901NEM_TMPL_STATIC DECLCALLBACK(int) nemHCWinUnsetForA20CheckerCallback(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys,
1902 PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1903{
1904 /* We'll just unmap the memory. */
1905 if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
1906 {
1907#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1908 int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1909 AssertRC(rc);
1910 if (RT_SUCCESS(rc))
1911#else
1912 HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1913 if (SUCCEEDED(hrc))
1914#endif
1915 {
1916 uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
1917 Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
1918 pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1919 }
1920 else
1921 {
1922#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1923 LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
1924 return rc;
1925#else
1926 LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
1927 GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
1928 return VERR_INTERNAL_ERROR_2;
1929#endif
1930 }
1931 }
1932 RT_NOREF(pVCpu, pvUser);
1933 return VINF_SUCCESS;
1934}
1935
1936
1937/**
1938 * Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
1939 *
1940 * @returns The PGMPhysNemQueryPageInfo result.
1941 * @param pVM The cross context VM structure.
1942 * @param pVCpu The cross context virtual CPU structure.
1943 * @param GCPhys The page to unmap.
1944 */
1945NEM_TMPL_STATIC int nemHCWinUnmapPageForA20Gate(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1946{
1947 PGMPHYSNEMPAGEINFO Info;
1948 return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /*fMakeWritable*/, &Info,
1949 nemHCWinUnsetForA20CheckerCallback, NULL);
1950}
1951
1952
1953void nemHCNativeNotifyHandlerPhysicalRegister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
1954{
1955 Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
1956 NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
1957}
1958
1959
1960void nemHCNativeNotifyHandlerPhysicalDeregister(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
1961 int fRestoreAsRAM, bool fRestoreAsRAM2)
1962{
1963 Log5(("nemHCNativeNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d fRestoreAsRAM=%d fRestoreAsRAM2=%d\n",
1964 GCPhys, cb, enmKind, fRestoreAsRAM, fRestoreAsRAM2));
1965 NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb); NOREF(fRestoreAsRAM); NOREF(fRestoreAsRAM2);
1966}
1967
1968
1969void nemHCNativeNotifyHandlerPhysicalModify(PVM pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
1970 RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
1971{
1972 Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
1973 GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
1974 NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
1975}
1976
1977
1978/**
1979 * Worker that maps pages into Hyper-V.
1980 *
1981 * This is used by the PGM physical page notifications as well as the memory
1982 * access VMEXIT handlers.
1983 *
1984 * @returns VBox status code.
1985 * @param pVM The cross context VM structure.
1986 * @param pVCpu The cross context virtual CPU structure of the
1987 * calling EMT.
1988 * @param GCPhysSrc The source page address.
1989 * @param GCPhysDst The hyper-V destination page. This may differ from
1990 * GCPhysSrc when A20 is disabled.
1991 * @param fPageProt NEM_PAGE_PROT_XXX.
1992 * @param pu2State Our page state (input/output).
1993 * @param fBackingChanged Set if the page backing is being changed.
1994 * @thread EMT(pVCpu)
1995 */
1996NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
1997 uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
1998{
1999#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2000 /*
2001 * When using the hypercalls instead of the ring-3 APIs, we don't need to
2002 * unmap memory before modifying it. We still want to track the state though,
2003 * since unmap will fail when called an unmapped page and we don't want to redo
2004 * upgrades/downgrades.
2005 */
2006 uint8_t const u2OldState = *pu2State;
2007 int rc;
2008 if (fPageProt == NEM_PAGE_PROT_NONE)
2009 {
2010 if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
2011 {
2012 rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
2013 if (RT_SUCCESS(rc))
2014 {
2015 *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2016 uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2017 Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
2018 }
2019 else
2020 AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
2021 }
2022 else
2023 rc = VINF_SUCCESS;
2024 }
2025 else if (fPageProt & NEM_PAGE_PROT_WRITE)
2026 {
2027 if (u2OldState != NEM_WIN_PAGE_STATE_WRITABLE || fBackingChanged)
2028 {
2029 rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
2030 HV_MAP_GPA_READABLE | HV_MAP_GPA_WRITABLE
2031 | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
2032 if (RT_SUCCESS(rc))
2033 {
2034 *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
2035 uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
2036 ? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
2037 Log5(("NEM GPA writable/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
2038 NOREF(cMappedPages);
2039 }
2040 else
2041 AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
2042 }
2043 else
2044 rc = VINF_SUCCESS;
2045 }
2046 else
2047 {
2048 if (u2OldState != NEM_WIN_PAGE_STATE_READABLE || fBackingChanged)
2049 {
2050 rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
2051 HV_MAP_GPA_READABLE | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
2052 if (RT_SUCCESS(rc))
2053 {
2054 *pu2State = NEM_WIN_PAGE_STATE_READABLE;
2055 uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
2056 ? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
2057 Log5(("NEM GPA read+exec/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
2058 NOREF(cMappedPages);
2059 }
2060 else
2061 AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
2062 }
2063 else
2064 rc = VINF_SUCCESS;
2065 }
2066
2067 return VINF_SUCCESS;
2068
2069#else
2070 /*
2071 * Looks like we need to unmap a page before we can change the backing
2072 * or even modify the protection. This is going to be *REALLY* efficient.
2073 * PGM lends us two bits to keep track of the state here.
2074 */
2075 uint8_t const u2OldState = *pu2State;
2076 uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
2077 : fPageProt & NEM_PAGE_PROT_READ ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED;
2078 if ( fBackingChanged
2079 || u2NewState != u2OldState)
2080 {
2081 if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
2082 {
2083# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2084 int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
2085 AssertRC(rc);
2086 if (RT_SUCCESS(rc))
2087 {
2088 *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2089 uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2090 if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
2091 {
2092 Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
2093 GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
2094 return VINF_SUCCESS;
2095 }
2096 }
2097 else
2098 {
2099 LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
2100 return rc;
2101 }
2102# else
2103 HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE);
2104 if (SUCCEEDED(hrc))
2105 {
2106 *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2107 uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2108 if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
2109 {
2110 Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
2111 GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
2112 return VINF_SUCCESS;
2113 }
2114 }
2115 else
2116 {
2117 LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
2118 GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2119 return VERR_NEM_INIT_FAILED;
2120 }
2121# endif
2122 }
2123 }
2124
2125 /*
2126 * Writeable mapping?
2127 */
2128 if (fPageProt & NEM_PAGE_PROT_WRITE)
2129 {
2130# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2131 int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
2132 HV_MAP_GPA_READABLE | HV_MAP_GPA_WRITABLE
2133 | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
2134 AssertRC(rc);
2135 if (RT_SUCCESS(rc))
2136 {
2137 *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
2138 uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2139 Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
2140 GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
2141 return VINF_SUCCESS;
2142 }
2143 LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
2144 return rc;
2145# else
2146 void *pvPage;
2147 int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
2148 if (RT_SUCCESS(rc))
2149 {
2150 HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE,
2151 WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute | WHvMapGpaRangeFlagWrite);
2152 if (SUCCEEDED(hrc))
2153 {
2154 *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
2155 uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2156 Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
2157 GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
2158 return VINF_SUCCESS;
2159 }
2160 LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
2161 GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2162 return VERR_NEM_INIT_FAILED;
2163 }
2164 LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
2165 return rc;
2166# endif
2167 }
2168
2169 if (fPageProt & NEM_PAGE_PROT_READ)
2170 {
2171# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
2172 int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
2173 HV_MAP_GPA_READABLE | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_EXECUTABLE_AGAIN);
2174 AssertRC(rc);
2175 if (RT_SUCCESS(rc))
2176 {
2177 *pu2State = NEM_WIN_PAGE_STATE_READABLE;
2178 uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2179 Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
2180 GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
2181 return VINF_SUCCESS;
2182 }
2183 LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
2184 return rc;
2185# else
2186 const void *pvPage;
2187 int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
2188 if (RT_SUCCESS(rc))
2189 {
2190 HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE,
2191 WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute);
2192 if (SUCCEEDED(hrc))
2193 {
2194 *pu2State = NEM_WIN_PAGE_STATE_READABLE;
2195 uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2196 Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
2197 GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
2198 return VINF_SUCCESS;
2199 }
2200 LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
2201 GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2202 return VERR_NEM_INIT_FAILED;
2203 }
2204 LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
2205 return rc;
2206# endif
2207 }
2208
2209 /* We already unmapped it above. */
2210 *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2211 return VINF_SUCCESS;
2212#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
2213}
2214
2215
2216NEM_TMPL_STATIC int nemHCJustUnmapPageFromHyperV(PVM pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
2217{
2218 if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED)
2219 {
2220 Log5(("nemHCJustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst));
2221 *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2222 return VINF_SUCCESS;
2223 }
2224
2225#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
2226 PVMCPU pVCpu = VMMGetCpu(pVM);
2227 int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
2228 AssertRC(rc);
2229 if (RT_SUCCESS(rc))
2230 {
2231 uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2232 Log5(("NEM GPA unmapped/just: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[*pu2State], cMappedPages));
2233 *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2234 return VINF_SUCCESS;
2235 }
2236 LogRel(("nemHCJustUnmapPageFromHyperV/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
2237 return rc;
2238#else
2239 HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
2240 if (SUCCEEDED(hrc))
2241 {
2242 uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
2243 *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2244 Log5(("nemHCJustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
2245 return VINF_SUCCESS;
2246 }
2247 LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n",
2248 GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2249 return VERR_INTERNAL_ERROR_3;
2250#endif
2251}
2252
2253
2254int nemHCNativeNotifyPhysPageAllocated(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
2255 PGMPAGETYPE enmType, uint8_t *pu2State)
2256{
2257 Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
2258 GCPhys, HCPhys, fPageProt, enmType, *pu2State));
2259 RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
2260
2261 int rc;
2262#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
2263 PVMCPU pVCpu = VMMGetCpu(pVM);
2264 if ( pVM->nem.s.fA20Enabled
2265 || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
2266 rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
2267 else
2268 {
2269 /* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
2270 rc = nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys | RT_BIT_32(20));
2271 if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys) && RT_SUCCESS(rc))
2272 rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
2273
2274 }
2275#else
2276 RT_NOREF_PV(fPageProt);
2277 if ( pVM->nem.s.fA20Enabled
2278 || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
2279 rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
2280 else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
2281 rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
2282 else
2283 rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */
2284#endif
2285 return rc;
2286}
2287
2288
2289void nemHCNativeNotifyPhysPageProtChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
2290 PGMPAGETYPE enmType, uint8_t *pu2State)
2291{
2292 Log5(("nemHCNativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
2293 GCPhys, HCPhys, fPageProt, enmType, *pu2State));
2294 RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
2295
2296#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
2297 PVMCPU pVCpu = VMMGetCpu(pVM);
2298 if ( pVM->nem.s.fA20Enabled
2299 || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
2300 nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /*fBackingChanged*/);
2301 else
2302 {
2303 /* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
2304 nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys | RT_BIT_32(20));
2305 if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
2306 nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /*fBackingChanged*/);
2307 }
2308#else
2309 RT_NOREF_PV(fPageProt);
2310 if ( pVM->nem.s.fA20Enabled
2311 || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
2312 nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
2313 else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
2314 nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
2315 /* else: ignore since we've got the alias page at this address. */
2316#endif
2317}
2318
2319
2320void nemHCNativeNotifyPhysPageChanged(PVM pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
2321 uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
2322{
2323 Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
2324 GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
2325 RT_NOREF_PV(HCPhysPrev); RT_NOREF_PV(HCPhysNew); RT_NOREF_PV(enmType);
2326
2327#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) || defined(IN_RING0)
2328 PVMCPU pVCpu = VMMGetCpu(pVM);
2329 if ( pVM->nem.s.fA20Enabled
2330 || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
2331 nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
2332 else
2333 {
2334 /* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
2335 nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys | RT_BIT_32(20));
2336 if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
2337 nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /*fBackingChanged*/);
2338 }
2339#else
2340 RT_NOREF_PV(fPageProt);
2341 if ( pVM->nem.s.fA20Enabled
2342 || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
2343 nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
2344 else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
2345 nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
2346 /* else: ignore since we've got the alias page at this address. */
2347#endif
2348}
2349
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette