HMR0A.asm@ 87401

Last change on this file since 87401 was 87401, checked in by vboxsync, 4 years ago
VMM/HMVMX: Use wrgsbase and wrfsbase instead of wrmsr if we can when restoring the host GS and FS selectors. Align gdtr and idtr in hmR0VmxExportHostSegmentRegs. Don't check SUPKERNELFEATURES_GDT_NEED_WRITABLE or call SUPR0GetCurrentGdtRw just for VMX_RESTORE_HOST_GDTR, we only need that for restoring TR.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 48.0 KB

Line
1	; $Id: HMR0A.asm 87401 2021-01-23 02:45:29Z vboxsync $
2	;; @file
3	; HM - Ring-0 VMX, SVM world-switch and helper routines.
4	;
5
6	;
7	; Copyright (C) 2006-2020 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.virtualbox.org. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17
18	;*********************************************************************************************************************************
19	;* Header Files *
20	;*********************************************************************************************************************************
21	;%define RT_ASM_WITH_SEH64 - trouble with SEH, alignment and (probably) 2nd pass optimizations.
22	%include "VBox/asmdefs.mac"
23	%include "VBox/err.mac"
24	%include "VBox/vmm/hm_vmx.mac"
25	%include "VBox/vmm/cpum.mac"
26	%include "VBox/vmm/vm.mac"
27	%include "iprt/x86.mac"
28	%include "HMInternal.mac"
29
30	%ifndef RT_ARCH_AMD64
31	%error AMD64 only.
32	%endif
33
34
35	;*********************************************************************************************************************************
36	;* Defined Constants And Macros *
37	;*********************************************************************************************************************************
38	;; The offset of the XMM registers in X86FXSTATE.
39	; Use define because I'm too lazy to convert the struct.
40	%define XMM_OFF_IN_X86FXSTATE 160
41
42	;; Spectre filler for 64-bit mode.
43	; Choosen to be an invalid address (also with 5 level paging).
44	%define SPECTRE_FILLER 0x02204204207fffff
45
46	;;
47	; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation.
48	;
49	%define VMX_SKIP_GDTR
50	%define VMX_SKIP_TR
51	%define VBOX_SKIP_RESTORE_SEG
52	%ifdef RT_OS_DARWIN
53	; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
54	; risk loading a stale LDT value or something invalid.
55	%define HM_64_BIT_USE_NULL_SEL
56	; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always.
57	; See @bugref{6875}.
58	%else
59	%define VMX_SKIP_IDTR
60	%endif
61
62
63	;; @def CALLEE_PRESERVED_REGISTER_COUNT
64	; Number of registers pushed by PUSH_CALLEE_PRESERVED_REGISTERS
65	%ifdef ASM_CALL64_GCC
66	%define CALLEE_PRESERVED_REGISTER_COUNT 5
67	%else
68	%define CALLEE_PRESERVED_REGISTER_COUNT 7
69	%endif
70
71	;; @def PUSH_CALLEE_PRESERVED_REGISTERS
72	; Macro for pushing all GPRs we must preserve for the caller.
73	%macro PUSH_CALLEE_PRESERVED_REGISTERS 0
74	push r15
75	SEH64_PUSH_GREG r15
76	%assign cbFrame cbFrame + 8
77	%assign frm_saved_r15 -cbFrame
78
79	push r14
80	SEH64_PUSH_GREG r14
81	%assign cbFrame cbFrame + 8
82	%assign frm_saved_r14 -cbFrame
83
84	push r13
85	SEH64_PUSH_GREG r13
86	%assign cbFrame cbFrame + 8
87	%assign frm_saved_r13 -cbFrame
88
89	push r12
90	SEH64_PUSH_GREG r12
91	%assign cbFrame cbFrame + 8
92	%assign frm_saved_r12 -cbFrame
93
94	push rbx
95	SEH64_PUSH_GREG rbx
96	%assign cbFrame cbFrame + 8
97	%assign frm_saved_rbx -cbFrame
98
99	%ifdef ASM_CALL64_MSC
100	push rsi
101	SEH64_PUSH_GREG rsi
102	%assign cbFrame cbFrame + 8
103	%assign frm_saved_rsi -cbFrame
104
105	push rdi
106	SEH64_PUSH_GREG rdi
107	%assign cbFrame cbFrame + 8
108	%assign frm_saved_rdi -cbFrame
109	%endif
110	%endmacro
111
112	;; @def POP_CALLEE_PRESERVED_REGISTERS
113	; Counterpart to PUSH_CALLEE_PRESERVED_REGISTERS for use in the epilogue.
114	%macro POP_CALLEE_PRESERVED_REGISTERS 0
115	%ifdef ASM_CALL64_MSC
116	pop rdi
117	%assign cbFrame cbFrame - 8
118	%undef frm_saved_rdi
119
120	pop rsi
121	%assign cbFrame cbFrame - 8
122	%undef frm_saved_rsi
123	%endif
124	pop rbx
125	%assign cbFrame cbFrame - 8
126	%undef frm_saved_rbx
127
128	pop r12
129	%assign cbFrame cbFrame - 8
130	%undef frm_saved_r12
131
132	pop r13
133	%assign cbFrame cbFrame - 8
134	%undef frm_saved_r13
135
136	pop r14
137	%assign cbFrame cbFrame - 8
138	%undef frm_saved_r14
139
140	pop r15
141	%assign cbFrame cbFrame - 8
142	%undef frm_saved_r15
143	%endmacro
144
145	;; @def PUSH_RELEVANT_SEGMENT_REGISTERS
146	; Macro saving all segment registers on the stack.
147	; @param 1 Full width register name.
148	; @param 2 16-bit register name for \a 1.
149
150	;; @def POP_RELEVANT_SEGMENT_REGISTERS
151	; Macro restoring all segment registers on the stack.
152	; @param 1 Full width register name.
153	; @param 2 16-bit register name for \a 1.
154	%ifdef VBOX_SKIP_RESTORE_SEG
155	%macro PUSH_RELEVANT_SEGMENT_REGISTERS 2
156	%endmacro
157
158	%macro POP_RELEVANT_SEGMENT_REGISTERS 2
159	%endmacro
160	%else ; !VBOX_SKIP_RESTORE_SEG
161	; Trashes, rax, rdx & rcx.
162	%macro PUSH_RELEVANT_SEGMENT_REGISTERS 2
163	%ifndef HM_64_BIT_USE_NULL_SEL
164	mov %2, es
165	push %1
166	mov %2, ds
167	push %1
168	%endif
169
170	; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode,
171	; Solaris OTOH doesn't and we must save it.
172	mov ecx, MSR_K8_FS_BASE
173	rdmsr
174	push rdx
175	push rax
176	%ifndef HM_64_BIT_USE_NULL_SEL
177	push fs
178	%endif
179
180	; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel.
181	; The same happens on exit.
182	mov ecx, MSR_K8_GS_BASE
183	rdmsr
184	push rdx
185	push rax
186	%ifndef HM_64_BIT_USE_NULL_SEL
187	push gs
188	%endif
189	%endmacro
190
191	; trashes, rax, rdx & rcx
192	%macro POP_RELEVANT_SEGMENT_REGISTERS 2
193	; Note: do not step through this code with a debugger!
194	%ifndef HM_64_BIT_USE_NULL_SEL
195	xor eax, eax
196	mov ds, ax
197	mov es, ax
198	mov fs, ax
199	mov gs, ax
200	%endif
201
202	%ifndef HM_64_BIT_USE_NULL_SEL
203	pop gs
204	%endif
205	pop rax
206	pop rdx
207	mov ecx, MSR_K8_GS_BASE
208	wrmsr
209
210	%ifndef HM_64_BIT_USE_NULL_SEL
211	pop fs
212	%endif
213	pop rax
214	pop rdx
215	mov ecx, MSR_K8_FS_BASE
216	wrmsr
217	; Now it's safe to step again
218
219	%ifndef HM_64_BIT_USE_NULL_SEL
220	pop %1
221	mov ds, %2
222	pop %1
223	mov es, %2
224	%endif
225	%endmacro
226	%endif ; VBOX_SKIP_RESTORE_SEG
227
228
229	;;
230	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
231	; @clobbers eax, edx, ecx
232	; @param 1 How to address CPUMCTX.
233	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
234	%macro INDIRECT_BRANCH_PREDICTION_BARRIER_CTX 2
235	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
236	jz %%no_indirect_branch_barrier
237	mov ecx, MSR_IA32_PRED_CMD
238	mov eax, MSR_IA32_PRED_CMD_F_IBPB
239	xor edx, edx
240	wrmsr
241	%%no_indirect_branch_barrier:
242	%endmacro
243
244	;;
245	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
246	; @clobbers eax, edx, ecx
247	; @param 1 How to address VMCPU.
248	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
249	%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2
250	test byte [%1 + VMCPU.cpum.GstCtx + CPUMCTX.fWorldSwitcher], %2
251	jz %%no_indirect_branch_barrier
252	mov ecx, MSR_IA32_PRED_CMD
253	mov eax, MSR_IA32_PRED_CMD_F_IBPB
254	xor edx, edx
255	wrmsr
256	%%no_indirect_branch_barrier:
257	%endmacro
258
259	;;
260	; Creates an indirect branch prediction and L1D barrier on CPUs that need and supports that.
261	; @clobbers eax, edx, ecx
262	; @param 1 How to address CPUMCTX.
263	; @param 2 Which IBPB flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
264	; @param 3 Which FLUSH flag to test for (CPUMCTX_WSF_L1D_ENTRY)
265	; @param 4 Which MDS flag to test for (CPUMCTX_WSF_MDS_ENTRY)
266	%macro INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER 4
267	; Only one test+jmp when disabled CPUs.
268	test byte [%1 + CPUMCTX.fWorldSwitcher], (%2 \| %3 \| %4)
269	jz %%no_barrier_needed
270
271	; The eax:edx value is the same for both.
272	AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
273	mov eax, MSR_IA32_PRED_CMD_F_IBPB
274	xor edx, edx
275
276	; Indirect branch barrier.
277	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
278	jz %%no_indirect_branch_barrier
279	mov ecx, MSR_IA32_PRED_CMD
280	wrmsr
281	%%no_indirect_branch_barrier:
282
283	; Level 1 data cache flush.
284	test byte [%1 + CPUMCTX.fWorldSwitcher], %3
285	jz %%no_cache_flush_barrier
286	mov ecx, MSR_IA32_FLUSH_CMD
287	wrmsr
288	jmp %%no_mds_buffer_flushing ; MDS flushing is included in L1D_FLUSH
289	%%no_cache_flush_barrier:
290
291	; MDS buffer flushing.
292	test byte [%1 + CPUMCTX.fWorldSwitcher], %4
293	jz %%no_mds_buffer_flushing
294	sub xSP, xSP
295	mov [xSP], ds
296	verw [xSP]
297	add xSP, xSP
298	%%no_mds_buffer_flushing:
299
300	%%no_barrier_needed:
301	%endmacro
302
303
304	;*********************************************************************************************************************************
305	;* External Symbols *
306	;*********************************************************************************************************************************
307	%ifdef VBOX_WITH_KERNEL_USING_XMM
308	extern NAME(CPUMIsGuestFPUStateActive)
309	%endif
310
311
312	BEGINCODE
313
314
315	;;
316	; Restores host-state fields.
317	;
318	; @returns VBox status code
319	; @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags.
320	; @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct.
321	;
322	ALIGNCODE(64)
323	BEGINPROC VMXRestoreHostState
324	%ifndef ASM_CALL64_GCC
325	; Use GCC's input registers since we'll be needing both rcx and rdx further
326	; down with the wrmsr instruction. Use the R10 and R11 register for saving
327	; RDI and RSI since MSC preserve the two latter registers.
328	mov r10, rdi
329	mov r11, rsi
330	mov rdi, rcx
331	mov rsi, rdx
332	%endif
333	SEH64_END_PROLOGUE
334
335	.restore_gdtr
336	test edi, VMX_RESTORE_HOST_GDTR
337	jz .restore_idtr
338	lgdt [rsi + VMXRESTOREHOST.HostGdtr]
339
340	.restore_idtr:
341	test edi, VMX_RESTORE_HOST_IDTR
342	jz .restore_ds
343	lidt [rsi + VMXRESTOREHOST.HostIdtr]
344
345	.restore_ds:
346	test edi, VMX_RESTORE_HOST_SEL_DS
347	jz .restore_es
348	mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
349	mov ds, eax
350
351	.restore_es:
352	test edi, VMX_RESTORE_HOST_SEL_ES
353	jz .restore_tr
354	mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
355	mov es, eax
356
357	.restore_tr:
358	test edi, VMX_RESTORE_HOST_SEL_TR
359	jz .restore_fs
360	; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
361	mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
362	mov ax, dx
363	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
364	test edi, VMX_RESTORE_HOST_GDT_READ_ONLY \| VMX_RESTORE_HOST_GDT_NEED_WRITABLE
365	jnz .gdt_readonly
366	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
367	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
368	ltr dx
369	jmp short .restore_fs
370	.gdt_readonly:
371	test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
372	jnz .gdt_readonly_need_writable
373	mov rcx, cr0
374	mov r9, rcx
375	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
376	and rcx, ~X86_CR0_WP
377	mov cr0, rcx
378	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
379	ltr dx
380	mov cr0, r9
381	jmp short .restore_fs
382	.gdt_readonly_need_writable:
383	add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw
384	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
385	lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
386	ltr dx
387	lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT
388
389	.restore_fs:
390	;
391	; When restoring the selector values for FS and GS, we'll temporarily trash
392	; the base address (at least the high 32-bit bits, but quite possibly the
393	; whole base address), the wrmsr will restore it correctly. (VT-x actually
394	; restores the base correctly when leaving guest mode, but not the selector
395	; value, so there is little problem with interrupts being enabled prior to
396	; this restore job.)
397	; We'll disable ints once for both FS and GS as that's probably faster.
398	;
399	test edi, VMX_RESTORE_HOST_SEL_FS \| VMX_RESTORE_HOST_SEL_GS
400	jz .restore_success
401	pushfq
402	cli ; (see above)
403
404	test edi, VMX_RESTORE_HOST_CAN_USE_WRFSBASE_AND_WRGSBASE
405	jz .restore_fs_using_wrmsr
406
407	.restore_fs_using_wrfsbase:
408	test edi, VMX_RESTORE_HOST_SEL_FS
409	jz .restore_gs_using_wrgsbase
410	mov rax, qword [rsi + VMXRESTOREHOST.uHostFSBase]
411	mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
412	mov fs, ecx
413	wrfsbase rax
414
415	.restore_gs_using_wrgsbase:
416	test edi, VMX_RESTORE_HOST_SEL_GS
417	jz .restore_flags
418	mov rax, qword [rsi + VMXRESTOREHOST.uHostGSBase]
419	mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
420	mov gs, ecx
421	wrgsbase rax
422	jmp .restore_flags
423
424	.restore_fs_using_wrmsr:
425	test edi, VMX_RESTORE_HOST_SEL_FS
426	jz .restore_gs_using_wrmsr
427	mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
428	mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
429	mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
430	mov fs, ecx
431	mov ecx, MSR_K8_FS_BASE
432	wrmsr
433
434	.restore_gs_using_wrmsr:
435	test edi, VMX_RESTORE_HOST_SEL_GS
436	jz .restore_flags
437	mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
438	mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
439	mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
440	mov gs, ecx
441	mov ecx, MSR_K8_GS_BASE
442	wrmsr
443
444	.restore_flags:
445	popfq
446
447	.restore_success:
448	mov eax, VINF_SUCCESS
449	%ifndef ASM_CALL64_GCC
450	; Restore RDI and RSI on MSC.
451	mov rdi, r10
452	mov rsi, r11
453	%endif
454	ret
455	ENDPROC VMXRestoreHostState
456
457
458	;;
459	; Dispatches an NMI to the host.
460	;
461	ALIGNCODE(16)
462	BEGINPROC VMXDispatchHostNmi
463	; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
464	SEH64_END_PROLOGUE
465	int 2
466	ret
467	ENDPROC VMXDispatchHostNmi
468
469
470	%ifdef VBOX_WITH_KERNEL_USING_XMM
471	;;
472	; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
473	; load the guest ones when necessary.
474	;
475	; @cproto DECLASM(int) hmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVM pVM,
476	; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
477	;
478	; @returns eax
479	;
480	; @param fResumeVM msc:rcx
481	; @param pCtx msc:rdx
482	; @param pvUnused msc:r8
483	; @param pVM msc:r9
484	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
485	; @param pfnStartVM msc:[rbp+38h]
486	;
487	; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
488	;
489	; @remarks Drivers shouldn't use AVX registers without saving+loading:
490	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
491	; However the compiler docs have different idea:
492	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
493	; We'll go with the former for now.
494	;
495	; ASSUMING 64-bit and windows for now.
496	;
497	ALIGNCODE(16)
498	BEGINPROC hmR0VMXStartVMWrapXMM
499	SEH64_END_PROLOGUE
500	push xBP
501	mov xBP, xSP
502	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
503
504	; Spill input parameters.
505	mov [xBP + 010h], rcx ; fResumeVM
506	mov [xBP + 018h], rdx ; pCtx
507	mov [xBP + 020h], r8 ; pvUnused
508	mov [xBP + 028h], r9 ; pVM
509
510	; Ask CPUM whether we've started using the FPU yet.
511	mov rcx, [xBP + 30h] ; pVCpu
512	call NAME(CPUMIsGuestFPUStateActive)
513	test al, al
514	jnz .guest_fpu_state_active
515
516	; No need to mess with XMM registers just call the start routine and return.
517	mov r11, [xBP + 38h] ; pfnStartVM
518	mov r10, [xBP + 30h] ; pVCpu
519	mov [xSP + 020h], r10
520	mov rcx, [xBP + 010h] ; fResumeVM
521	mov rdx, [xBP + 018h] ; pCtx
522	mov r8, [xBP + 020h] ; pvUnused
523	mov r9, [xBP + 028h] ; pVM
524	call r11
525
526	leave
527	ret
528
529	ALIGNCODE(8)
530	.guest_fpu_state_active:
531	; Save the non-volatile host XMM registers.
532	movdqa [rsp + 040h + 000h], xmm6
533	movdqa [rsp + 040h + 010h], xmm7
534	movdqa [rsp + 040h + 020h], xmm8
535	movdqa [rsp + 040h + 030h], xmm9
536	movdqa [rsp + 040h + 040h], xmm10
537	movdqa [rsp + 040h + 050h], xmm11
538	movdqa [rsp + 040h + 060h], xmm12
539	movdqa [rsp + 040h + 070h], xmm13
540	movdqa [rsp + 040h + 080h], xmm14
541	movdqa [rsp + 040h + 090h], xmm15
542	stmxcsr [rsp + 040h + 0a0h]
543
544	mov r10, [xBP + 018h] ; pCtx
545	mov eax, [r10 + CPUMCTX.fXStateMask]
546	test eax, eax
547	jz .guest_fpu_state_manually
548
549	;
550	; Using XSAVE to load the guest XMM, YMM and ZMM registers.
551	;
552	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
553	xor edx, edx
554	mov r10, [r10 + CPUMCTX.pXStateR0]
555	xrstor [r10]
556
557	; Make the call (same as in the other case).
558	mov r11, [xBP + 38h] ; pfnStartVM
559	mov r10, [xBP + 30h] ; pVCpu
560	mov [xSP + 020h], r10
561	mov rcx, [xBP + 010h] ; fResumeVM
562	mov rdx, [xBP + 018h] ; pCtx
563	mov r8, [xBP + 020h] ; pvUnused
564	mov r9, [xBP + 028h] ; pVM
565	call r11
566
567	mov r11d, eax ; save return value (xsave below uses eax)
568
569	; Save the guest XMM registers.
570	mov r10, [xBP + 018h] ; pCtx
571	mov eax, [r10 + CPUMCTX.fXStateMask]
572	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
573	xor edx, edx
574	mov r10, [r10 + CPUMCTX.pXStateR0]
575	xsave [r10]
576
577	mov eax, r11d ; restore return value
578
579	.restore_non_volatile_host_xmm_regs:
580	; Load the non-volatile host XMM registers.
581	movdqa xmm6, [rsp + 040h + 000h]
582	movdqa xmm7, [rsp + 040h + 010h]
583	movdqa xmm8, [rsp + 040h + 020h]
584	movdqa xmm9, [rsp + 040h + 030h]
585	movdqa xmm10, [rsp + 040h + 040h]
586	movdqa xmm11, [rsp + 040h + 050h]
587	movdqa xmm12, [rsp + 040h + 060h]
588	movdqa xmm13, [rsp + 040h + 070h]
589	movdqa xmm14, [rsp + 040h + 080h]
590	movdqa xmm15, [rsp + 040h + 090h]
591	ldmxcsr [rsp + 040h + 0a0h]
592	leave
593	ret
594
595	;
596	; No XSAVE, load and save the guest XMM registers manually.
597	;
598	.guest_fpu_state_manually:
599	; Load the full guest XMM register state.
600	mov r10, [r10 + CPUMCTX.pXStateR0]
601	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
602	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
603	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
604	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
605	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
606	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
607	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
608	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
609	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
610	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
611	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
612	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
613	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
614	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
615	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
616	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
617	ldmxcsr [r10 + X86FXSTATE.MXCSR]
618
619	; Make the call (same as in the other case).
620	mov r11, [xBP + 38h] ; pfnStartVM
621	mov r10, [xBP + 30h] ; pVCpu
622	mov [xSP + 020h], r10
623	mov rcx, [xBP + 010h] ; fResumeVM
624	mov rdx, [xBP + 018h] ; pCtx
625	mov r8, [xBP + 020h] ; pvUnused
626	mov r9, [xBP + 028h] ; pVM
627	call r11
628
629	; Save the guest XMM registers.
630	mov r10, [xBP + 018h] ; pCtx
631	mov r10, [r10 + CPUMCTX.pXStateR0]
632	stmxcsr [r10 + X86FXSTATE.MXCSR]
633	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
634	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
635	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
636	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
637	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
638	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
639	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
640	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
641	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
642	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
643	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
644	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
645	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
646	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
647	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
648	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
649	jmp .restore_non_volatile_host_xmm_regs
650	ENDPROC hmR0VMXStartVMWrapXMM
651	%endif ; VBOX_WITH_KERNEL_USING_XMM
652
653
654	;; @def RESTORE_STATE_VM64
655	; Macro restoring essential host state and updating guest state
656	; for 64-bit host, 64-bit guest for VT-x.
657	;
658	%macro RESTORE_STATE_VM64 0
659	; Restore base and limit of the IDTR & GDTR.
660	%ifndef VMX_SKIP_IDTR
661	lidt [xSP]
662	add xSP, xCB * 2
663	%endif
664	%ifndef VMX_SKIP_GDTR
665	lgdt [xSP]
666	add xSP, xCB * 2
667	%endif
668
669	push xDI
670	%ifndef VMX_SKIP_TR
671	mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR)
672	%else
673	mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR)
674	%endif
675
676	mov qword [xDI + CPUMCTX.eax], rax
677	mov rax, SPECTRE_FILLER
678	mov qword [xDI + CPUMCTX.ebx], rbx
679	mov rbx, rax
680	mov qword [xDI + CPUMCTX.ecx], rcx
681	mov rcx, rax
682	mov qword [xDI + CPUMCTX.edx], rdx
683	mov rdx, rax
684	mov qword [xDI + CPUMCTX.esi], rsi
685	mov rsi, rax
686	mov qword [xDI + CPUMCTX.ebp], rbp
687	mov rbp, rax
688	mov qword [xDI + CPUMCTX.r8], r8
689	mov r8, rax
690	mov qword [xDI + CPUMCTX.r9], r9
691	mov r9, rax
692	mov qword [xDI + CPUMCTX.r10], r10
693	mov r10, rax
694	mov qword [xDI + CPUMCTX.r11], r11
695	mov r11, rax
696	mov qword [xDI + CPUMCTX.r12], r12
697	mov r12, rax
698	mov qword [xDI + CPUMCTX.r13], r13
699	mov r13, rax
700	mov qword [xDI + CPUMCTX.r14], r14
701	mov r14, rax
702	mov qword [xDI + CPUMCTX.r15], r15
703	mov r15, rax
704	mov rax, cr2
705	mov qword [xDI + CPUMCTX.cr2], rax
706
707	pop xAX ; The guest rdi we pushed above
708	mov qword [xDI + CPUMCTX.edi], rax
709
710	; Fight spectre.
711	INDIRECT_BRANCH_PREDICTION_BARRIER_CTX xDI, CPUMCTX_WSF_IBPB_EXIT
712
713	%ifndef VMX_SKIP_TR
714	; Restore TSS selector; must mark it as not busy before using ltr!
715	; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p).
716	; @todo get rid of sgdt
717	pop xBX ; Saved TR
718	sub xSP, xCB * 2
719	sgdt [xSP]
720	mov xAX, xBX
721	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
722	add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset
723	and dword [xAX + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
724	ltr bx
725	add xSP, xCB * 2
726	%endif
727
728	pop xAX ; Saved LDTR
729	cmp eax, 0
730	je %%skip_ldt_write64
731	lldt ax
732
733	%%skip_ldt_write64:
734	pop xSI ; pCtx (needed in rsi by the macros below)
735
736	; Restore segment registers.
737	POP_RELEVANT_SEGMENT_REGISTERS xAX, ax
738
739	; Restore the host XCR0 if necessary.
740	pop xCX
741	test ecx, ecx
742	jnz %%xcr0_after_skip
743	pop xAX
744	pop xDX
745	xsetbv ; ecx is already zero.
746	%%xcr0_after_skip:
747
748	; Restore general purpose registers.
749	POP_CALLEE_PRESERVED_REGISTERS
750	%endmacro
751
752
753	;;
754	; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
755	;
756	; @returns VBox status code
757	; @param fResume msc:rcx, gcc:rdi Whether to use vmlauch/vmresume.
758	; @param pCtx msc:rdx, gcc:rsi Pointer to the guest-CPU context.
759	; @param pvUnused msc:r8, gcc:rdx Unused argument.
760	; @param pVM msc:r9, gcc:rcx The cross context VM structure.
761	; @param pVCpu msc:[ebp+30], gcc:r8 The cross context virtual CPU structure of the calling EMT.
762	;
763	ALIGNCODE(16)
764	BEGINPROC VMXR0StartVM64
765	push xBP
766	mov xBP, xSP
767
768	pushf
769	cli
770
771	; Save all general purpose host registers.
772	%assign cbFrame 0
773	PUSH_CALLEE_PRESERVED_REGISTERS
774	SEH64_END_PROLOGUE
775
776	; First we have to save some final CPU context registers.
777	lea r10, [.vmlaunch64_done wrt rip]
778	mov rax, VMX_VMCS_HOST_RIP ; return address (too difficult to continue after VMLAUNCH?)
779	vmwrite rax, r10
780	; Note: ASSUMES success!
781
782	;
783	; Unify the input parameter registers.
784	;
785	%ifdef ASM_CALL64_GCC
786	; fResume already in rdi
787	; pCtx already in rsi
788	mov rbx, rdx ; pvUnused
789	%else
790	mov rdi, rcx ; fResume
791	mov rsi, rdx ; pCtx
792	mov rbx, r8 ; pvUnused
793	%endif
794
795	;
796	; Save the host XCR0 and load the guest one if necessary.
797	; Note! Trashes rdx and rcx.
798	;
799	%ifdef ASM_CALL64_MSC
800	mov rax, [xBP + 30h] ; pVCpu
801	%else
802	mov rax, r8 ; pVCpu
803	%endif
804	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
805	jz .xcr0_before_skip
806
807	xor ecx, ecx
808	xgetbv ; save the host one on the stack
809	push xDX
810	push xAX
811
812	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest one
813	mov edx, [xSI + CPUMCTX.aXcr + 4]
814	xor ecx, ecx ; paranoia
815	xsetbv
816
817	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
818	jmp .xcr0_before_done
819
820	.xcr0_before_skip:
821	push 3fh ; indicate that we need not
822	.xcr0_before_done:
823
824	;
825	; Save segment registers.
826	; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
827	;
828	PUSH_RELEVANT_SEGMENT_REGISTERS xAX, ax
829
830	; Save the pCtx pointer.
831	push xSI
832
833	; Save host LDTR.
834	xor eax, eax
835	sldt ax
836	push xAX
837
838	%ifndef VMX_SKIP_TR
839	; The host TR limit is reset to 0x67; save & restore it manually.
840	str eax
841	push xAX
842	%endif
843
844	%ifndef VMX_SKIP_GDTR
845	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
846	sub xSP, xCB * 2
847	sgdt [xSP]
848	%endif
849	%ifndef VMX_SKIP_IDTR
850	sub xSP, xCB * 2
851	sidt [xSP]
852	%endif
853
854	; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
855	mov rbx, qword [xSI + CPUMCTX.cr2]
856	mov rdx, cr2
857	cmp rbx, rdx
858	je .skip_cr2_write
859	mov cr2, rbx
860
861	.skip_cr2_write:
862	mov eax, VMX_VMCS_HOST_RSP
863	vmwrite xAX, xSP
864	; Note: ASSUMES success!
865	; Don't mess with ESP anymore!!!
866
867	; Fight spectre and similar.
868	INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY, CPUMCTX_WSF_L1D_ENTRY, CPUMCTX_WSF_MDS_ENTRY
869
870	; Load guest general purpose registers.
871	mov rax, qword [xSI + CPUMCTX.eax]
872	mov rbx, qword [xSI + CPUMCTX.ebx]
873	mov rcx, qword [xSI + CPUMCTX.ecx]
874	mov rdx, qword [xSI + CPUMCTX.edx]
875	mov rbp, qword [xSI + CPUMCTX.ebp]
876	mov r8, qword [xSI + CPUMCTX.r8]
877	mov r9, qword [xSI + CPUMCTX.r9]
878	mov r10, qword [xSI + CPUMCTX.r10]
879	mov r11, qword [xSI + CPUMCTX.r11]
880	mov r12, qword [xSI + CPUMCTX.r12]
881	mov r13, qword [xSI + CPUMCTX.r13]
882	mov r14, qword [xSI + CPUMCTX.r14]
883	mov r15, qword [xSI + CPUMCTX.r15]
884
885	; Resume or start VM?
886	cmp xDI, 0 ; fResume
887
888	; Load guest rdi & rsi.
889	mov rdi, qword [xSI + CPUMCTX.edi]
890	mov rsi, qword [xSI + CPUMCTX.esi]
891
892	je .vmlaunch64_launch
893
894	vmresume
895	jc near .vmxstart64_invalid_vmcs_ptr
896	jz near .vmxstart64_start_failed
897	jmp .vmlaunch64_done; ; here if vmresume detected a failure
898
899	.vmlaunch64_launch:
900	vmlaunch
901	jc near .vmxstart64_invalid_vmcs_ptr
902	jz near .vmxstart64_start_failed
903	jmp .vmlaunch64_done; ; here if vmlaunch detected a failure
904
905	ALIGNCODE(16)
906	.vmlaunch64_done:
907	RESTORE_STATE_VM64
908	mov eax, VINF_SUCCESS
909
910	.vmstart64_end:
911	popf
912	pop xBP
913	ret
914
915	.vmxstart64_invalid_vmcs_ptr:
916	RESTORE_STATE_VM64
917	mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
918	jmp .vmstart64_end
919
920	.vmxstart64_start_failed:
921	RESTORE_STATE_VM64
922	mov eax, VERR_VMX_UNABLE_TO_START_VM
923	jmp .vmstart64_end
924	ENDPROC VMXR0StartVM64
925
926
927	;;
928	; Clears the MDS buffers using VERW.
929	ALIGNCODE(16)
930	BEGINPROC hmR0MdsClear
931	SEH64_END_PROLOGUE
932	sub xSP, xCB
933	mov [xSP], ds
934	verw [xSP]
935	add xSP, xCB
936	ret
937	ENDPROC hmR0MdsClear
938
939
940	;;
941	; hmR0SvmVmRun template
942	;
943	; @param 1 The suffix of the variation.
944	; @param 2 fLoadSaveGuestXcr0 value
945	; @param 3 The CPUMCTX_WSF_IBPB_ENTRY + CPUMCTX_WSF_IBPB_EXIT value.
946	; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
947	; Drivers shouldn't use AVX registers without saving+loading:
948	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
949	; However the compiler docs have different idea:
950	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
951	; We'll go with the former for now.
952	;
953	%macro hmR0SvmVmRunTemplate 4
954
955	;;
956	; Prepares for and executes VMRUN (32-bit and 64-bit guests).
957	;
958	; @returns VBox status code
959	; @param pVM msc:rcx,gcc:rdi The cross context VM structure (unused).
960	; @param pVCpu msc:rdx,gcc:rsi The cross context virtual CPU structure of the calling EMT.
961	; @param HCPhysVmcb msc:r8, gcc:rdx Physical address of guest VMCB.
962	;
963	ALIGNCODE(64) ; This + immediate optimizations causes serious trouble for yasm and the SEH frames: prologue -28 bytes, must be <256
964	; So the SEH64_XXX stuff is currently not operational.
965	BEGINPROC RT_CONCAT(hmR0SvmVmRun,%1)
966	%ifdef VBOX_WITH_KERNEL_USING_XMM
967	%if %4 = 0
968	;
969	; The non-saving variant will currently check the two SSE preconditions and pick
970	; the right variant to continue with. Later we can see if we can't manage to
971	; move these decisions into hmR0SvmUpdateVmRunFunction().
972	;
973	%ifdef ASM_CALL64_MSC
974	test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
975	%else
976	test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
977	%endif
978	jz .save_xmm_no_need
979	%ifdef ASM_CALL64_MSC
980	cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
981	%else
982	cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
983	%endif
984	je RT_CONCAT3(hmR0SvmVmRun,%1,_SseManual)
985	jmp RT_CONCAT3(hmR0SvmVmRun,%1,_SseXSave)
986	.save_xmm_no_need:
987	%endif
988	%endif
989	push rbp
990	SEH64_PUSH_xBP
991	mov rbp, rsp
992	SEH64_SET_FRAME_xBP 0
993	pushf
994	%assign cbFrame 30h
995	%if %4 != 0
996	%assign cbFrame cbFrame + 16 * 11 ; Reserve space for 10x 128-bit XMM registers and MXCSR (32-bit)
997	%endif
998	%assign cbBaseFrame cbFrame
999	sub rsp, cbFrame - 8h ; We subtract 8 bytes for the above pushf
1000	SEH64_ALLOCATE_STACK cbFrame ; And we have CALLEE_PRESERVED_REGISTER_COUNT following it.
1001
1002	%define frm_fRFlags -008h
1003	%define frm_uHostXcr0 -018h ; 128-bit
1004	;%define frm_fNoRestoreXcr0 -020h ; Non-zero if we should skip XCR0 restoring.
1005	%define frm_pGstCtx -028h ; Where we stash guest CPU context for use after the vmrun.
1006	%define frm_HCPhysVmcbHost -030h ; Where we stash HCPhysVmcbHost for the vmload after vmrun.
1007	%if %4 != 0
1008	%define frm_saved_xmm6 -040h
1009	%define frm_saved_xmm7 -050h
1010	%define frm_saved_xmm8 -060h
1011	%define frm_saved_xmm9 -070h
1012	%define frm_saved_xmm10 -080h
1013	%define frm_saved_xmm11 -090h
1014	%define frm_saved_xmm12 -0a0h
1015	%define frm_saved_xmm13 -0b0h
1016	%define frm_saved_xmm14 -0c0h
1017	%define frm_saved_xmm15 -0d0h
1018	%define frm_saved_mxcsr -0e0h
1019	%endif
1020
1021	; Manual save and restore:
1022	; - General purpose registers except RIP, RSP, RAX
1023	;
1024	; Trashed:
1025	; - CR2 (we don't care)
1026	; - LDTR (reset to 0)
1027	; - DRx (presumably not changed at all)
1028	; - DR7 (reset to 0x400)
1029
1030	; Save all general purpose host registers.
1031	PUSH_CALLEE_PRESERVED_REGISTERS
1032	SEH64_END_PROLOGUE
1033	%if cbFrame != (cbBaseFrame + 8 * CALLEE_PRESERVED_REGISTER_COUNT)
1034	%error Bad cbFrame value
1035	%endif
1036
1037	; Shuffle parameter registers so that r8=HCPhysVmcb and rsi=pVCpu. (rdx & rcx will soon be trashed.)
1038	%ifdef ASM_CALL64_GCC
1039	mov r8, rdx ; Put HCPhysVmcb in r8 like on MSC as rdx is trashed below.
1040	%else
1041	mov rsi, rdx ; Put pVCpu in rsi like on GCC as rdx is trashed below.
1042	;mov rdi, rcx ; Put pVM in rdi like on GCC as rcx is trashed below.
1043	%endif
1044
1045	%ifdef VBOX_STRICT
1046	;
1047	; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
1048	;
1049	cmp byte [rsi + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], %2
1050	mov eax, VERR_SVM_VMRUN_PRECOND_0
1051	jne .failure_return
1052
1053	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fWorldSwitcher]
1054	and eax, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT
1055	cmp eax, %3
1056	mov eax, VERR_SVM_VMRUN_PRECOND_1
1057	jne .failure_return
1058
1059	%ifdef VBOX_WITH_KERNEL_USING_XMM
1060	mov eax, VERR_SVM_VMRUN_PRECOND_2
1061	test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1062	%if %4 = 0
1063	;jnz .failure_return
1064	%else
1065	jz .failure_return
1066
1067	mov eax, VERR_SVM_VMRUN_PRECOND_3
1068	cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1069	%if %4 = 1
1070	jne .failure_return
1071	%elif %4 = 2
1072	je .failure_return
1073	%else
1074	%error Invalid template parameter 4.
1075	%endif
1076	%endif
1077	%endif
1078	%endif ; VBOX_STRICT
1079
1080	%if %4 != 0
1081	; Save the non-volatile SSE host register state.
1082	movdqa [rbp + frm_saved_xmm6 ], xmm6
1083	movdqa [rbp + frm_saved_xmm7 ], xmm7
1084	movdqa [rbp + frm_saved_xmm8 ], xmm8
1085	movdqa [rbp + frm_saved_xmm9 ], xmm9
1086	movdqa [rbp + frm_saved_xmm10], xmm10
1087	movdqa [rbp + frm_saved_xmm11], xmm11
1088	movdqa [rbp + frm_saved_xmm12], xmm12
1089	movdqa [rbp + frm_saved_xmm13], xmm13
1090	movdqa [rbp + frm_saved_xmm14], xmm14
1091	movdqa [rbp + frm_saved_xmm15], xmm15
1092	stmxcsr [rbp + frm_saved_mxcsr]
1093
1094	; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx.
1095	mov rcx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.pXStateR0]
1096	%if %4 = 1 ; manual
1097	movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h]
1098	movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h]
1099	movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h]
1100	movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h]
1101	movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h]
1102	movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h]
1103	movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h]
1104	movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h]
1105	movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h]
1106	movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h]
1107	movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h]
1108	movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h]
1109	movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h]
1110	movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h]
1111	movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h]
1112	movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h]
1113	ldmxcsr [rcx + X86FXSTATE.MXCSR]
1114	%elif %4 = 2 ; use xrstor/xsave
1115	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
1116	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1117	xor edx, edx
1118	xrstor [rcx]
1119	%else
1120	%error invalid template parameter 4
1121	%endif
1122	%endif
1123
1124	%if %2 != 0
1125	; Save the host XCR0 and load the guest one if necessary.
1126	xor ecx, ecx
1127	xgetbv ; save the host XCR0 on the stack
1128	mov [rbp + frm_uHostXcr0 + 8], rdx
1129	mov [rbp + frm_uHostXcr0 ], rax
1130
1131	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr] ; load the guest XCR0
1132	mov edx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr + 4]
1133	xor ecx, ecx ; paranoia
1134	xsetbv
1135	%endif
1136
1137	; Save host fs, gs, sysenter msr etc.
1138	mov rax, [rsi + VMCPU.hm + HMCPU.u + HMCPUSVM.HCPhysVmcbHost]
1139	mov qword [rbp + frm_HCPhysVmcbHost], rax ; save for the vmload after vmrun
1140	lea rsi, [rsi + VMCPU.cpum.GstCtx]
1141	mov qword [rbp + frm_pGstCtx], rsi
1142	vmsave
1143
1144	%if %3 & CPUMCTX_WSF_IBPB_ENTRY
1145	; Fight spectre (trashes rax, rdx and rcx).
1146	mov ecx, MSR_IA32_PRED_CMD
1147	mov eax, MSR_IA32_PRED_CMD_F_IBPB
1148	xor edx, edx
1149	wrmsr
1150	%endif
1151
1152	; Setup rax for VMLOAD.
1153	mov rax, r8 ; HCPhysVmcb (64 bits physical address; take low dword only)
1154
1155	; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
1156	mov rbx, qword [rsi + CPUMCTX.ebx]
1157	mov rcx, qword [rsi + CPUMCTX.ecx]
1158	mov rdx, qword [rsi + CPUMCTX.edx]
1159	mov rdi, qword [rsi + CPUMCTX.edi]
1160	mov rbp, qword [rsi + CPUMCTX.ebp]
1161	mov r8, qword [rsi + CPUMCTX.r8]
1162	mov r9, qword [rsi + CPUMCTX.r9]
1163	mov r10, qword [rsi + CPUMCTX.r10]
1164	mov r11, qword [rsi + CPUMCTX.r11]
1165	mov r12, qword [rsi + CPUMCTX.r12]
1166	mov r13, qword [rsi + CPUMCTX.r13]
1167	mov r14, qword [rsi + CPUMCTX.r14]
1168	mov r15, qword [rsi + CPUMCTX.r15]
1169	mov rsi, qword [rsi + CPUMCTX.esi]
1170
1171	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1172	clgi
1173	sti
1174
1175	; Load guest FS, GS, Sysenter MSRs etc.
1176	vmload
1177
1178	; Run the VM.
1179	vmrun
1180
1181	; Save guest fs, gs, sysenter msr etc.
1182	vmsave
1183
1184	; Load host fs, gs, sysenter msr etc.
1185	mov rax, [rsp + cbFrame + frm_HCPhysVmcbHost] ; load HCPhysVmcbHost (rbp is not operational yet, thus rsp)
1186	vmload
1187
1188	; Set the global interrupt flag again, but execute cli to make sure IF=0.
1189	cli
1190	stgi
1191
1192	; Pop pVCpu (saved above) and save the guest GPRs (sans RSP and RAX).
1193	mov rax, [rsp + cbFrame + frm_pGstCtx] ; (rbp still not operational)
1194
1195	mov qword [rax + CPUMCTX.ebp], rbp
1196	lea rbp, [rsp + cbFrame]
1197	mov qword [rax + CPUMCTX.ecx], rcx
1198	mov rcx, SPECTRE_FILLER
1199	mov qword [rax + CPUMCTX.edx], rdx
1200	mov rdx, rcx
1201	mov qword [rax + CPUMCTX.r8], r8
1202	mov r8, rcx
1203	mov qword [rax + CPUMCTX.r9], r9
1204	mov r9, rcx
1205	mov qword [rax + CPUMCTX.r10], r10
1206	mov r10, rcx
1207	mov qword [rax + CPUMCTX.r11], r11
1208	mov r11, rcx
1209	mov qword [rax + CPUMCTX.edi], rdi
1210	%ifdef ASM_CALL64_MSC
1211	mov rdi, [rbp + frm_saved_rdi]
1212	%else
1213	mov rdi, rcx
1214	%endif
1215	mov qword [rax + CPUMCTX.esi], rsi
1216	%ifdef ASM_CALL64_MSC
1217	mov rsi, [rbp + frm_saved_rsi]
1218	%else
1219	mov rsi, rcx
1220	%endif
1221	mov qword [rax + CPUMCTX.ebx], rbx
1222	mov rbx, [rbp + frm_saved_rbx]
1223	mov qword [rax + CPUMCTX.r12], r12
1224	mov r12, [rbp + frm_saved_r12]
1225	mov qword [rax + CPUMCTX.r13], r13
1226	mov r13, [rbp + frm_saved_r13]
1227	mov qword [rax + CPUMCTX.r14], r14
1228	mov r14, [rbp + frm_saved_r14]
1229	mov qword [rax + CPUMCTX.r15], r15
1230	mov r15, [rbp + frm_saved_r15]
1231
1232	%if %4 != 0
1233	; Set r8 = &pVCpu->cpum.GstCtx; for use below when saving and restoring SSE state.
1234	mov r8, rax
1235	%endif
1236
1237	; Fight spectre. Note! Trashes rax, rdx and rcx!
1238	%if %3 & CPUMCTX_WSF_IBPB_EXIT
1239	; Fight spectre (trashes rax, rdx and rcx).
1240	mov ecx, MSR_IA32_PRED_CMD
1241	mov eax, MSR_IA32_PRED_CMD_F_IBPB
1242	xor edx, edx
1243	wrmsr
1244	%endif
1245
1246	%if %2 != 0
1247	; Restore the host xcr0.
1248	xor ecx, ecx
1249	mov rdx, [rbp + frm_uHostXcr0 + 8]
1250	mov rax, [rbp + frm_uHostXcr0]
1251	xsetbv
1252	%endif
1253
1254	%if %4 != 0
1255	; Save the guest SSE state related to non-volatile and volatile SSE registers.
1256	mov rcx, [r8 + CPUMCTX.pXStateR0]
1257	%if %4 = 1 ; manual
1258	stmxcsr [rcx + X86FXSTATE.MXCSR]
1259	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1260	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1261	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1262	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1263	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1264	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1265	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1266	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1267	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1268	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1269	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1270	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1271	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1272	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1273	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1274	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1275	%elif %4 = 2 ; use xrstor/xsave
1276	mov eax, [r8 + CPUMCTX.fXStateMask]
1277	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1278	xor edx, edx
1279	xsave [rcx]
1280	%else
1281	%error invalid template parameter 4
1282	%endif
1283
1284	; Restore the host non-volatile SSE register state.
1285	ldmxcsr [rbp + frm_saved_mxcsr]
1286	movdqa [rbp + frm_saved_xmm6 ], xmm6
1287	movdqa [rbp + frm_saved_xmm7 ], xmm7
1288	movdqa [rbp + frm_saved_xmm8 ], xmm8
1289	movdqa [rbp + frm_saved_xmm9 ], xmm9
1290	movdqa [rbp + frm_saved_xmm10], xmm10
1291	movdqa [rbp + frm_saved_xmm11], xmm11
1292	movdqa [rbp + frm_saved_xmm12], xmm12
1293	movdqa [rbp + frm_saved_xmm13], xmm13
1294	movdqa [rbp + frm_saved_xmm14], xmm14
1295	movdqa [rbp + frm_saved_xmm15], xmm15
1296	%endif ; %4 != 0
1297
1298	; Epilogue (assumes we restored volatile registers above when saving the guest GPRs).
1299	mov eax, VINF_SUCCESS
1300	add rsp, cbFrame - 8h
1301	popf
1302	leave
1303	ret
1304
1305	%ifdef VBOX_STRICT
1306	; Precondition checks failed.
1307	.failure_return:
1308	POP_CALLEE_PRESERVED_REGISTERS
1309	%if cbFrame != cbBaseFrame
1310	%error Bad frame size value: cbFrame
1311	%endif
1312	add rsp, cbFrame - 8h
1313	popf
1314	leave
1315	ret
1316	%endif
1317
1318	%undef frm_uHostXcr0
1319	%undef frm_fNoRestoreXcr0
1320	%undef frm_pVCpu
1321	%undef frm_HCPhysVmcbHost
1322	%undef cbFrame
1323	ENDPROC RT_CONCAT(hmR0SvmVmRun,%1)
1324
1325	%endmacro ; hmR0SvmVmRunTemplate
1326
1327	;
1328	; Instantiate the hmR0SvmVmRun various variations.
1329	;
1330	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit, 0, 0, 0
1331	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit, 1, 0, 0
1332	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY, 0
1333	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY, 0
1334	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_EXIT, 0
1335	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_EXIT, 0
1336	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 0
1337	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 0
1338	%ifdef VBOX_WITH_KERNEL_USING_XMM
1339	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 0, 0, 1
1340	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 1, 0, 1
1341	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_ENTRY, 1
1342	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_ENTRY, 1
1343	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_EXIT, 1
1344	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_EXIT, 1
1345	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 1
1346	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 1
1347
1348	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 0, 0, 2
1349	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 1, 0, 2
1350	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_ENTRY, 2
1351	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_ENTRY, 2
1352	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_EXIT, 2
1353	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_EXIT, 2
1354	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 2
1355	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 2
1356	%endif
1357

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 87401

Download in other formats: