HMR0A.asm@ 87390

Last change on this file since 87390 was 87385, checked in by vboxsync, 4 years ago
VMM/HMVMXR0: Put VMX_RESTORE_HOST_REQUIRED at the top to simplify check for calling VMXRestoreHostState.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 47.2 KB

Line
1	; $Id: HMR0A.asm 87385 2021-01-22 23:03:58Z vboxsync $
2	;; @file
3	; HM - Ring-0 VMX, SVM world-switch and helper routines.
4	;
5
6	;
7	; Copyright (C) 2006-2020 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.virtualbox.org. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17
18	;*********************************************************************************************************************************
19	;* Header Files *
20	;*********************************************************************************************************************************
21	;%define RT_ASM_WITH_SEH64 - trouble with SEH, alignment and (probably) 2nd pass optimizations.
22	%include "VBox/asmdefs.mac"
23	%include "VBox/err.mac"
24	%include "VBox/vmm/hm_vmx.mac"
25	%include "VBox/vmm/cpum.mac"
26	%include "VBox/vmm/vm.mac"
27	%include "iprt/x86.mac"
28	%include "HMInternal.mac"
29
30	%ifndef RT_ARCH_AMD64
31	%error AMD64 only.
32	%endif
33
34
35	;*********************************************************************************************************************************
36	;* Defined Constants And Macros *
37	;*********************************************************************************************************************************
38	;; The offset of the XMM registers in X86FXSTATE.
39	; Use define because I'm too lazy to convert the struct.
40	%define XMM_OFF_IN_X86FXSTATE 160
41
42	;; Spectre filler for 64-bit mode.
43	; Choosen to be an invalid address (also with 5 level paging).
44	%define SPECTRE_FILLER 0x02204204207fffff
45
46	;;
47	; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation.
48	;
49	%define VMX_SKIP_GDTR
50	%define VMX_SKIP_TR
51	%define VBOX_SKIP_RESTORE_SEG
52	%ifdef RT_OS_DARWIN
53	; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
54	; risk loading a stale LDT value or something invalid.
55	%define HM_64_BIT_USE_NULL_SEL
56	; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always.
57	; See @bugref{6875}.
58	%else
59	%define VMX_SKIP_IDTR
60	%endif
61
62
63	;; @def CALLEE_PRESERVED_REGISTER_COUNT
64	; Number of registers pushed by PUSH_CALLEE_PRESERVED_REGISTERS
65	%ifdef ASM_CALL64_GCC
66	%define CALLEE_PRESERVED_REGISTER_COUNT 5
67	%else
68	%define CALLEE_PRESERVED_REGISTER_COUNT 7
69	%endif
70
71	;; @def PUSH_CALLEE_PRESERVED_REGISTERS
72	; Macro for pushing all GPRs we must preserve for the caller.
73	%macro PUSH_CALLEE_PRESERVED_REGISTERS 0
74	push r15
75	SEH64_PUSH_GREG r15
76	%assign cbFrame cbFrame + 8
77	%assign frm_saved_r15 -cbFrame
78
79	push r14
80	SEH64_PUSH_GREG r14
81	%assign cbFrame cbFrame + 8
82	%assign frm_saved_r14 -cbFrame
83
84	push r13
85	SEH64_PUSH_GREG r13
86	%assign cbFrame cbFrame + 8
87	%assign frm_saved_r13 -cbFrame
88
89	push r12
90	SEH64_PUSH_GREG r12
91	%assign cbFrame cbFrame + 8
92	%assign frm_saved_r12 -cbFrame
93
94	push rbx
95	SEH64_PUSH_GREG rbx
96	%assign cbFrame cbFrame + 8
97	%assign frm_saved_rbx -cbFrame
98
99	%ifdef ASM_CALL64_MSC
100	push rsi
101	SEH64_PUSH_GREG rsi
102	%assign cbFrame cbFrame + 8
103	%assign frm_saved_rsi -cbFrame
104
105	push rdi
106	SEH64_PUSH_GREG rdi
107	%assign cbFrame cbFrame + 8
108	%assign frm_saved_rdi -cbFrame
109	%endif
110	%endmacro
111
112	;; @def POP_CALLEE_PRESERVED_REGISTERS
113	; Counterpart to PUSH_CALLEE_PRESERVED_REGISTERS for use in the epilogue.
114	%macro POP_CALLEE_PRESERVED_REGISTERS 0
115	%ifdef ASM_CALL64_MSC
116	pop rdi
117	%assign cbFrame cbFrame - 8
118	%undef frm_saved_rdi
119
120	pop rsi
121	%assign cbFrame cbFrame - 8
122	%undef frm_saved_rsi
123	%endif
124	pop rbx
125	%assign cbFrame cbFrame - 8
126	%undef frm_saved_rbx
127
128	pop r12
129	%assign cbFrame cbFrame - 8
130	%undef frm_saved_r12
131
132	pop r13
133	%assign cbFrame cbFrame - 8
134	%undef frm_saved_r13
135
136	pop r14
137	%assign cbFrame cbFrame - 8
138	%undef frm_saved_r14
139
140	pop r15
141	%assign cbFrame cbFrame - 8
142	%undef frm_saved_r15
143	%endmacro
144
145	;; @def PUSH_RELEVANT_SEGMENT_REGISTERS
146	; Macro saving all segment registers on the stack.
147	; @param 1 Full width register name.
148	; @param 2 16-bit register name for \a 1.
149
150	;; @def POP_RELEVANT_SEGMENT_REGISTERS
151	; Macro restoring all segment registers on the stack.
152	; @param 1 Full width register name.
153	; @param 2 16-bit register name for \a 1.
154	%ifdef VBOX_SKIP_RESTORE_SEG
155	%macro PUSH_RELEVANT_SEGMENT_REGISTERS 2
156	%endmacro
157
158	%macro POP_RELEVANT_SEGMENT_REGISTERS 2
159	%endmacro
160	%else ; !VBOX_SKIP_RESTORE_SEG
161	; Trashes, rax, rdx & rcx.
162	%macro PUSH_RELEVANT_SEGMENT_REGISTERS 2
163	%ifndef HM_64_BIT_USE_NULL_SEL
164	mov %2, es
165	push %1
166	mov %2, ds
167	push %1
168	%endif
169
170	; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode,
171	; Solaris OTOH doesn't and we must save it.
172	mov ecx, MSR_K8_FS_BASE
173	rdmsr
174	push rdx
175	push rax
176	%ifndef HM_64_BIT_USE_NULL_SEL
177	push fs
178	%endif
179
180	; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel.
181	; The same happens on exit.
182	mov ecx, MSR_K8_GS_BASE
183	rdmsr
184	push rdx
185	push rax
186	%ifndef HM_64_BIT_USE_NULL_SEL
187	push gs
188	%endif
189	%endmacro
190
191	; trashes, rax, rdx & rcx
192	%macro POP_RELEVANT_SEGMENT_REGISTERS 2
193	; Note: do not step through this code with a debugger!
194	%ifndef HM_64_BIT_USE_NULL_SEL
195	xor eax, eax
196	mov ds, ax
197	mov es, ax
198	mov fs, ax
199	mov gs, ax
200	%endif
201
202	%ifndef HM_64_BIT_USE_NULL_SEL
203	pop gs
204	%endif
205	pop rax
206	pop rdx
207	mov ecx, MSR_K8_GS_BASE
208	wrmsr
209
210	%ifndef HM_64_BIT_USE_NULL_SEL
211	pop fs
212	%endif
213	pop rax
214	pop rdx
215	mov ecx, MSR_K8_FS_BASE
216	wrmsr
217	; Now it's safe to step again
218
219	%ifndef HM_64_BIT_USE_NULL_SEL
220	pop %1
221	mov ds, %2
222	pop %1
223	mov es, %2
224	%endif
225	%endmacro
226	%endif ; VBOX_SKIP_RESTORE_SEG
227
228
229	;;
230	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
231	; @clobbers eax, edx, ecx
232	; @param 1 How to address CPUMCTX.
233	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
234	%macro INDIRECT_BRANCH_PREDICTION_BARRIER_CTX 2
235	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
236	jz %%no_indirect_branch_barrier
237	mov ecx, MSR_IA32_PRED_CMD
238	mov eax, MSR_IA32_PRED_CMD_F_IBPB
239	xor edx, edx
240	wrmsr
241	%%no_indirect_branch_barrier:
242	%endmacro
243
244	;;
245	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
246	; @clobbers eax, edx, ecx
247	; @param 1 How to address VMCPU.
248	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
249	%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2
250	test byte [%1 + VMCPU.cpum.GstCtx + CPUMCTX.fWorldSwitcher], %2
251	jz %%no_indirect_branch_barrier
252	mov ecx, MSR_IA32_PRED_CMD
253	mov eax, MSR_IA32_PRED_CMD_F_IBPB
254	xor edx, edx
255	wrmsr
256	%%no_indirect_branch_barrier:
257	%endmacro
258
259	;;
260	; Creates an indirect branch prediction and L1D barrier on CPUs that need and supports that.
261	; @clobbers eax, edx, ecx
262	; @param 1 How to address CPUMCTX.
263	; @param 2 Which IBPB flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
264	; @param 3 Which FLUSH flag to test for (CPUMCTX_WSF_L1D_ENTRY)
265	; @param 4 Which MDS flag to test for (CPUMCTX_WSF_MDS_ENTRY)
266	%macro INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER 4
267	; Only one test+jmp when disabled CPUs.
268	test byte [%1 + CPUMCTX.fWorldSwitcher], (%2 \| %3 \| %4)
269	jz %%no_barrier_needed
270
271	; The eax:edx value is the same for both.
272	AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
273	mov eax, MSR_IA32_PRED_CMD_F_IBPB
274	xor edx, edx
275
276	; Indirect branch barrier.
277	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
278	jz %%no_indirect_branch_barrier
279	mov ecx, MSR_IA32_PRED_CMD
280	wrmsr
281	%%no_indirect_branch_barrier:
282
283	; Level 1 data cache flush.
284	test byte [%1 + CPUMCTX.fWorldSwitcher], %3
285	jz %%no_cache_flush_barrier
286	mov ecx, MSR_IA32_FLUSH_CMD
287	wrmsr
288	jmp %%no_mds_buffer_flushing ; MDS flushing is included in L1D_FLUSH
289	%%no_cache_flush_barrier:
290
291	; MDS buffer flushing.
292	test byte [%1 + CPUMCTX.fWorldSwitcher], %4
293	jz %%no_mds_buffer_flushing
294	sub xSP, xSP
295	mov [xSP], ds
296	verw [xSP]
297	add xSP, xSP
298	%%no_mds_buffer_flushing:
299
300	%%no_barrier_needed:
301	%endmacro
302
303
304	;*********************************************************************************************************************************
305	;* External Symbols *
306	;*********************************************************************************************************************************
307	%ifdef VBOX_WITH_KERNEL_USING_XMM
308	extern NAME(CPUMIsGuestFPUStateActive)
309	%endif
310
311
312	BEGINCODE
313
314
315	;;
316	; Restores host-state fields.
317	;
318	; @returns VBox status code
319	; @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags.
320	; @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct.
321	;
322	ALIGNCODE(64)
323	BEGINPROC VMXRestoreHostState
324	%ifndef ASM_CALL64_GCC
325	; Use GCC's input registers since we'll be needing both rcx and rdx further
326	; down with the wrmsr instruction. Use the R10 and R11 register for saving
327	; RDI and RSI since MSC preserve the two latter registers.
328	mov r10, rdi
329	mov r11, rsi
330	mov rdi, rcx
331	mov rsi, rdx
332	%endif
333	SEH64_END_PROLOGUE
334
335	test edi, VMX_RESTORE_HOST_GDTR
336	jz .test_idtr
337	lgdt [rsi + VMXRESTOREHOST.HostGdtr]
338
339	.test_idtr:
340	test edi, VMX_RESTORE_HOST_IDTR
341	jz .test_ds
342	lidt [rsi + VMXRESTOREHOST.HostIdtr]
343
344	.test_ds:
345	test edi, VMX_RESTORE_HOST_SEL_DS
346	jz .test_es
347	mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
348	mov ds, eax
349
350	.test_es:
351	test edi, VMX_RESTORE_HOST_SEL_ES
352	jz .test_tr
353	mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
354	mov es, eax
355
356	.test_tr:
357	test edi, VMX_RESTORE_HOST_SEL_TR
358	jz .test_fs
359	; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
360	mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
361	mov ax, dx
362	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
363	test edi, VMX_RESTORE_HOST_GDT_READ_ONLY \| VMX_RESTORE_HOST_GDT_NEED_WRITABLE
364	jnz .gdt_readonly
365	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
366	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
367	ltr dx
368	jmp short .test_fs
369	.gdt_readonly:
370	test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
371	jnz .gdt_readonly_need_writable
372	mov rcx, cr0
373	mov r9, rcx
374	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
375	and rcx, ~X86_CR0_WP
376	mov cr0, rcx
377	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
378	ltr dx
379	mov cr0, r9
380	jmp short .test_fs
381	.gdt_readonly_need_writable:
382	add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw
383	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
384	lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
385	ltr dx
386	lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT
387
388	.test_fs:
389	;
390	; When restoring the selector values for FS and GS, we'll temporarily trash
391	; the base address (at least the high 32-bit bits, but quite possibly the
392	; whole base address), the wrmsr will restore it correctly. (VT-x actually
393	; restores the base correctly when leaving guest mode, but not the selector
394	; value, so there is little problem with interrupts being enabled prior to
395	; this restore job.)
396	; We'll disable ints once for both FS and GS as that's probably faster.
397	;
398	test edi, VMX_RESTORE_HOST_SEL_FS \| VMX_RESTORE_HOST_SEL_GS
399	jz .restore_success
400	pushfq
401	cli ; (see above)
402
403	test edi, VMX_RESTORE_HOST_SEL_FS
404	jz .test_gs
405	mov ax, word [rsi + VMXRESTOREHOST.uHostSelFS]
406	mov fs, eax
407	mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
408	mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
409	mov ecx, MSR_K8_FS_BASE
410	wrmsr
411
412	.test_gs:
413	test edi, VMX_RESTORE_HOST_SEL_GS
414	jz .restore_flags
415	mov ax, word [rsi + VMXRESTOREHOST.uHostSelGS]
416	mov gs, eax
417	mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
418	mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
419	mov ecx, MSR_K8_GS_BASE
420	wrmsr
421
422	.restore_flags:
423	popfq
424
425	.restore_success:
426	mov eax, VINF_SUCCESS
427	%ifndef ASM_CALL64_GCC
428	; Restore RDI and RSI on MSC.
429	mov rdi, r10
430	mov rsi, r11
431	%endif
432	ret
433	ENDPROC VMXRestoreHostState
434
435
436	;;
437	; Dispatches an NMI to the host.
438	;
439	ALIGNCODE(16)
440	BEGINPROC VMXDispatchHostNmi
441	; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
442	SEH64_END_PROLOGUE
443	int 2
444	ret
445	ENDPROC VMXDispatchHostNmi
446
447
448	%ifdef VBOX_WITH_KERNEL_USING_XMM
449	;;
450	; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
451	; load the guest ones when necessary.
452	;
453	; @cproto DECLASM(int) hmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVM pVM,
454	; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
455	;
456	; @returns eax
457	;
458	; @param fResumeVM msc:rcx
459	; @param pCtx msc:rdx
460	; @param pvUnused msc:r8
461	; @param pVM msc:r9
462	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
463	; @param pfnStartVM msc:[rbp+38h]
464	;
465	; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
466	;
467	; @remarks Drivers shouldn't use AVX registers without saving+loading:
468	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
469	; However the compiler docs have different idea:
470	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
471	; We'll go with the former for now.
472	;
473	; ASSUMING 64-bit and windows for now.
474	;
475	ALIGNCODE(16)
476	BEGINPROC hmR0VMXStartVMWrapXMM
477	SEH64_END_PROLOGUE
478	push xBP
479	mov xBP, xSP
480	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
481
482	; Spill input parameters.
483	mov [xBP + 010h], rcx ; fResumeVM
484	mov [xBP + 018h], rdx ; pCtx
485	mov [xBP + 020h], r8 ; pvUnused
486	mov [xBP + 028h], r9 ; pVM
487
488	; Ask CPUM whether we've started using the FPU yet.
489	mov rcx, [xBP + 30h] ; pVCpu
490	call NAME(CPUMIsGuestFPUStateActive)
491	test al, al
492	jnz .guest_fpu_state_active
493
494	; No need to mess with XMM registers just call the start routine and return.
495	mov r11, [xBP + 38h] ; pfnStartVM
496	mov r10, [xBP + 30h] ; pVCpu
497	mov [xSP + 020h], r10
498	mov rcx, [xBP + 010h] ; fResumeVM
499	mov rdx, [xBP + 018h] ; pCtx
500	mov r8, [xBP + 020h] ; pvUnused
501	mov r9, [xBP + 028h] ; pVM
502	call r11
503
504	leave
505	ret
506
507	ALIGNCODE(8)
508	.guest_fpu_state_active:
509	; Save the non-volatile host XMM registers.
510	movdqa [rsp + 040h + 000h], xmm6
511	movdqa [rsp + 040h + 010h], xmm7
512	movdqa [rsp + 040h + 020h], xmm8
513	movdqa [rsp + 040h + 030h], xmm9
514	movdqa [rsp + 040h + 040h], xmm10
515	movdqa [rsp + 040h + 050h], xmm11
516	movdqa [rsp + 040h + 060h], xmm12
517	movdqa [rsp + 040h + 070h], xmm13
518	movdqa [rsp + 040h + 080h], xmm14
519	movdqa [rsp + 040h + 090h], xmm15
520	stmxcsr [rsp + 040h + 0a0h]
521
522	mov r10, [xBP + 018h] ; pCtx
523	mov eax, [r10 + CPUMCTX.fXStateMask]
524	test eax, eax
525	jz .guest_fpu_state_manually
526
527	;
528	; Using XSAVE to load the guest XMM, YMM and ZMM registers.
529	;
530	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
531	xor edx, edx
532	mov r10, [r10 + CPUMCTX.pXStateR0]
533	xrstor [r10]
534
535	; Make the call (same as in the other case).
536	mov r11, [xBP + 38h] ; pfnStartVM
537	mov r10, [xBP + 30h] ; pVCpu
538	mov [xSP + 020h], r10
539	mov rcx, [xBP + 010h] ; fResumeVM
540	mov rdx, [xBP + 018h] ; pCtx
541	mov r8, [xBP + 020h] ; pvUnused
542	mov r9, [xBP + 028h] ; pVM
543	call r11
544
545	mov r11d, eax ; save return value (xsave below uses eax)
546
547	; Save the guest XMM registers.
548	mov r10, [xBP + 018h] ; pCtx
549	mov eax, [r10 + CPUMCTX.fXStateMask]
550	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
551	xor edx, edx
552	mov r10, [r10 + CPUMCTX.pXStateR0]
553	xsave [r10]
554
555	mov eax, r11d ; restore return value
556
557	.restore_non_volatile_host_xmm_regs:
558	; Load the non-volatile host XMM registers.
559	movdqa xmm6, [rsp + 040h + 000h]
560	movdqa xmm7, [rsp + 040h + 010h]
561	movdqa xmm8, [rsp + 040h + 020h]
562	movdqa xmm9, [rsp + 040h + 030h]
563	movdqa xmm10, [rsp + 040h + 040h]
564	movdqa xmm11, [rsp + 040h + 050h]
565	movdqa xmm12, [rsp + 040h + 060h]
566	movdqa xmm13, [rsp + 040h + 070h]
567	movdqa xmm14, [rsp + 040h + 080h]
568	movdqa xmm15, [rsp + 040h + 090h]
569	ldmxcsr [rsp + 040h + 0a0h]
570	leave
571	ret
572
573	;
574	; No XSAVE, load and save the guest XMM registers manually.
575	;
576	.guest_fpu_state_manually:
577	; Load the full guest XMM register state.
578	mov r10, [r10 + CPUMCTX.pXStateR0]
579	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
580	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
581	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
582	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
583	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
584	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
585	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
586	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
587	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
588	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
589	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
590	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
591	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
592	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
593	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
594	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
595	ldmxcsr [r10 + X86FXSTATE.MXCSR]
596
597	; Make the call (same as in the other case).
598	mov r11, [xBP + 38h] ; pfnStartVM
599	mov r10, [xBP + 30h] ; pVCpu
600	mov [xSP + 020h], r10
601	mov rcx, [xBP + 010h] ; fResumeVM
602	mov rdx, [xBP + 018h] ; pCtx
603	mov r8, [xBP + 020h] ; pvUnused
604	mov r9, [xBP + 028h] ; pVM
605	call r11
606
607	; Save the guest XMM registers.
608	mov r10, [xBP + 018h] ; pCtx
609	mov r10, [r10 + CPUMCTX.pXStateR0]
610	stmxcsr [r10 + X86FXSTATE.MXCSR]
611	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
612	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
613	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
614	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
615	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
616	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
617	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
618	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
619	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
620	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
621	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
622	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
623	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
624	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
625	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
626	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
627	jmp .restore_non_volatile_host_xmm_regs
628	ENDPROC hmR0VMXStartVMWrapXMM
629	%endif ; VBOX_WITH_KERNEL_USING_XMM
630
631
632	;; @def RESTORE_STATE_VM64
633	; Macro restoring essential host state and updating guest state
634	; for 64-bit host, 64-bit guest for VT-x.
635	;
636	%macro RESTORE_STATE_VM64 0
637	; Restore base and limit of the IDTR & GDTR.
638	%ifndef VMX_SKIP_IDTR
639	lidt [xSP]
640	add xSP, xCB * 2
641	%endif
642	%ifndef VMX_SKIP_GDTR
643	lgdt [xSP]
644	add xSP, xCB * 2
645	%endif
646
647	push xDI
648	%ifndef VMX_SKIP_TR
649	mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR)
650	%else
651	mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR)
652	%endif
653
654	mov qword [xDI + CPUMCTX.eax], rax
655	mov rax, SPECTRE_FILLER
656	mov qword [xDI + CPUMCTX.ebx], rbx
657	mov rbx, rax
658	mov qword [xDI + CPUMCTX.ecx], rcx
659	mov rcx, rax
660	mov qword [xDI + CPUMCTX.edx], rdx
661	mov rdx, rax
662	mov qword [xDI + CPUMCTX.esi], rsi
663	mov rsi, rax
664	mov qword [xDI + CPUMCTX.ebp], rbp
665	mov rbp, rax
666	mov qword [xDI + CPUMCTX.r8], r8
667	mov r8, rax
668	mov qword [xDI + CPUMCTX.r9], r9
669	mov r9, rax
670	mov qword [xDI + CPUMCTX.r10], r10
671	mov r10, rax
672	mov qword [xDI + CPUMCTX.r11], r11
673	mov r11, rax
674	mov qword [xDI + CPUMCTX.r12], r12
675	mov r12, rax
676	mov qword [xDI + CPUMCTX.r13], r13
677	mov r13, rax
678	mov qword [xDI + CPUMCTX.r14], r14
679	mov r14, rax
680	mov qword [xDI + CPUMCTX.r15], r15
681	mov r15, rax
682	mov rax, cr2
683	mov qword [xDI + CPUMCTX.cr2], rax
684
685	pop xAX ; The guest rdi we pushed above
686	mov qword [xDI + CPUMCTX.edi], rax
687
688	; Fight spectre.
689	INDIRECT_BRANCH_PREDICTION_BARRIER_CTX xDI, CPUMCTX_WSF_IBPB_EXIT
690
691	%ifndef VMX_SKIP_TR
692	; Restore TSS selector; must mark it as not busy before using ltr!
693	; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p).
694	; @todo get rid of sgdt
695	pop xBX ; Saved TR
696	sub xSP, xCB * 2
697	sgdt [xSP]
698	mov xAX, xBX
699	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
700	add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset
701	and dword [xAX + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
702	ltr bx
703	add xSP, xCB * 2
704	%endif
705
706	pop xAX ; Saved LDTR
707	cmp eax, 0
708	je %%skip_ldt_write64
709	lldt ax
710
711	%%skip_ldt_write64:
712	pop xSI ; pCtx (needed in rsi by the macros below)
713
714	; Restore segment registers.
715	POP_RELEVANT_SEGMENT_REGISTERS xAX, ax
716
717	; Restore the host XCR0 if necessary.
718	pop xCX
719	test ecx, ecx
720	jnz %%xcr0_after_skip
721	pop xAX
722	pop xDX
723	xsetbv ; ecx is already zero.
724	%%xcr0_after_skip:
725
726	; Restore general purpose registers.
727	POP_CALLEE_PRESERVED_REGISTERS
728	%endmacro
729
730
731	;;
732	; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
733	;
734	; @returns VBox status code
735	; @param fResume msc:rcx, gcc:rdi Whether to use vmlauch/vmresume.
736	; @param pCtx msc:rdx, gcc:rsi Pointer to the guest-CPU context.
737	; @param pvUnused msc:r8, gcc:rdx Unused argument.
738	; @param pVM msc:r9, gcc:rcx The cross context VM structure.
739	; @param pVCpu msc:[ebp+30], gcc:r8 The cross context virtual CPU structure of the calling EMT.
740	;
741	ALIGNCODE(16)
742	BEGINPROC VMXR0StartVM64
743	push xBP
744	mov xBP, xSP
745
746	pushf
747	cli
748
749	; Save all general purpose host registers.
750	%assign cbFrame 0
751	PUSH_CALLEE_PRESERVED_REGISTERS
752	SEH64_END_PROLOGUE
753
754	; First we have to save some final CPU context registers.
755	lea r10, [.vmlaunch64_done wrt rip]
756	mov rax, VMX_VMCS_HOST_RIP ; return address (too difficult to continue after VMLAUNCH?)
757	vmwrite rax, r10
758	; Note: ASSUMES success!
759
760	;
761	; Unify the input parameter registers.
762	;
763	%ifdef ASM_CALL64_GCC
764	; fResume already in rdi
765	; pCtx already in rsi
766	mov rbx, rdx ; pvUnused
767	%else
768	mov rdi, rcx ; fResume
769	mov rsi, rdx ; pCtx
770	mov rbx, r8 ; pvUnused
771	%endif
772
773	;
774	; Save the host XCR0 and load the guest one if necessary.
775	; Note! Trashes rdx and rcx.
776	;
777	%ifdef ASM_CALL64_MSC
778	mov rax, [xBP + 30h] ; pVCpu
779	%else
780	mov rax, r8 ; pVCpu
781	%endif
782	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
783	jz .xcr0_before_skip
784
785	xor ecx, ecx
786	xgetbv ; save the host one on the stack
787	push xDX
788	push xAX
789
790	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest one
791	mov edx, [xSI + CPUMCTX.aXcr + 4]
792	xor ecx, ecx ; paranoia
793	xsetbv
794
795	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
796	jmp .xcr0_before_done
797
798	.xcr0_before_skip:
799	push 3fh ; indicate that we need not
800	.xcr0_before_done:
801
802	;
803	; Save segment registers.
804	; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
805	;
806	PUSH_RELEVANT_SEGMENT_REGISTERS xAX, ax
807
808	; Save the pCtx pointer.
809	push xSI
810
811	; Save host LDTR.
812	xor eax, eax
813	sldt ax
814	push xAX
815
816	%ifndef VMX_SKIP_TR
817	; The host TR limit is reset to 0x67; save & restore it manually.
818	str eax
819	push xAX
820	%endif
821
822	%ifndef VMX_SKIP_GDTR
823	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
824	sub xSP, xCB * 2
825	sgdt [xSP]
826	%endif
827	%ifndef VMX_SKIP_IDTR
828	sub xSP, xCB * 2
829	sidt [xSP]
830	%endif
831
832	; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
833	mov rbx, qword [xSI + CPUMCTX.cr2]
834	mov rdx, cr2
835	cmp rbx, rdx
836	je .skip_cr2_write
837	mov cr2, rbx
838
839	.skip_cr2_write:
840	mov eax, VMX_VMCS_HOST_RSP
841	vmwrite xAX, xSP
842	; Note: ASSUMES success!
843	; Don't mess with ESP anymore!!!
844
845	; Fight spectre and similar.
846	INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY, CPUMCTX_WSF_L1D_ENTRY, CPUMCTX_WSF_MDS_ENTRY
847
848	; Load guest general purpose registers.
849	mov rax, qword [xSI + CPUMCTX.eax]
850	mov rbx, qword [xSI + CPUMCTX.ebx]
851	mov rcx, qword [xSI + CPUMCTX.ecx]
852	mov rdx, qword [xSI + CPUMCTX.edx]
853	mov rbp, qword [xSI + CPUMCTX.ebp]
854	mov r8, qword [xSI + CPUMCTX.r8]
855	mov r9, qword [xSI + CPUMCTX.r9]
856	mov r10, qword [xSI + CPUMCTX.r10]
857	mov r11, qword [xSI + CPUMCTX.r11]
858	mov r12, qword [xSI + CPUMCTX.r12]
859	mov r13, qword [xSI + CPUMCTX.r13]
860	mov r14, qword [xSI + CPUMCTX.r14]
861	mov r15, qword [xSI + CPUMCTX.r15]
862
863	; Resume or start VM?
864	cmp xDI, 0 ; fResume
865
866	; Load guest rdi & rsi.
867	mov rdi, qword [xSI + CPUMCTX.edi]
868	mov rsi, qword [xSI + CPUMCTX.esi]
869
870	je .vmlaunch64_launch
871
872	vmresume
873	jc near .vmxstart64_invalid_vmcs_ptr
874	jz near .vmxstart64_start_failed
875	jmp .vmlaunch64_done; ; here if vmresume detected a failure
876
877	.vmlaunch64_launch:
878	vmlaunch
879	jc near .vmxstart64_invalid_vmcs_ptr
880	jz near .vmxstart64_start_failed
881	jmp .vmlaunch64_done; ; here if vmlaunch detected a failure
882
883	ALIGNCODE(16)
884	.vmlaunch64_done:
885	RESTORE_STATE_VM64
886	mov eax, VINF_SUCCESS
887
888	.vmstart64_end:
889	popf
890	pop xBP
891	ret
892
893	.vmxstart64_invalid_vmcs_ptr:
894	RESTORE_STATE_VM64
895	mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
896	jmp .vmstart64_end
897
898	.vmxstart64_start_failed:
899	RESTORE_STATE_VM64
900	mov eax, VERR_VMX_UNABLE_TO_START_VM
901	jmp .vmstart64_end
902	ENDPROC VMXR0StartVM64
903
904
905	;;
906	; Clears the MDS buffers using VERW.
907	ALIGNCODE(16)
908	BEGINPROC hmR0MdsClear
909	SEH64_END_PROLOGUE
910	sub xSP, xCB
911	mov [xSP], ds
912	verw [xSP]
913	add xSP, xCB
914	ret
915	ENDPROC hmR0MdsClear
916
917
918	;;
919	; hmR0SvmVmRun template
920	;
921	; @param 1 The suffix of the variation.
922	; @param 2 fLoadSaveGuestXcr0 value
923	; @param 3 The CPUMCTX_WSF_IBPB_ENTRY + CPUMCTX_WSF_IBPB_EXIT value.
924	; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
925	; Drivers shouldn't use AVX registers without saving+loading:
926	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
927	; However the compiler docs have different idea:
928	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
929	; We'll go with the former for now.
930	;
931	%macro hmR0SvmVmRunTemplate 4
932
933	;;
934	; Prepares for and executes VMRUN (32-bit and 64-bit guests).
935	;
936	; @returns VBox status code
937	; @param pVM msc:rcx,gcc:rdi The cross context VM structure (unused).
938	; @param pVCpu msc:rdx,gcc:rsi The cross context virtual CPU structure of the calling EMT.
939	; @param HCPhysVmcb msc:r8, gcc:rdx Physical address of guest VMCB.
940	;
941	ALIGNCODE(64) ; This + immediate optimizations causes serious trouble for yasm and the SEH frames: prologue -28 bytes, must be <256
942	; So the SEH64_XXX stuff is currently not operational.
943	BEGINPROC RT_CONCAT(hmR0SvmVmRun,%1)
944	%ifdef VBOX_WITH_KERNEL_USING_XMM
945	%if %4 = 0
946	;
947	; The non-saving variant will currently check the two SSE preconditions and pick
948	; the right variant to continue with. Later we can see if we can't manage to
949	; move these decisions into hmR0SvmUpdateVmRunFunction().
950	;
951	%ifdef ASM_CALL64_MSC
952	test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
953	%else
954	test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
955	%endif
956	jz .save_xmm_no_need
957	%ifdef ASM_CALL64_MSC
958	cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
959	%else
960	cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
961	%endif
962	je RT_CONCAT3(hmR0SvmVmRun,%1,_SseManual)
963	jmp RT_CONCAT3(hmR0SvmVmRun,%1,_SseXSave)
964	.save_xmm_no_need:
965	%endif
966	%endif
967	push rbp
968	SEH64_PUSH_xBP
969	mov rbp, rsp
970	SEH64_SET_FRAME_xBP 0
971	pushf
972	%assign cbFrame 30h
973	%if %4 != 0
974	%assign cbFrame cbFrame + 16 * 11 ; Reserve space for 10x 128-bit XMM registers and MXCSR (32-bit)
975	%endif
976	%assign cbBaseFrame cbFrame
977	sub rsp, cbFrame - 8h ; We subtract 8 bytes for the above pushf
978	SEH64_ALLOCATE_STACK cbFrame ; And we have CALLEE_PRESERVED_REGISTER_COUNT following it.
979
980	%define frm_fRFlags -008h
981	%define frm_uHostXcr0 -018h ; 128-bit
982	;%define frm_fNoRestoreXcr0 -020h ; Non-zero if we should skip XCR0 restoring.
983	%define frm_pGstCtx -028h ; Where we stash guest CPU context for use after the vmrun.
984	%define frm_HCPhysVmcbHost -030h ; Where we stash HCPhysVmcbHost for the vmload after vmrun.
985	%if %4 != 0
986	%define frm_saved_xmm6 -040h
987	%define frm_saved_xmm7 -050h
988	%define frm_saved_xmm8 -060h
989	%define frm_saved_xmm9 -070h
990	%define frm_saved_xmm10 -080h
991	%define frm_saved_xmm11 -090h
992	%define frm_saved_xmm12 -0a0h
993	%define frm_saved_xmm13 -0b0h
994	%define frm_saved_xmm14 -0c0h
995	%define frm_saved_xmm15 -0d0h
996	%define frm_saved_mxcsr -0e0h
997	%endif
998
999	; Manual save and restore:
1000	; - General purpose registers except RIP, RSP, RAX
1001	;
1002	; Trashed:
1003	; - CR2 (we don't care)
1004	; - LDTR (reset to 0)
1005	; - DRx (presumably not changed at all)
1006	; - DR7 (reset to 0x400)
1007
1008	; Save all general purpose host registers.
1009	PUSH_CALLEE_PRESERVED_REGISTERS
1010	SEH64_END_PROLOGUE
1011	%if cbFrame != (cbBaseFrame + 8 * CALLEE_PRESERVED_REGISTER_COUNT)
1012	%error Bad cbFrame value
1013	%endif
1014
1015	; Shuffle parameter registers so that r8=HCPhysVmcb and rsi=pVCpu. (rdx & rcx will soon be trashed.)
1016	%ifdef ASM_CALL64_GCC
1017	mov r8, rdx ; Put HCPhysVmcb in r8 like on MSC as rdx is trashed below.
1018	%else
1019	mov rsi, rdx ; Put pVCpu in rsi like on GCC as rdx is trashed below.
1020	;mov rdi, rcx ; Put pVM in rdi like on GCC as rcx is trashed below.
1021	%endif
1022
1023	%ifdef VBOX_STRICT
1024	;
1025	; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
1026	;
1027	cmp byte [rsi + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], %2
1028	mov eax, VERR_SVM_VMRUN_PRECOND_0
1029	jne .failure_return
1030
1031	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fWorldSwitcher]
1032	and eax, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT
1033	cmp eax, %3
1034	mov eax, VERR_SVM_VMRUN_PRECOND_1
1035	jne .failure_return
1036
1037	%ifdef VBOX_WITH_KERNEL_USING_XMM
1038	mov eax, VERR_SVM_VMRUN_PRECOND_2
1039	test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1040	%if %4 = 0
1041	;jnz .failure_return
1042	%else
1043	jz .failure_return
1044
1045	mov eax, VERR_SVM_VMRUN_PRECOND_3
1046	cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1047	%if %4 = 1
1048	jne .failure_return
1049	%elif %4 = 2
1050	je .failure_return
1051	%else
1052	%error Invalid template parameter 4.
1053	%endif
1054	%endif
1055	%endif
1056	%endif ; VBOX_STRICT
1057
1058	%if %4 != 0
1059	; Save the non-volatile SSE host register state.
1060	movdqa [rbp + frm_saved_xmm6 ], xmm6
1061	movdqa [rbp + frm_saved_xmm7 ], xmm7
1062	movdqa [rbp + frm_saved_xmm8 ], xmm8
1063	movdqa [rbp + frm_saved_xmm9 ], xmm9
1064	movdqa [rbp + frm_saved_xmm10], xmm10
1065	movdqa [rbp + frm_saved_xmm11], xmm11
1066	movdqa [rbp + frm_saved_xmm12], xmm12
1067	movdqa [rbp + frm_saved_xmm13], xmm13
1068	movdqa [rbp + frm_saved_xmm14], xmm14
1069	movdqa [rbp + frm_saved_xmm15], xmm15
1070	stmxcsr [rbp + frm_saved_mxcsr]
1071
1072	; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx.
1073	mov rcx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.pXStateR0]
1074	%if %4 = 1 ; manual
1075	movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h]
1076	movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h]
1077	movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h]
1078	movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h]
1079	movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h]
1080	movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h]
1081	movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h]
1082	movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h]
1083	movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h]
1084	movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h]
1085	movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h]
1086	movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h]
1087	movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h]
1088	movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h]
1089	movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h]
1090	movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h]
1091	ldmxcsr [rcx + X86FXSTATE.MXCSR]
1092	%elif %4 = 2 ; use xrstor/xsave
1093	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
1094	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1095	xor edx, edx
1096	xrstor [rcx]
1097	%else
1098	%error invalid template parameter 4
1099	%endif
1100	%endif
1101
1102	%if %2 != 0
1103	; Save the host XCR0 and load the guest one if necessary.
1104	xor ecx, ecx
1105	xgetbv ; save the host XCR0 on the stack
1106	mov [rbp + frm_uHostXcr0 + 8], rdx
1107	mov [rbp + frm_uHostXcr0 ], rax
1108
1109	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr] ; load the guest XCR0
1110	mov edx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr + 4]
1111	xor ecx, ecx ; paranoia
1112	xsetbv
1113	%endif
1114
1115	; Save host fs, gs, sysenter msr etc.
1116	mov rax, [rsi + VMCPU.hm + HMCPU.u + HMCPUSVM.HCPhysVmcbHost]
1117	mov qword [rbp + frm_HCPhysVmcbHost], rax ; save for the vmload after vmrun
1118	lea rsi, [rsi + VMCPU.cpum.GstCtx]
1119	mov qword [rbp + frm_pGstCtx], rsi
1120	vmsave
1121
1122	%if %3 & CPUMCTX_WSF_IBPB_ENTRY
1123	; Fight spectre (trashes rax, rdx and rcx).
1124	mov ecx, MSR_IA32_PRED_CMD
1125	mov eax, MSR_IA32_PRED_CMD_F_IBPB
1126	xor edx, edx
1127	wrmsr
1128	%endif
1129
1130	; Setup rax for VMLOAD.
1131	mov rax, r8 ; HCPhysVmcb (64 bits physical address; take low dword only)
1132
1133	; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
1134	mov rbx, qword [rsi + CPUMCTX.ebx]
1135	mov rcx, qword [rsi + CPUMCTX.ecx]
1136	mov rdx, qword [rsi + CPUMCTX.edx]
1137	mov rdi, qword [rsi + CPUMCTX.edi]
1138	mov rbp, qword [rsi + CPUMCTX.ebp]
1139	mov r8, qword [rsi + CPUMCTX.r8]
1140	mov r9, qword [rsi + CPUMCTX.r9]
1141	mov r10, qword [rsi + CPUMCTX.r10]
1142	mov r11, qword [rsi + CPUMCTX.r11]
1143	mov r12, qword [rsi + CPUMCTX.r12]
1144	mov r13, qword [rsi + CPUMCTX.r13]
1145	mov r14, qword [rsi + CPUMCTX.r14]
1146	mov r15, qword [rsi + CPUMCTX.r15]
1147	mov rsi, qword [rsi + CPUMCTX.esi]
1148
1149	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1150	clgi
1151	sti
1152
1153	; Load guest FS, GS, Sysenter MSRs etc.
1154	vmload
1155
1156	; Run the VM.
1157	vmrun
1158
1159	; Save guest fs, gs, sysenter msr etc.
1160	vmsave
1161
1162	; Load host fs, gs, sysenter msr etc.
1163	mov rax, [rsp + cbFrame + frm_HCPhysVmcbHost] ; load HCPhysVmcbHost (rbp is not operational yet, thus rsp)
1164	vmload
1165
1166	; Set the global interrupt flag again, but execute cli to make sure IF=0.
1167	cli
1168	stgi
1169
1170	; Pop pVCpu (saved above) and save the guest GPRs (sans RSP and RAX).
1171	mov rax, [rsp + cbFrame + frm_pGstCtx] ; (rbp still not operational)
1172
1173	mov qword [rax + CPUMCTX.ebp], rbp
1174	lea rbp, [rsp + cbFrame]
1175	mov qword [rax + CPUMCTX.ecx], rcx
1176	mov rcx, SPECTRE_FILLER
1177	mov qword [rax + CPUMCTX.edx], rdx
1178	mov rdx, rcx
1179	mov qword [rax + CPUMCTX.r8], r8
1180	mov r8, rcx
1181	mov qword [rax + CPUMCTX.r9], r9
1182	mov r9, rcx
1183	mov qword [rax + CPUMCTX.r10], r10
1184	mov r10, rcx
1185	mov qword [rax + CPUMCTX.r11], r11
1186	mov r11, rcx
1187	mov qword [rax + CPUMCTX.edi], rdi
1188	%ifdef ASM_CALL64_MSC
1189	mov rdi, [rbp + frm_saved_rdi]
1190	%else
1191	mov rdi, rcx
1192	%endif
1193	mov qword [rax + CPUMCTX.esi], rsi
1194	%ifdef ASM_CALL64_MSC
1195	mov rsi, [rbp + frm_saved_rsi]
1196	%else
1197	mov rsi, rcx
1198	%endif
1199	mov qword [rax + CPUMCTX.ebx], rbx
1200	mov rbx, [rbp + frm_saved_rbx]
1201	mov qword [rax + CPUMCTX.r12], r12
1202	mov r12, [rbp + frm_saved_r12]
1203	mov qword [rax + CPUMCTX.r13], r13
1204	mov r13, [rbp + frm_saved_r13]
1205	mov qword [rax + CPUMCTX.r14], r14
1206	mov r14, [rbp + frm_saved_r14]
1207	mov qword [rax + CPUMCTX.r15], r15
1208	mov r15, [rbp + frm_saved_r15]
1209
1210	%if %4 != 0
1211	; Set r8 = &pVCpu->cpum.GstCtx; for use below when saving and restoring SSE state.
1212	mov r8, rax
1213	%endif
1214
1215	; Fight spectre. Note! Trashes rax, rdx and rcx!
1216	%if %3 & CPUMCTX_WSF_IBPB_EXIT
1217	; Fight spectre (trashes rax, rdx and rcx).
1218	mov ecx, MSR_IA32_PRED_CMD
1219	mov eax, MSR_IA32_PRED_CMD_F_IBPB
1220	xor edx, edx
1221	wrmsr
1222	%endif
1223
1224	%if %2 != 0
1225	; Restore the host xcr0.
1226	xor ecx, ecx
1227	mov rdx, [rbp + frm_uHostXcr0 + 8]
1228	mov rax, [rbp + frm_uHostXcr0]
1229	xsetbv
1230	%endif
1231
1232	%if %4 != 0
1233	; Save the guest SSE state related to non-volatile and volatile SSE registers.
1234	mov rcx, [r8 + CPUMCTX.pXStateR0]
1235	%if %4 = 1 ; manual
1236	stmxcsr [rcx + X86FXSTATE.MXCSR]
1237	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1238	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1239	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1240	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1241	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1242	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1243	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1244	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1245	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1246	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1247	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1248	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1249	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1250	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1251	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1252	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1253	%elif %4 = 2 ; use xrstor/xsave
1254	mov eax, [r8 + CPUMCTX.fXStateMask]
1255	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1256	xor edx, edx
1257	xsave [rcx]
1258	%else
1259	%error invalid template parameter 4
1260	%endif
1261
1262	; Restore the host non-volatile SSE register state.
1263	ldmxcsr [rbp + frm_saved_mxcsr]
1264	movdqa [rbp + frm_saved_xmm6 ], xmm6
1265	movdqa [rbp + frm_saved_xmm7 ], xmm7
1266	movdqa [rbp + frm_saved_xmm8 ], xmm8
1267	movdqa [rbp + frm_saved_xmm9 ], xmm9
1268	movdqa [rbp + frm_saved_xmm10], xmm10
1269	movdqa [rbp + frm_saved_xmm11], xmm11
1270	movdqa [rbp + frm_saved_xmm12], xmm12
1271	movdqa [rbp + frm_saved_xmm13], xmm13
1272	movdqa [rbp + frm_saved_xmm14], xmm14
1273	movdqa [rbp + frm_saved_xmm15], xmm15
1274	%endif ; %4 != 0
1275
1276	; Epilogue (assumes we restored volatile registers above when saving the guest GPRs).
1277	mov eax, VINF_SUCCESS
1278	add rsp, cbFrame - 8h
1279	popf
1280	leave
1281	ret
1282
1283	%ifdef VBOX_STRICT
1284	; Precondition checks failed.
1285	.failure_return:
1286	POP_CALLEE_PRESERVED_REGISTERS
1287	%if cbFrame != cbBaseFrame
1288	%error Bad frame size value: cbFrame
1289	%endif
1290	add rsp, cbFrame - 8h
1291	popf
1292	leave
1293	ret
1294	%endif
1295
1296	%undef frm_uHostXcr0
1297	%undef frm_fNoRestoreXcr0
1298	%undef frm_pVCpu
1299	%undef frm_HCPhysVmcbHost
1300	%undef cbFrame
1301	ENDPROC RT_CONCAT(hmR0SvmVmRun,%1)
1302
1303	%endmacro ; hmR0SvmVmRunTemplate
1304
1305	;
1306	; Instantiate the hmR0SvmVmRun various variations.
1307	;
1308	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit, 0, 0, 0
1309	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit, 1, 0, 0
1310	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY, 0
1311	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY, 0
1312	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_EXIT, 0
1313	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_EXIT, 0
1314	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 0
1315	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 0
1316	%ifdef VBOX_WITH_KERNEL_USING_XMM
1317	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 0, 0, 1
1318	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 1, 0, 1
1319	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_ENTRY, 1
1320	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_ENTRY, 1
1321	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_EXIT, 1
1322	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_EXIT, 1
1323	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 1
1324	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 1
1325
1326	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 0, 0, 2
1327	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 1, 0, 2
1328	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_ENTRY, 2
1329	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_ENTRY, 2
1330	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_EXIT, 2
1331	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_EXIT, 2
1332	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 2
1333	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 2
1334	%endif
1335

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 87390

Download in other formats: