HMR0A.asm@ 87440

Last change on this file since 87440 was 87440, checked in by vboxsync, 4 years ago
VMM/HMVMX: More hmR0VmxStartVm work.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 63.8 KB

Line
1	; $Id: HMR0A.asm 87440 2021-01-26 18:18:57Z vboxsync $
2	;; @file
3	; HM - Ring-0 VMX, SVM world-switch and helper routines.
4	;
5
6	;
7	; Copyright (C) 2006-2020 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.virtualbox.org. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17
18	;*********************************************************************************************************************************
19	;* Header Files *
20	;*********************************************************************************************************************************
21	;%define RT_ASM_WITH_SEH64 - trouble with SEH, alignment and (probably) 2nd pass optimizations.
22	%include "VBox/asmdefs.mac"
23	%include "VBox/err.mac"
24	%include "VBox/vmm/hm_vmx.mac"
25	%include "VBox/vmm/cpum.mac"
26	%include "VBox/vmm/vm.mac"
27	%include "iprt/x86.mac"
28	%include "HMInternal.mac"
29
30	%ifndef RT_ARCH_AMD64
31	%error AMD64 only.
32	%endif
33
34
35	;*********************************************************************************************************************************
36	;* Defined Constants And Macros *
37	;*********************************************************************************************************************************
38	;; The offset of the XMM registers in X86FXSTATE.
39	; Use define because I'm too lazy to convert the struct.
40	%define XMM_OFF_IN_X86FXSTATE 160
41
42	;; Spectre filler for 64-bit mode.
43	; Choosen to be an invalid address (also with 5 level paging).
44	%define SPECTRE_FILLER 0x02204204207fffff
45
46	;;
47	; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation.
48	;
49	; @note This is normally done by hmR0VmxExportHostSegmentRegs and VMXRestoreHostState,
50	; so much of this is untested code.
51	; @{
52	%define VMX_SKIP_GDTR
53	%define VMX_SKIP_TR
54	%define VBOX_SKIP_RESTORE_SEG
55	%ifdef RT_OS_DARWIN
56	; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
57	; risk loading a stale LDT value or something invalid.
58	%define HM_64_BIT_USE_NULL_SEL
59	; Darwin (Mavericks) uses IDTR limit to store the CPU number so we need to always restore it.
60	; See @bugref{6875}.
61	%undef VMX_SKIP_IDTR
62	%else
63	%define VMX_SKIP_IDTR
64	%endif
65	;; @}
66
67	;; @def CALLEE_PRESERVED_REGISTER_COUNT
68	; Number of registers pushed by PUSH_CALLEE_PRESERVED_REGISTERS
69	%ifdef ASM_CALL64_GCC
70	%define CALLEE_PRESERVED_REGISTER_COUNT 5
71	%else
72	%define CALLEE_PRESERVED_REGISTER_COUNT 7
73	%endif
74
75	;; @def PUSH_CALLEE_PRESERVED_REGISTERS
76	; Macro for pushing all GPRs we must preserve for the caller.
77	%macro PUSH_CALLEE_PRESERVED_REGISTERS 0
78	push r15
79	SEH64_PUSH_GREG r15
80	%assign cbFrame cbFrame + 8
81	%assign frm_saved_r15 -cbFrame
82
83	push r14
84	SEH64_PUSH_GREG r14
85	%assign cbFrame cbFrame + 8
86	%assign frm_saved_r14 -cbFrame
87
88	push r13
89	SEH64_PUSH_GREG r13
90	%assign cbFrame cbFrame + 8
91	%assign frm_saved_r13 -cbFrame
92
93	push r12
94	SEH64_PUSH_GREG r12
95	%assign cbFrame cbFrame + 8
96	%assign frm_saved_r12 -cbFrame
97
98	push rbx
99	SEH64_PUSH_GREG rbx
100	%assign cbFrame cbFrame + 8
101	%assign frm_saved_rbx -cbFrame
102
103	%ifdef ASM_CALL64_MSC
104	push rsi
105	SEH64_PUSH_GREG rsi
106	%assign cbFrame cbFrame + 8
107	%assign frm_saved_rsi -cbFrame
108
109	push rdi
110	SEH64_PUSH_GREG rdi
111	%assign cbFrame cbFrame + 8
112	%assign frm_saved_rdi -cbFrame
113	%endif
114	%endmacro
115
116	;; @def POP_CALLEE_PRESERVED_REGISTERS
117	; Counterpart to PUSH_CALLEE_PRESERVED_REGISTERS for use in the epilogue.
118	%macro POP_CALLEE_PRESERVED_REGISTERS 0
119	%ifdef ASM_CALL64_MSC
120	pop rdi
121	%assign cbFrame cbFrame - 8
122	%undef frm_saved_rdi
123
124	pop rsi
125	%assign cbFrame cbFrame - 8
126	%undef frm_saved_rsi
127	%endif
128	pop rbx
129	%assign cbFrame cbFrame - 8
130	%undef frm_saved_rbx
131
132	pop r12
133	%assign cbFrame cbFrame - 8
134	%undef frm_saved_r12
135
136	pop r13
137	%assign cbFrame cbFrame - 8
138	%undef frm_saved_r13
139
140	pop r14
141	%assign cbFrame cbFrame - 8
142	%undef frm_saved_r14
143
144	pop r15
145	%assign cbFrame cbFrame - 8
146	%undef frm_saved_r15
147	%endmacro
148
149
150	;; @def PUSH_RELEVANT_SEGMENT_REGISTERS
151	; Macro saving all segment registers on the stack.
152	; @param 1 Full width register name.
153	; @param 2 16-bit register name for \a 1.
154	; @cobbers rax, rdx, rcx
155	%macro PUSH_RELEVANT_SEGMENT_REGISTERS 2
156	%ifndef VBOX_SKIP_RESTORE_SEG
157	%error untested code. probably does not work any more!
158	%ifndef HM_64_BIT_USE_NULL_SEL
159	mov %2, es
160	push %1
161	mov %2, ds
162	push %1
163	%endif
164
165	; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode,
166	; Solaris OTOH doesn't and we must save it.
167	mov ecx, MSR_K8_FS_BASE
168	rdmsr
169	push rdx
170	push rax
171	%ifndef HM_64_BIT_USE_NULL_SEL
172	push fs
173	%endif
174
175	; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel.
176	; The same happens on exit.
177	mov ecx, MSR_K8_GS_BASE
178	rdmsr
179	push rdx
180	push rax
181	%ifndef HM_64_BIT_USE_NULL_SEL
182	push gs
183	%endif
184	%endif ; !VBOX_SKIP_RESTORE_SEG
185	%endmacro ; PUSH_RELEVANT_SEGMENT_REGISTERS
186
187	;; @def POP_RELEVANT_SEGMENT_REGISTERS
188	; Macro restoring all segment registers on the stack.
189	; @param 1 Full width register name.
190	; @param 2 16-bit register name for \a 1.
191	; @cobbers rax, rdx, rcx
192	%macro POP_RELEVANT_SEGMENT_REGISTERS 2
193	%ifndef VBOX_SKIP_RESTORE_SEG
194	%error untested code. probably does not work any more!
195	; Note: do not step through this code with a debugger!
196	%ifndef HM_64_BIT_USE_NULL_SEL
197	xor eax, eax
198	mov ds, ax
199	mov es, ax
200	mov fs, ax
201	mov gs, ax
202	%endif
203
204	%ifndef HM_64_BIT_USE_NULL_SEL
205	pop gs
206	%endif
207	pop rax
208	pop rdx
209	mov ecx, MSR_K8_GS_BASE
210	wrmsr
211
212	%ifndef HM_64_BIT_USE_NULL_SEL
213	pop fs
214	%endif
215	pop rax
216	pop rdx
217	mov ecx, MSR_K8_FS_BASE
218	wrmsr
219	; Now it's safe to step again
220
221	%ifndef HM_64_BIT_USE_NULL_SEL
222	pop %1
223	mov ds, %2
224	pop %1
225	mov es, %2
226	%endif
227	%endif ; !VBOX_SKIP_RESTORE_SEG
228	%endmacro ; POP_RELEVANT_SEGMENT_REGISTERS
229
230
231	;;
232	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
233	; @clobbers eax, edx, ecx
234	; @param 1 How to address CPUMCTX.
235	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
236	%macro INDIRECT_BRANCH_PREDICTION_BARRIER_CTX 2
237	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
238	jz %%no_indirect_branch_barrier
239	mov ecx, MSR_IA32_PRED_CMD
240	mov eax, MSR_IA32_PRED_CMD_F_IBPB
241	xor edx, edx
242	wrmsr
243	%%no_indirect_branch_barrier:
244	%endmacro
245
246	;;
247	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
248	; @clobbers eax, edx, ecx
249	; @param 1 How to address VMCPU.
250	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
251	%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2
252	test byte [%1 + VMCPU.cpum.GstCtx + CPUMCTX.fWorldSwitcher], %2
253	jz %%no_indirect_branch_barrier
254	mov ecx, MSR_IA32_PRED_CMD
255	mov eax, MSR_IA32_PRED_CMD_F_IBPB
256	xor edx, edx
257	wrmsr
258	%%no_indirect_branch_barrier:
259	%endmacro
260
261	;;
262	; Creates an indirect branch prediction and L1D barrier on CPUs that need and supports that.
263	; @clobbers eax, edx, ecx
264	; @param 1 How to address CPUMCTX.
265	; @param 2 Which IBPB flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
266	; @param 3 Which FLUSH flag to test for (CPUMCTX_WSF_L1D_ENTRY)
267	; @param 4 Which MDS flag to test for (CPUMCTX_WSF_MDS_ENTRY)
268	%macro INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER 4
269	; Only one test+jmp when disabled CPUs.
270	test byte [%1 + CPUMCTX.fWorldSwitcher], (%2 \| %3 \| %4)
271	jz %%no_barrier_needed
272
273	; The eax:edx value is the same for both.
274	AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
275	mov eax, MSR_IA32_PRED_CMD_F_IBPB
276	xor edx, edx
277
278	; Indirect branch barrier.
279	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
280	jz %%no_indirect_branch_barrier
281	mov ecx, MSR_IA32_PRED_CMD
282	wrmsr
283	%%no_indirect_branch_barrier:
284
285	; Level 1 data cache flush.
286	test byte [%1 + CPUMCTX.fWorldSwitcher], %3
287	jz %%no_cache_flush_barrier
288	mov ecx, MSR_IA32_FLUSH_CMD
289	wrmsr
290	jmp %%no_mds_buffer_flushing ; MDS flushing is included in L1D_FLUSH
291	%%no_cache_flush_barrier:
292
293	; MDS buffer flushing.
294	test byte [%1 + CPUMCTX.fWorldSwitcher], %4
295	jz %%no_mds_buffer_flushing
296	sub xSP, xSP
297	mov [xSP], ds
298	verw [xSP]
299	add xSP, xSP
300	%%no_mds_buffer_flushing:
301
302	%%no_barrier_needed:
303	%endmacro
304
305
306	;*********************************************************************************************************************************
307	;* External Symbols *
308	;*********************************************************************************************************************************
309	%ifdef VBOX_WITH_KERNEL_USING_XMM
310	extern NAME(CPUMIsGuestFPUStateActive)
311	%endif
312
313
314	BEGINCODE
315
316
317	;;
318	; Used on platforms with poor inline assembly support to retrieve all the
319	; info from the CPU and put it in the @a pRestoreHost structure.
320	;
321	; @returns VBox status code
322	; @param pRestoreHost msc: rcx gcc: rdi Pointer to the RestoreHost struct.
323	; @param fHaveFsGsBase msc: dl gcc: sil Whether we can use rdfsbase or not.
324	;
325	ALIGNCODE(64)
326	BEGINPROC hmR0VmxExportHostSegmentRegsAsmHlp
327	%ifdef ASM_CALL64_MSC
328	%define pRestoreHost rcx
329	%elifdef ASM_CALL64_GCC
330	%define pRestoreHost rdi
331	%else
332	%error Unknown calling convension.
333	%endif
334	SEH64_END_PROLOGUE
335
336	; Start with the FS and GS base so we can trash DL/SIL.
337	%ifdef ASM_CALL64_MSC
338	or dl, dl
339	%else
340	or sil, sil
341	%endif
342	jz .use_rdmsr_for_fs_and_gs_base
343	rdfsbase rax
344	mov [pRestoreHost + VMXRESTOREHOST.uHostFSBase], rax
345	rdgsbase rax
346	mov [pRestoreHost + VMXRESTOREHOST.uHostGSBase], rax
347	.done_fs_and_gs_base:
348
349	; TR, GDTR and IDTR
350	str [pRestoreHost + VMXRESTOREHOST.uHostSelTR]
351	sgdt [pRestoreHost + VMXRESTOREHOST.HostGdtr]
352	sidt [pRestoreHost + VMXRESTOREHOST.HostIdtr]
353
354	; Segment registers.
355	xor eax, eax
356	mov eax, cs
357	mov [pRestoreHost + VMXRESTOREHOST.uHostSelCS], ax
358
359	mov eax, ss
360	mov [pRestoreHost + VMXRESTOREHOST.uHostSelSS], ax
361
362	mov eax, gs
363	mov [pRestoreHost + VMXRESTOREHOST.uHostSelGS], ax
364
365	mov eax, fs
366	mov [pRestoreHost + VMXRESTOREHOST.uHostSelFS], ax
367
368	mov eax, es
369	mov [pRestoreHost + VMXRESTOREHOST.uHostSelES], ax
370
371	mov eax, ds
372	mov [pRestoreHost + VMXRESTOREHOST.uHostSelDS], ax
373
374	ret
375
376	ALIGNCODE(16)
377	.use_rdmsr_for_fs_and_gs_base:
378	%ifdef ASM_CALL64_MSC
379	mov r8, pRestoreHost
380	%endif
381
382	mov ecx, MSR_K8_FS_BASE
383	rdmsr
384	shl rdx, 32
385	or rdx, rax
386	mov [r8 + VMXRESTOREHOST.uHostFSBase], rax
387
388	mov ecx, MSR_K8_GS_BASE
389	rdmsr
390	shl rdx, 32
391	or rdx, rax
392	mov [r8 + VMXRESTOREHOST.uHostGSBase], rax
393
394	%ifdef ASM_CALL64_MSC
395	mov pRestoreHost, r8
396	%endif
397	jmp .done_fs_and_gs_base
398	%undef pRestoreHost
399	ENDPROC hmR0VmxExportHostSegmentRegsAsmHlp
400
401
402	;;
403	; Restores host-state fields.
404	;
405	; @returns VBox status code
406	; @param f32RestoreHost msc: ecx gcc: edi RestoreHost flags.
407	; @param pRestoreHost msc: rdx gcc: rsi Pointer to the RestoreHost struct.
408	;
409	ALIGNCODE(64)
410	BEGINPROC VMXRestoreHostState
411	%ifndef ASM_CALL64_GCC
412	; Use GCC's input registers since we'll be needing both rcx and rdx further
413	; down with the wrmsr instruction. Use the R10 and R11 register for saving
414	; RDI and RSI since MSC preserve the two latter registers.
415	mov r10, rdi
416	mov r11, rsi
417	mov rdi, rcx
418	mov rsi, rdx
419	%endif
420	SEH64_END_PROLOGUE
421
422	.restore_gdtr:
423	test edi, VMX_RESTORE_HOST_GDTR
424	jz .restore_idtr
425	lgdt [rsi + VMXRESTOREHOST.HostGdtr]
426
427	.restore_idtr:
428	test edi, VMX_RESTORE_HOST_IDTR
429	jz .restore_ds
430	lidt [rsi + VMXRESTOREHOST.HostIdtr]
431
432	.restore_ds:
433	test edi, VMX_RESTORE_HOST_SEL_DS
434	jz .restore_es
435	mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
436	mov ds, eax
437
438	.restore_es:
439	test edi, VMX_RESTORE_HOST_SEL_ES
440	jz .restore_tr
441	mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
442	mov es, eax
443
444	.restore_tr:
445	test edi, VMX_RESTORE_HOST_SEL_TR
446	jz .restore_fs
447	; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
448	mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
449	mov ax, dx
450	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
451	test edi, VMX_RESTORE_HOST_GDT_READ_ONLY \| VMX_RESTORE_HOST_GDT_NEED_WRITABLE
452	jnz .gdt_readonly_or_need_writable
453	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
454	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
455	ltr dx
456
457	.restore_fs:
458	;
459	; When restoring the selector values for FS and GS, we'll temporarily trash
460	; the base address (at least the high 32-bit bits, but quite possibly the
461	; whole base address), the wrmsr will restore it correctly. (VT-x actually
462	; restores the base correctly when leaving guest mode, but not the selector
463	; value, so there is little problem with interrupts being enabled prior to
464	; this restore job.)
465	; We'll disable ints once for both FS and GS as that's probably faster.
466	;
467	test edi, VMX_RESTORE_HOST_SEL_FS \| VMX_RESTORE_HOST_SEL_GS
468	jz .restore_success
469	pushfq
470	cli ; (see above)
471
472	test edi, VMX_RESTORE_HOST_CAN_USE_WRFSBASE_AND_WRGSBASE
473	jz .restore_fs_using_wrmsr
474
475	.restore_fs_using_wrfsbase:
476	test edi, VMX_RESTORE_HOST_SEL_FS
477	jz .restore_gs_using_wrgsbase
478	mov rax, qword [rsi + VMXRESTOREHOST.uHostFSBase]
479	mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
480	mov fs, ecx
481	wrfsbase rax
482
483	.restore_gs_using_wrgsbase:
484	test edi, VMX_RESTORE_HOST_SEL_GS
485	jz .restore_flags
486	mov rax, qword [rsi + VMXRESTOREHOST.uHostGSBase]
487	mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
488	mov gs, ecx
489	wrgsbase rax
490
491	.restore_flags:
492	popfq
493
494	.restore_success:
495	mov eax, VINF_SUCCESS
496	%ifndef ASM_CALL64_GCC
497	; Restore RDI and RSI on MSC.
498	mov rdi, r10
499	mov rsi, r11
500	%endif
501	ret
502
503	ALIGNCODE(8)
504	.gdt_readonly_or_need_writable:
505	test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
506	jnz .gdt_readonly_need_writable
507	.gdt_readonly:
508	mov rcx, cr0
509	mov r9, rcx
510	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
511	and rcx, ~X86_CR0_WP
512	mov cr0, rcx
513	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
514	ltr dx
515	mov cr0, r9
516	jmp .restore_fs
517
518	ALIGNCODE(8)
519	.gdt_readonly_need_writable:
520	add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw
521	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
522	lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
523	ltr dx
524	lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT
525	jmp .restore_fs
526
527	ALIGNCODE(8)
528	.restore_fs_using_wrmsr:
529	test edi, VMX_RESTORE_HOST_SEL_FS
530	jz .restore_gs_using_wrmsr
531	mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
532	mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
533	mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS]
534	mov fs, ecx
535	mov ecx, MSR_K8_FS_BASE
536	wrmsr
537
538	.restore_gs_using_wrmsr:
539	test edi, VMX_RESTORE_HOST_SEL_GS
540	jz .restore_flags
541	mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
542	mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
543	mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS]
544	mov gs, ecx
545	mov ecx, MSR_K8_GS_BASE
546	wrmsr
547	jmp .restore_flags
548	ENDPROC VMXRestoreHostState
549
550
551	;;
552	; Dispatches an NMI to the host.
553	;
554	ALIGNCODE(16)
555	BEGINPROC VMXDispatchHostNmi
556	; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
557	SEH64_END_PROLOGUE
558	int 2
559	ret
560	ENDPROC VMXDispatchHostNmi
561
562
563	%ifdef VBOX_WITH_KERNEL_USING_XMM
564	;;
565	; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
566	; load the guest ones when necessary.
567	;
568	; @cproto DECLASM(int) hmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVM pVM,
569	; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
570	;
571	; @returns eax
572	;
573	; @param pVM msc:rcx
574	; @param pVCpu msc:rdx The cross context virtual CPU structure of the calling EMT.
575	; @param fResumeVM msc:r8l
576	; @param pfnStartVM msc:r9
577	;
578	; Old:
579	; @param fResumeVM msc:rcx
580	; @param pCtx msc:rdx
581	; @param pvUnused msc:r8
582	; @param pVM msc:r9
583	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
584	; @param pfnStartVM msc:[rbp+38h]
585	;
586	; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
587	;
588	; @remarks Drivers shouldn't use AVX registers without saving+loading:
589	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
590	; However the compiler docs have different idea:
591	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
592	; We'll go with the former for now.
593	;
594	; ASSUMING 64-bit and windows for now.
595	;
596	ALIGNCODE(16)
597	BEGINPROC hmR0VMXStartVMWrapXMM
598	SEH64_END_PROLOGUE
599	push xBP
600	mov xBP, xSP
601	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
602
603	; Spill input parameters.
604	mov [xBP + 010h], rcx ; pVM
605	mov [xBP + 018h], rdx ; pVCpu
606	mov [xBP + 020h], r8 ; fResumeVM
607	mov [xBP + 028h], r9 ; pfnStartVM
608
609	; Ask CPUM whether we've started using the FPU yet.
610	mov rcx, [xBP + 018h] ; pVCpu
611	call NAME(CPUMIsGuestFPUStateActive)
612	test al, al
613	jnz .guest_fpu_state_active
614
615	; No need to mess with XMM registers just call the start routine and return.
616	mov r9, [xBP + 028h] ; pfnStartVM
617	mov rcx, [xBP + 010h] ; pVM
618	mov rdx, [xBP + 018h] ; pVCpu
619	mov r8, [xBP + 020h] ; fResume
620	call r9
621
622	leave
623	ret
624
625	ALIGNCODE(8)
626	.guest_fpu_state_active:
627	; Save the non-volatile host XMM registers.
628	movdqa [rsp + 040h + 000h], xmm6
629	movdqa [rsp + 040h + 010h], xmm7
630	movdqa [rsp + 040h + 020h], xmm8
631	movdqa [rsp + 040h + 030h], xmm9
632	movdqa [rsp + 040h + 040h], xmm10
633	movdqa [rsp + 040h + 050h], xmm11
634	movdqa [rsp + 040h + 060h], xmm12
635	movdqa [rsp + 040h + 070h], xmm13
636	movdqa [rsp + 040h + 080h], xmm14
637	movdqa [rsp + 040h + 090h], xmm15
638	stmxcsr [rsp + 040h + 0a0h]
639
640	mov r10, [xBP + 018h] ; pVCpu
641	mov eax, [r10 + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
642	test eax, eax
643	jz .guest_fpu_state_manually
644
645	;
646	; Using XSAVE to load the guest XMM, YMM and ZMM registers.
647	;
648	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
649	xor edx, edx
650	mov r10, [r10 + VMCPU.cpum.GstCtx + CPUMCTX.pXStateR0]
651	xrstor [r10]
652
653	; Make the call (same as in the other case).
654	mov r9, [xBP + 028h] ; pfnStartVM
655	mov rcx, [xBP + 010h] ; pVM
656	mov rdx, [xBP + 018h] ; pVCpu
657	mov r8, [xBP + 020h] ; fResume
658	call r9
659
660	mov r11d, eax ; save return value (xsave below uses eax)
661
662	; Save the guest XMM registers.
663	mov r10, [xBP + 018h] ; pVCpu
664	mov eax, [r10 + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
665	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
666	xor edx, edx
667	mov r10, [r10 + VMCPU.cpum.GstCtx + CPUMCTX.pXStateR0]
668	xsave [r10]
669
670	mov eax, r11d ; restore return value
671
672	.restore_non_volatile_host_xmm_regs:
673	; Load the non-volatile host XMM registers.
674	movdqa xmm6, [rsp + 040h + 000h]
675	movdqa xmm7, [rsp + 040h + 010h]
676	movdqa xmm8, [rsp + 040h + 020h]
677	movdqa xmm9, [rsp + 040h + 030h]
678	movdqa xmm10, [rsp + 040h + 040h]
679	movdqa xmm11, [rsp + 040h + 050h]
680	movdqa xmm12, [rsp + 040h + 060h]
681	movdqa xmm13, [rsp + 040h + 070h]
682	movdqa xmm14, [rsp + 040h + 080h]
683	movdqa xmm15, [rsp + 040h + 090h]
684	ldmxcsr [rsp + 040h + 0a0h]
685	leave
686	ret
687
688	;
689	; No XSAVE, load and save the guest XMM registers manually.
690	;
691	.guest_fpu_state_manually:
692	; Load the full guest XMM register state.
693	mov r10, [r10 + VMCPU.cpum.GstCtx + CPUMCTX.pXStateR0]
694	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
695	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
696	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
697	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
698	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
699	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
700	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
701	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
702	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
703	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
704	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
705	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
706	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
707	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
708	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
709	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
710	ldmxcsr [r10 + X86FXSTATE.MXCSR]
711
712	; Make the call (same as in the other case).
713	mov r9, [xBP + 028h] ; pfnStartVM
714	mov rcx, [xBP + 010h] ; pVM
715	mov rdx, [xBP + 018h] ; pVCpu
716	mov r8, [xBP + 020h] ; fResume
717	call r9
718
719	; Save the guest XMM registers.
720	mov r10, [xBP + 018h] ; pVCpu
721	mov r10, [r10 + VMCPU.cpum.GstCtx + CPUMCTX.pXStateR0]
722	stmxcsr [r10 + X86FXSTATE.MXCSR]
723	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
724	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
725	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
726	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
727	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
728	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
729	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
730	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
731	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
732	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
733	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
734	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
735	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
736	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
737	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
738	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
739	jmp .restore_non_volatile_host_xmm_regs
740	ENDPROC hmR0VMXStartVMWrapXMM
741	%endif ; VBOX_WITH_KERNEL_USING_XMM
742
743
744	;;
745	; hmR0VmxStartVm template
746	;
747	; @param 1 The suffix of the variation.
748	; @param 2 fLoadSaveGuestXcr0 value
749	; @param 3 The CPUMCTX_WSF_IBPB_ENTRY + CPUMCTX_WSF_IBPB_EXIT value.
750	; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
751	; Drivers shouldn't use AVX registers without saving+loading:
752	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
753	; However the compiler docs have different idea:
754	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
755	; We'll go with the former for now.
756	;
757	%macro hmR0VmxStartVmTemplate 4
758
759	;;
760	; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
761	;
762	; @returns VBox status code
763	; @param pVM msc:rcx, gcc:rdi The cross context VM structure. (unused)
764	; @param pVCpu msc:rdx, gcc:rsi The cross context virtual CPU structure of the calling EMT.
765	; @param fResume msc:r8l, gcc:dl Whether to use vmlauch/vmresume.
766	;
767	ALIGNCODE(64)
768	BEGINPROC RT_CONCAT(hmR0VmxStartVm,%1)
769	push xBP
770	mov xBP, xSP
771	SEH64_SET_FRAME_xBP 0
772	pushf
773	cli
774
775	%define frm_fRFlags -008h
776	%define frm_pGstCtx -010h ; Where we stash guest CPU context for use after the vmrun.
777	%define frm_uHostXcr0 -020h ; 128-bit
778	%define frm_saved_gdtr -02ah ; 16+64: Only used when VMX_SKIP_GDTR isn't defined
779	%define frm_saved_tr -02ch ; 16-bit: Only used when VMX_SKIP_TR isn't defined
780	%define frm_MDS_seg -030h ; 16-bit: Temporary storage for the MDS flushing.
781	%define frm_saved_idtr -03ah ; 16+64: Only used when VMX_SKIP_IDTR isn't defined
782	%define frm_saved_ldtr -03ch ; 16-bit: always saved.
783	%define frm_rcError -040h ; 32-bit: Error status code (not used in the success path)
784	%define frm_guest_rax -048h ; Temporary storage slot for guest RAX.
785	%assign cbFrame 048h
786	%assign cbBaseFrame cbFrame
787	sub rsp, cbFrame - 8
788
789	; Save all general purpose host registers.
790	PUSH_CALLEE_PRESERVED_REGISTERS
791	;PUSH_RELEVANT_SEGMENT_REGISTERS xAX, ax - currently broken
792	SEH64_END_PROLOGUE
793
794	;
795	; Unify the input parameter registers: rsi=pVCpu, bl=fResume, rdi=&pVCpu->cpum.GstCtx;
796	;
797	%ifdef ASM_CALL64_GCC
798	mov ebx, edx ; fResume
799	%else
800	mov rsi, rdx ; pVCpu
801	mov ebx, r8d ; fResume
802	%endif
803	lea rdi, [rsi + VMCPU.cpum.GstCtx]
804	mov [rbp + frm_pGstCtx], rdi
805
806	%ifdef VBOX_STRICT
807	;
808	; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
809	;
810	cmp byte [rsi + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], %2
811	mov eax, VERR_VMX_STARTVM_PRECOND_0
812	jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
813
814	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fWorldSwitcher]
815	and eax, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT
816	cmp eax, %3
817	mov eax, VERR_VMX_STARTVM_PRECOND_1
818	jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
819
820	; %ifdef VBOX_WITH_KERNEL_USING_XMM
821	; mov eax, VERR_VMX_STARTVM_PRECOND_2
822	; test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
823	; %if %4 = 0
824	; ;jnz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
825	; %else
826	; jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
827	;
828	; mov eax, VERR_VMX_STARTVM_PRECOND_3
829	; cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
830	; %if %4 = 1
831	; jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
832	; %elif %4 = 2
833	; je NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return)
834	; %else
835	; %error Invalid template parameter 4.
836	; %endif
837	; %endif
838	; %endif
839	%endif ; VBOX_STRICT
840
841	%if %2 != 0
842	; Save the host XCR0 and load the guest one if necessary.
843	; Note! Trashes rax, rdx and rcx.
844	xor ecx, ecx
845	xgetbv ; save the host one on the stack
846	mov [rbp + frm_uHostXcr0], eax
847	mov [rbp + frm_uHostXcr0 + 4], edx
848
849	mov eax, [rdi + CPUMCTX.aXcr] ; load the guest one
850	mov edx, [rdi + CPUMCTX.aXcr + 4]
851	xor ecx, ecx ; paranoia; indicate that we must restore XCR0 (popped into ecx, thus 0)
852	xsetbv
853	%endif
854
855	; Save host LDTR.
856	sldt word [rbp + frm_saved_ldtr]
857
858	%ifndef VMX_SKIP_TR
859	; The host TR limit is reset to 0x67; save & restore it manually.
860	str word [rbp + frm_saved_tr]
861	%endif
862
863	%ifndef VMX_SKIP_GDTR
864	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
865	sgdt [rbp + frm_saved_gdtr]
866	%endif
867	%ifndef VMX_SKIP_IDTR
868	sidt [rbp + frm_saved_idtr]
869	%endif
870
871	; Load CR2 if necessary (expensive as writing CR2 is a synchronizing instruction - (bird: still expensive on 10980xe)).
872	mov rcx, qword [rdi + CPUMCTX.cr2]
873	mov rdx, cr2
874	cmp rcx, rdx
875	je .skip_cr2_write
876	mov cr2, rcx
877	.skip_cr2_write:
878
879	; Set the vmlaunch/vmresume "return" host RIP and RSP values if they've changed (unlikly).
880	; The vmwrite isn't quite for free (on an 10980xe at least), thus we check if anything changed
881	; before writing here.
882	lea rcx, [NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) wrt rip]
883	cmp rcx, [rsi + VMCPU.hm + HMCPU.u + HMCPUVMX.uHostRIP]
884	jne .write_host_rip
885	.wrote_host_rip:
886	cmp rsp, [rsi + VMCPU.hm + HMCPU.u + HMCPUVMX.uHostRSP]
887	jne .write_host_rsp
888	.wrote_host_rsp:
889
890	;
891	; Fight spectre and similar. Trashes rax, rcx, and rdx.
892	;
893	%if %3 & (CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY) ; The eax:edx value is the same for the first two.
894	AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
895	mov eax, MSR_IA32_PRED_CMD_F_IBPB
896	xor edx, edx
897	%endif
898	%if %3 & CPUMCTX_WSF_IBPB_ENTRY ; Indirect branch barrier.
899	mov ecx, MSR_IA32_PRED_CMD
900	wrmsr
901	%endif
902	%if %3 & CPUMCTX_WSF_L1D_ENTRY ; Level 1 data cache flush.
903	mov ecx, MSR_IA32_FLUSH_CMD
904	wrmsr
905	%elif %3 & CPUMCTX_WSF_MDS_ENTRY ; MDS flushing is included in L1D_FLUSH
906	mov word [rbp + frm_MDS_seg], ds
907	verw word [rbp + frm_MDS_seg]
908	%endif
909
910	; Resume or start VM?
911	cmp bl, 0 ; fResume
912
913	; Load guest general purpose registers.
914	mov rax, qword [rdi + CPUMCTX.eax]
915	mov rbx, qword [rdi + CPUMCTX.ebx]
916	mov rcx, qword [rdi + CPUMCTX.ecx]
917	mov rdx, qword [rdi + CPUMCTX.edx]
918	mov rbp, qword [rdi + CPUMCTX.ebp]
919	mov rsi, qword [rdi + CPUMCTX.esi]
920	mov r8, qword [rdi + CPUMCTX.r8]
921	mov r9, qword [rdi + CPUMCTX.r9]
922	mov r10, qword [rdi + CPUMCTX.r10]
923	mov r11, qword [rdi + CPUMCTX.r11]
924	mov r12, qword [rdi + CPUMCTX.r12]
925	mov r13, qword [rdi + CPUMCTX.r13]
926	mov r14, qword [rdi + CPUMCTX.r14]
927	mov r15, qword [rdi + CPUMCTX.r15]
928	mov rdi, qword [rdi + CPUMCTX.edi]
929
930	je .vmlaunch64_launch
931
932	vmresume
933	jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr)
934	jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed)
935	jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmresume detected a failure
936
937	.vmlaunch64_launch:
938	vmlaunch
939	jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr)
940	jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed)
941	jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmlaunch detected a failure
942
943
944	; Put these two outside the normal code path as they should rarely change.
945	ALIGNCODE(8)
946	.write_host_rip:
947	mov [rsi + VMCPU.hm + HMCPU.u + HMCPUVMX.uHostRIP], rcx
948	mov eax, VMX_VMCS_HOST_RIP ;; @todo It is only strictly necessary to write VMX_VMCS_HOST_RIP when
949	vmwrite rax, rcx ;; the VMXVMCSINFO::pfnStartVM function changes (eventually
950	%ifdef VBOX_STRICT ;; take the Windows/SSE stuff into account then)...
951	jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed)
952	%endif
953	jmp .wrote_host_rip
954
955	ALIGNCODE(8)
956	.write_host_rsp:
957	mov [rsi + VMCPU.hm + HMCPU.u + HMCPUVMX.uHostRSP], rsp
958	mov eax, VMX_VMCS_HOST_RSP
959	vmwrite rax, rsp
960	%ifdef VBOX_STRICT
961	jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed)
962	%endif
963	jmp .wrote_host_rsp
964
965	ALIGNCODE(64)
966	GLOBALNAME RT_CONCAT(hmR0VmxStartVmHostRIP,%1)
967
968	;;
969	; Common restore logic for success and error paths. We duplicate this because we
970	; don't want to waste writing the VINF_SUCCESS return value to the stack in the
971	; regular code path.
972	;
973	; @param 1 Zero if regular return, non-zero if error return. Controls label emission.
974	; @param 2 fLoadSaveGuestXcr0 value
975	; @param 3 The (CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY) + CPUMCTX_WSF_IBPB_EXIT value.
976	; The entry values are either all set or not at all, as we're too lazy to flesh out all the variants.
977	;
978	; @note Important that this does not modify cbFrame or rsp.
979	%macro RESTORE_STATE_VMX 3
980	; Restore base and limit of the IDTR & GDTR.
981	%ifndef VMX_SKIP_IDTR
982	lidt [rsp + cbFrame + frm_saved_idtr]
983	%endif
984	%ifndef VMX_SKIP_GDTR
985	lgdt [rsp + cbFrame + frm_saved_gdtr]
986	%endif
987
988	; Save the guest state and restore the non-volatile registers. We use rax=pGstCtx here.
989	mov [rsp + cbFrame + frm_guest_rax], rax
990	mov rax, [rsp + cbFrame + frm_pGstCtx]
991
992	mov qword [rax + CPUMCTX.ebp], rbp
993	lea rbp, [rsp + cbFrame] ; re-establish the frame pointer as early as possible.
994	mov qword [rax + CPUMCTX.ecx], rcx
995	mov rcx, SPECTRE_FILLER
996	mov qword [rax + CPUMCTX.edx], rdx
997	mov rdx, [rbp + frm_guest_rax]
998	mov qword [rax + CPUMCTX.eax], rdx
999	mov rdx, rcx
1000	mov qword [rax + CPUMCTX.r8], r8
1001	mov r8, rcx
1002	mov qword [rax + CPUMCTX.r9], r9
1003	mov r9, rcx
1004	mov qword [rax + CPUMCTX.r10], r10
1005	mov r10, rcx
1006	mov qword [rax + CPUMCTX.r11], r11
1007	mov r11, rcx
1008	mov qword [rax + CPUMCTX.esi], rsi
1009	%ifdef ASM_CALL64_MSC
1010	mov rsi, [rbp + frm_saved_rsi]
1011	%else
1012	mov rbx, rcx
1013	%endif
1014	mov qword [rax + CPUMCTX.edi], rdi
1015	%ifdef ASM_CALL64_MSC
1016	mov rdi, [rbp + frm_saved_rdi]
1017	%else
1018	mov rbx, rcx
1019	%endif
1020	mov qword [rax + CPUMCTX.ebx], rbx
1021	mov rbx, [rbp + frm_saved_rbx]
1022	mov qword [rax + CPUMCTX.r12], r12
1023	mov r12, [rbp + frm_saved_r12]
1024	mov qword [rax + CPUMCTX.r13], r13
1025	mov r13, [rbp + frm_saved_r13]
1026	mov qword [rax + CPUMCTX.r14], r14
1027	mov r14, [rbp + frm_saved_r14]
1028	mov qword [rax + CPUMCTX.r15], r15
1029	mov r15, [rbp + frm_saved_r15]
1030
1031	mov rdx, cr2
1032	mov qword [rax + CPUMCTX.cr2], rdx
1033	mov rdx, rcx
1034
1035	%if %3 & CPUMCTX_WSF_IBPB_EXIT
1036	; Fight spectre (trashes rax, rdx and rcx).
1037	%if %1 = 0 ; Skip this in failure branch (=> guru)
1038	mov ecx, MSR_IA32_PRED_CMD
1039	mov eax, MSR_IA32_PRED_CMD_F_IBPB
1040	xor edx, edx
1041	wrmsr
1042	%endif
1043	%endif
1044
1045	%ifndef VMX_SKIP_TR
1046	; Restore TSS selector; must mark it as not busy before using ltr!
1047	; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p).
1048	%ifndef VMX_SKIP_GDTR
1049	lgdt [rbp + frm_saved_gdtr]
1050	%endif
1051	movzx eax, word [rbp + frm_saved_tr]
1052	mov ecx, eax
1053	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
1054	add rax, [rbp + frm_saved_gdtr + 2] ; eax <- GDTR.address + descriptor offset
1055	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
1056	ltr cx
1057	%endif
1058	movzx edx, word [rbp + frm_saved_ldtr]
1059	test edx, edx
1060	jz %%skip_ldt_write
1061	lldt dx
1062	%%skip_ldt_write:
1063
1064	%if %1 != 0
1065	.return_after_vmwrite_error:
1066	%endif
1067	; Restore segment registers.
1068	;POP_RELEVANT_SEGMENT_REGISTERS rax, ax - currently broken.
1069
1070	%if %2 != 0
1071	; Restore the host XCR0.
1072	xor ecx, ecx
1073	mov eax, [rbp + frm_uHostXcr0]
1074	mov edx, [rbp + frm_uHostXcr0 + 4]
1075	xsetbv
1076	%endif
1077	%endmacro ; RESTORE_STATE_VMX
1078
1079
1080	RESTORE_STATE_VMX 0, %2, %3
1081	mov eax, VINF_SUCCESS
1082
1083	.vmstart64_end:
1084	lea rsp, [rbp + frm_fRFlags]
1085	popf
1086	leave
1087	ret
1088
1089	;
1090	; Error returns.
1091	;
1092	%ifdef VBOX_STRICT
1093	.vmwrite_failed:
1094	mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_FIELD
1095	jz .return_after_vmwrite_error
1096	mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR
1097	jmp .return_after_vmwrite_error
1098	%endif
1099	.vmxstart64_invalid_vmcs_ptr:
1100	mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1101	jmp .vmstart64_error_return
1102	.vmxstart64_start_failed:
1103	mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_UNABLE_TO_START_VM
1104	.vmstart64_error_return:
1105	RESTORE_STATE_VMX 1, %2, %3
1106	mov eax, [rbp + frm_rcError]
1107	jmp .vmstart64_end
1108
1109	%ifdef VBOX_STRICT
1110	; Precondition checks failed.
1111	.precond_failure_return:
1112	POP_CALLEE_PRESERVED_REGISTERS
1113	%if cbFrame != cbBaseFrame
1114	%error Bad frame size value: cbFrame, expected cbBaseFrame
1115	%endif
1116	jmp .vmstart64_end
1117	%endif
1118
1119	%undef frm_fRFlags
1120	%undef frm_pGstCtx
1121	%undef frm_uHostXcr0
1122	%undef frm_saved_gdtr
1123	%undef frm_saved_tr
1124	%undef frm_fNoRestoreXcr0
1125	%undef frm_saved_idtr
1126	%undef frm_saved_ldtr
1127	%undef frm_rcError
1128	%undef frm_guest_rax
1129	%undef cbFrame
1130	ENDPROC RT_CONCAT(hmR0VmxStartVm,%1)
1131
1132	%endmacro ; hmR0VmxStartVmTemplate
1133
1134	%macro hmR0VmxStartVmSseTemplate 1
1135	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit, 0, 0 \| 0 \| 0 \| 0 , %1
1136	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit, 1, 0 \| 0 \| 0 \| 0 , %1
1137	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| 0 \| 0 , %1
1138	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| 0 \| 0 , %1
1139	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit, 0, 0 \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| 0 , %1
1140	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit, 1, 0 \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| 0 , %1
1141	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| 0 , %1
1142	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| 0 , %1
1143	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit, 0, 0 \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1144	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit, 1, 0 \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1145	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1146	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1147	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit, 0, 0 \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1148	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit, 1, 0 \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1149	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1150	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| 0 , %1
1151	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit, 0, 0 \| 0 \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1152	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit, 1, 0 \| 0 \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1153	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1154	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1155	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit, 0, 0 \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1156	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit, 1, 0 \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1157	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1158	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| 0 \| CPUMCTX_WSF_IBPB_EXIT, %1
1159	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit, 0, 0 \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1160	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit, 1, 0 \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1161	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1162	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| 0 \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1163	hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit, 0, 0 \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1164	hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit, 1, 0 \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1165	hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1166	hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_L1D_ENTRY \| CPUMCTX_WSF_MDS_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, %1
1167	%endmacro
1168	hmR0VmxStartVmSseTemplate 0
1169
1170
1171	;;
1172	; Clears the MDS buffers using VERW.
1173	ALIGNCODE(16)
1174	BEGINPROC hmR0MdsClear
1175	SEH64_END_PROLOGUE
1176	sub xSP, xCB
1177	mov [xSP], ds
1178	verw [xSP]
1179	add xSP, xCB
1180	ret
1181	ENDPROC hmR0MdsClear
1182
1183
1184	;;
1185	; hmR0SvmVmRun template
1186	;
1187	; @param 1 The suffix of the variation.
1188	; @param 2 fLoadSaveGuestXcr0 value
1189	; @param 3 The CPUMCTX_WSF_IBPB_ENTRY + CPUMCTX_WSF_IBPB_EXIT value.
1190	; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor.
1191	; Drivers shouldn't use AVX registers without saving+loading:
1192	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
1193	; However the compiler docs have different idea:
1194	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
1195	; We'll go with the former for now.
1196	;
1197	%macro hmR0SvmVmRunTemplate 4
1198
1199	;;
1200	; Prepares for and executes VMRUN (32-bit and 64-bit guests).
1201	;
1202	; @returns VBox status code
1203	; @param pVM msc:rcx,gcc:rdi The cross context VM structure (unused).
1204	; @param pVCpu msc:rdx,gcc:rsi The cross context virtual CPU structure of the calling EMT.
1205	; @param HCPhysVmcb msc:r8, gcc:rdx Physical address of guest VMCB.
1206	;
1207	ALIGNCODE(64) ; This + immediate optimizations causes serious trouble for yasm and the SEH frames: prologue -28 bytes, must be <256
1208	; So the SEH64_XXX stuff is currently not operational.
1209	BEGINPROC RT_CONCAT(hmR0SvmVmRun,%1)
1210	%ifdef VBOX_WITH_KERNEL_USING_XMM
1211	%if %4 = 0
1212	;
1213	; The non-saving variant will currently check the two SSE preconditions and pick
1214	; the right variant to continue with. Later we can see if we can't manage to
1215	; move these decisions into hmR0SvmUpdateVmRunFunction().
1216	;
1217	%ifdef ASM_CALL64_MSC
1218	test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1219	%else
1220	test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1221	%endif
1222	jz .save_xmm_no_need
1223	%ifdef ASM_CALL64_MSC
1224	cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1225	%else
1226	cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1227	%endif
1228	je RT_CONCAT3(hmR0SvmVmRun,%1,_SseManual)
1229	jmp RT_CONCAT3(hmR0SvmVmRun,%1,_SseXSave)
1230	.save_xmm_no_need:
1231	%endif
1232	%endif
1233	push rbp
1234	SEH64_PUSH_xBP
1235	mov rbp, rsp
1236	SEH64_SET_FRAME_xBP 0
1237	pushf
1238	%assign cbFrame 30h
1239	%if %4 != 0
1240	%assign cbFrame cbFrame + 16 * 11 ; Reserve space for 10x 128-bit XMM registers and MXCSR (32-bit)
1241	%endif
1242	%assign cbBaseFrame cbFrame
1243	sub rsp, cbFrame - 8h ; We subtract 8 bytes for the above pushf
1244	SEH64_ALLOCATE_STACK cbFrame ; And we have CALLEE_PRESERVED_REGISTER_COUNT following it.
1245
1246	%define frm_fRFlags -008h
1247	%define frm_uHostXcr0 -018h ; 128-bit
1248	;%define frm_fNoRestoreXcr0 -020h ; Non-zero if we should skip XCR0 restoring.
1249	%define frm_pGstCtx -028h ; Where we stash guest CPU context for use after the vmrun.
1250	%define frm_HCPhysVmcbHost -030h ; Where we stash HCPhysVmcbHost for the vmload after vmrun.
1251	%if %4 != 0
1252	%define frm_saved_xmm6 -040h
1253	%define frm_saved_xmm7 -050h
1254	%define frm_saved_xmm8 -060h
1255	%define frm_saved_xmm9 -070h
1256	%define frm_saved_xmm10 -080h
1257	%define frm_saved_xmm11 -090h
1258	%define frm_saved_xmm12 -0a0h
1259	%define frm_saved_xmm13 -0b0h
1260	%define frm_saved_xmm14 -0c0h
1261	%define frm_saved_xmm15 -0d0h
1262	%define frm_saved_mxcsr -0e0h
1263	%endif
1264
1265	; Manual save and restore:
1266	; - General purpose registers except RIP, RSP, RAX
1267	;
1268	; Trashed:
1269	; - CR2 (we don't care)
1270	; - LDTR (reset to 0)
1271	; - DRx (presumably not changed at all)
1272	; - DR7 (reset to 0x400)
1273
1274	; Save all general purpose host registers.
1275	PUSH_CALLEE_PRESERVED_REGISTERS
1276	SEH64_END_PROLOGUE
1277	%if cbFrame != (cbBaseFrame + 8 * CALLEE_PRESERVED_REGISTER_COUNT)
1278	%error Bad cbFrame value
1279	%endif
1280
1281	; Shuffle parameter registers so that r8=HCPhysVmcb and rsi=pVCpu. (rdx & rcx will soon be trashed.)
1282	%ifdef ASM_CALL64_GCC
1283	mov r8, rdx ; Put HCPhysVmcb in r8 like on MSC as rdx is trashed below.
1284	%else
1285	mov rsi, rdx ; Put pVCpu in rsi like on GCC as rdx is trashed below.
1286	;mov rdi, rcx ; Put pVM in rdi like on GCC as rcx is trashed below.
1287	%endif
1288
1289	%ifdef VBOX_STRICT
1290	;
1291	; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change.
1292	;
1293	cmp byte [rsi + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], %2
1294	mov eax, VERR_SVM_VMRUN_PRECOND_0
1295	jne .failure_return
1296
1297	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fWorldSwitcher]
1298	and eax, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT
1299	cmp eax, %3
1300	mov eax, VERR_SVM_VMRUN_PRECOND_1
1301	jne .failure_return
1302
1303	%ifdef VBOX_WITH_KERNEL_USING_XMM
1304	mov eax, VERR_SVM_VMRUN_PRECOND_2
1305	test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1
1306	%if %4 = 0
1307	;jnz .failure_return
1308	%else
1309	jz .failure_return
1310
1311	mov eax, VERR_SVM_VMRUN_PRECOND_3
1312	cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0
1313	%if %4 = 1
1314	jne .failure_return
1315	%elif %4 = 2
1316	je .failure_return
1317	%else
1318	%error Invalid template parameter 4.
1319	%endif
1320	%endif
1321	%endif
1322	%endif ; VBOX_STRICT
1323
1324	%if %4 != 0
1325	; Save the non-volatile SSE host register state.
1326	movdqa [rbp + frm_saved_xmm6 ], xmm6
1327	movdqa [rbp + frm_saved_xmm7 ], xmm7
1328	movdqa [rbp + frm_saved_xmm8 ], xmm8
1329	movdqa [rbp + frm_saved_xmm9 ], xmm9
1330	movdqa [rbp + frm_saved_xmm10], xmm10
1331	movdqa [rbp + frm_saved_xmm11], xmm11
1332	movdqa [rbp + frm_saved_xmm12], xmm12
1333	movdqa [rbp + frm_saved_xmm13], xmm13
1334	movdqa [rbp + frm_saved_xmm14], xmm14
1335	movdqa [rbp + frm_saved_xmm15], xmm15
1336	stmxcsr [rbp + frm_saved_mxcsr]
1337
1338	; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx.
1339	mov rcx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.pXStateR0]
1340	%if %4 = 1 ; manual
1341	movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h]
1342	movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h]
1343	movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h]
1344	movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h]
1345	movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h]
1346	movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h]
1347	movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h]
1348	movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h]
1349	movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h]
1350	movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h]
1351	movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h]
1352	movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h]
1353	movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h]
1354	movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h]
1355	movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h]
1356	movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h]
1357	ldmxcsr [rcx + X86FXSTATE.MXCSR]
1358	%elif %4 = 2 ; use xrstor/xsave
1359	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask]
1360	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1361	xor edx, edx
1362	xrstor [rcx]
1363	%else
1364	%error invalid template parameter 4
1365	%endif
1366	%endif
1367
1368	%if %2 != 0
1369	; Save the host XCR0 and load the guest one if necessary.
1370	xor ecx, ecx
1371	xgetbv ; save the host XCR0 on the stack
1372	mov [rbp + frm_uHostXcr0 + 8], rdx
1373	mov [rbp + frm_uHostXcr0 ], rax
1374
1375	mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr] ; load the guest XCR0
1376	mov edx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr + 4]
1377	xor ecx, ecx ; paranoia
1378	xsetbv
1379	%endif
1380
1381	; Save host fs, gs, sysenter msr etc.
1382	mov rax, [rsi + VMCPU.hm + HMCPU.u + HMCPUSVM.HCPhysVmcbHost]
1383	mov qword [rbp + frm_HCPhysVmcbHost], rax ; save for the vmload after vmrun
1384	lea rsi, [rsi + VMCPU.cpum.GstCtx]
1385	mov qword [rbp + frm_pGstCtx], rsi
1386	vmsave
1387
1388	%if %3 & CPUMCTX_WSF_IBPB_ENTRY
1389	; Fight spectre (trashes rax, rdx and rcx).
1390	mov ecx, MSR_IA32_PRED_CMD
1391	mov eax, MSR_IA32_PRED_CMD_F_IBPB
1392	xor edx, edx
1393	wrmsr
1394	%endif
1395
1396	; Setup rax for VMLOAD.
1397	mov rax, r8 ; HCPhysVmcb (64 bits physical address; take low dword only)
1398
1399	; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
1400	mov rbx, qword [rsi + CPUMCTX.ebx]
1401	mov rcx, qword [rsi + CPUMCTX.ecx]
1402	mov rdx, qword [rsi + CPUMCTX.edx]
1403	mov rdi, qword [rsi + CPUMCTX.edi]
1404	mov rbp, qword [rsi + CPUMCTX.ebp]
1405	mov r8, qword [rsi + CPUMCTX.r8]
1406	mov r9, qword [rsi + CPUMCTX.r9]
1407	mov r10, qword [rsi + CPUMCTX.r10]
1408	mov r11, qword [rsi + CPUMCTX.r11]
1409	mov r12, qword [rsi + CPUMCTX.r12]
1410	mov r13, qword [rsi + CPUMCTX.r13]
1411	mov r14, qword [rsi + CPUMCTX.r14]
1412	mov r15, qword [rsi + CPUMCTX.r15]
1413	mov rsi, qword [rsi + CPUMCTX.esi]
1414
1415	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1416	clgi
1417	sti
1418
1419	; Load guest FS, GS, Sysenter MSRs etc.
1420	vmload
1421
1422	; Run the VM.
1423	vmrun
1424
1425	; Save guest fs, gs, sysenter msr etc.
1426	vmsave
1427
1428	; Load host fs, gs, sysenter msr etc.
1429	mov rax, [rsp + cbFrame + frm_HCPhysVmcbHost] ; load HCPhysVmcbHost (rbp is not operational yet, thus rsp)
1430	vmload
1431
1432	; Set the global interrupt flag again, but execute cli to make sure IF=0.
1433	cli
1434	stgi
1435
1436	; Pop pVCpu (saved above) and save the guest GPRs (sans RSP and RAX).
1437	mov rax, [rsp + cbFrame + frm_pGstCtx] ; (rbp still not operational)
1438
1439	mov qword [rax + CPUMCTX.ebp], rbp
1440	lea rbp, [rsp + cbFrame]
1441	mov qword [rax + CPUMCTX.ecx], rcx
1442	mov rcx, SPECTRE_FILLER
1443	mov qword [rax + CPUMCTX.edx], rdx
1444	mov rdx, rcx
1445	mov qword [rax + CPUMCTX.r8], r8
1446	mov r8, rcx
1447	mov qword [rax + CPUMCTX.r9], r9
1448	mov r9, rcx
1449	mov qword [rax + CPUMCTX.r10], r10
1450	mov r10, rcx
1451	mov qword [rax + CPUMCTX.r11], r11
1452	mov r11, rcx
1453	mov qword [rax + CPUMCTX.edi], rdi
1454	%ifdef ASM_CALL64_MSC
1455	mov rdi, [rbp + frm_saved_rdi]
1456	%else
1457	mov rdi, rcx
1458	%endif
1459	mov qword [rax + CPUMCTX.esi], rsi
1460	%ifdef ASM_CALL64_MSC
1461	mov rsi, [rbp + frm_saved_rsi]
1462	%else
1463	mov rsi, rcx
1464	%endif
1465	mov qword [rax + CPUMCTX.ebx], rbx
1466	mov rbx, [rbp + frm_saved_rbx]
1467	mov qword [rax + CPUMCTX.r12], r12
1468	mov r12, [rbp + frm_saved_r12]
1469	mov qword [rax + CPUMCTX.r13], r13
1470	mov r13, [rbp + frm_saved_r13]
1471	mov qword [rax + CPUMCTX.r14], r14
1472	mov r14, [rbp + frm_saved_r14]
1473	mov qword [rax + CPUMCTX.r15], r15
1474	mov r15, [rbp + frm_saved_r15]
1475
1476	%if %4 != 0
1477	; Set r8 = &pVCpu->cpum.GstCtx; for use below when saving and restoring SSE state.
1478	mov r8, rax
1479	%endif
1480
1481	%if %3 & CPUMCTX_WSF_IBPB_EXIT
1482	; Fight spectre (trashes rax, rdx and rcx).
1483	mov ecx, MSR_IA32_PRED_CMD
1484	mov eax, MSR_IA32_PRED_CMD_F_IBPB
1485	xor edx, edx
1486	wrmsr
1487	%endif
1488
1489	%if %2 != 0
1490	; Restore the host xcr0.
1491	xor ecx, ecx
1492	mov rdx, [rbp + frm_uHostXcr0 + 8]
1493	mov rax, [rbp + frm_uHostXcr0]
1494	xsetbv
1495	%endif
1496
1497	%if %4 != 0
1498	; Save the guest SSE state related to non-volatile and volatile SSE registers.
1499	mov rcx, [r8 + CPUMCTX.pXStateR0]
1500	%if %4 = 1 ; manual
1501	stmxcsr [rcx + X86FXSTATE.MXCSR]
1502	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1503	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1504	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1505	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1506	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1507	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1508	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1509	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1510	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1511	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1512	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1513	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1514	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1515	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1516	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1517	movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1518	%elif %4 = 2 ; use xrstor/xsave
1519	mov eax, [r8 + CPUMCTX.fXStateMask]
1520	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1521	xor edx, edx
1522	xsave [rcx]
1523	%else
1524	%error invalid template parameter 4
1525	%endif
1526
1527	; Restore the host non-volatile SSE register state.
1528	ldmxcsr [rbp + frm_saved_mxcsr]
1529	movdqa [rbp + frm_saved_xmm6 ], xmm6
1530	movdqa [rbp + frm_saved_xmm7 ], xmm7
1531	movdqa [rbp + frm_saved_xmm8 ], xmm8
1532	movdqa [rbp + frm_saved_xmm9 ], xmm9
1533	movdqa [rbp + frm_saved_xmm10], xmm10
1534	movdqa [rbp + frm_saved_xmm11], xmm11
1535	movdqa [rbp + frm_saved_xmm12], xmm12
1536	movdqa [rbp + frm_saved_xmm13], xmm13
1537	movdqa [rbp + frm_saved_xmm14], xmm14
1538	movdqa [rbp + frm_saved_xmm15], xmm15
1539	%endif ; %4 != 0
1540
1541	; Epilogue (assumes we restored volatile registers above when saving the guest GPRs).
1542	mov eax, VINF_SUCCESS
1543	add rsp, cbFrame - 8h
1544	popf
1545	leave
1546	ret
1547
1548	%ifdef VBOX_STRICT
1549	; Precondition checks failed.
1550	.failure_return:
1551	POP_CALLEE_PRESERVED_REGISTERS
1552	%if cbFrame != cbBaseFrame
1553	%error Bad frame size value: cbFrame
1554	%endif
1555	add rsp, cbFrame - 8h
1556	popf
1557	leave
1558	ret
1559	%endif
1560
1561	%undef frm_uHostXcr0
1562	%undef frm_fNoRestoreXcr0
1563	%undef frm_pVCpu
1564	%undef frm_HCPhysVmcbHost
1565	%undef cbFrame
1566	ENDPROC RT_CONCAT(hmR0SvmVmRun,%1)
1567
1568	%endmacro ; hmR0SvmVmRunTemplate
1569
1570	;
1571	; Instantiate the hmR0SvmVmRun various variations.
1572	;
1573	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit, 0, 0, 0
1574	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit, 1, 0, 0
1575	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY, 0
1576	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY, 0
1577	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_EXIT, 0
1578	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_EXIT, 0
1579	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 0
1580	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 0
1581	%ifdef VBOX_WITH_KERNEL_USING_XMM
1582	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 0, 0, 1
1583	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 1, 0, 1
1584	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_ENTRY, 1
1585	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_ENTRY, 1
1586	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_EXIT, 1
1587	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_EXIT, 1
1588	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 1
1589	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 1
1590
1591	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 0, 0, 2
1592	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 1, 0, 2
1593	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_ENTRY, 2
1594	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_ENTRY, 2
1595	hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_EXIT, 2
1596	hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_EXIT, 2
1597	hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 0, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 2
1598	hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 1, CPUMCTX_WSF_IBPB_ENTRY \| CPUMCTX_WSF_IBPB_EXIT, 2
1599	%endif
1600

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 87440

Download in other formats: