HMR0A.asm@ 72893

Last change on this file since 72893 was 72855, checked in by vboxsync, 6 years ago
VMM/HM: Use enum for Tagged-TLB flush types, and related cleanup.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 63.7 KB

Line
1	; $Id: HMR0A.asm 72855 2018-07-04 08:36:12Z vboxsync $
2	;; @file
3	; HM - Ring-0 VMX, SVM world-switch and helper routines
4	;
5
6	;
7	; Copyright (C) 2006-2017 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.virtualbox.org. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17
18	;*********************************************************************************************************************************
19	;* Header Files *
20	;*********************************************************************************************************************************
21	%include "VBox/asmdefs.mac"
22	%include "VBox/err.mac"
23	%include "VBox/vmm/hm_vmx.mac"
24	%include "VBox/vmm/cpum.mac"
25	%include "VBox/vmm/vm.mac"
26	%include "iprt/x86.mac"
27	%include "HMInternal.mac"
28
29	%ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely.
30	%macro vmwrite 2,
31	int3
32	%endmacro
33	%define vmlaunch int3
34	%define vmresume int3
35	%define vmsave int3
36	%define vmload int3
37	%define vmrun int3
38	%define clgi int3
39	%define stgi int3
40	%macro invlpga 2,
41	int3
42	%endmacro
43	%endif
44
45	;*********************************************************************************************************************************
46	;* Defined Constants And Macros *
47	;*********************************************************************************************************************************
48	;; The offset of the XMM registers in X86FXSTATE.
49	; Use define because I'm too lazy to convert the struct.
50	%define XMM_OFF_IN_X86FXSTATE 160
51
52	;; Spectre filler for 32-bit mode.
53	; Some user space address that points to a 4MB page boundrary in hope that it
54	; will somehow make it less useful.
55	%define SPECTRE_FILLER32 0x227fffff
56	;; Spectre filler for 64-bit mode.
57	; Choosen to be an invalid address (also with 5 level paging).
58	%define SPECTRE_FILLER64 0x02204204207fffff
59	;; Spectre filler for the current CPU mode.
60	%ifdef RT_ARCH_AMD64
61	%define SPECTRE_FILLER SPECTRE_FILLER64
62	%else
63	%define SPECTRE_FILLER SPECTRE_FILLER32
64	%endif
65
66	;;
67	; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation
68	;
69	%ifdef RT_ARCH_AMD64
70	%define VMX_SKIP_GDTR
71	%define VMX_SKIP_TR
72	%define VBOX_SKIP_RESTORE_SEG
73	%ifdef RT_OS_DARWIN
74	; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
75	; risk loading a stale LDT value or something invalid.
76	%define HM_64_BIT_USE_NULL_SEL
77	; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always.
78	; See @bugref{6875}.
79	%else
80	%define VMX_SKIP_IDTR
81	%endif
82	%endif
83
84	;; @def MYPUSHAD
85	; Macro generating an equivalent to pushad
86
87	;; @def MYPOPAD
88	; Macro generating an equivalent to popad
89
90	;; @def MYPUSHSEGS
91	; Macro saving all segment registers on the stack.
92	; @param 1 full width register name
93	; @param 2 16-bit register name for \a 1.
94
95	;; @def MYPOPSEGS
96	; Macro restoring all segment registers on the stack
97	; @param 1 full width register name
98	; @param 2 16-bit register name for \a 1.
99
100	%ifdef ASM_CALL64_GCC
101	%macro MYPUSHAD64 0
102	push r15
103	push r14
104	push r13
105	push r12
106	push rbx
107	%endmacro
108	%macro MYPOPAD64 0
109	pop rbx
110	pop r12
111	pop r13
112	pop r14
113	pop r15
114	%endmacro
115
116	%else ; ASM_CALL64_MSC
117	%macro MYPUSHAD64 0
118	push r15
119	push r14
120	push r13
121	push r12
122	push rbx
123	push rsi
124	push rdi
125	%endmacro
126	%macro MYPOPAD64 0
127	pop rdi
128	pop rsi
129	pop rbx
130	pop r12
131	pop r13
132	pop r14
133	pop r15
134	%endmacro
135	%endif
136
137	%ifdef VBOX_SKIP_RESTORE_SEG
138	%macro MYPUSHSEGS64 2
139	%endmacro
140
141	%macro MYPOPSEGS64 2
142	%endmacro
143	%else ; !VBOX_SKIP_RESTORE_SEG
144	; trashes, rax, rdx & rcx
145	%macro MYPUSHSEGS64 2
146	%ifndef HM_64_BIT_USE_NULL_SEL
147	mov %2, es
148	push %1
149	mov %2, ds
150	push %1
151	%endif
152
153	; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode, Solaris OTOH doesn't and we must save it.
154	mov ecx, MSR_K8_FS_BASE
155	rdmsr
156	push rdx
157	push rax
158	%ifndef HM_64_BIT_USE_NULL_SEL
159	push fs
160	%endif
161
162	; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel. The same happens on exit
163	mov ecx, MSR_K8_GS_BASE
164	rdmsr
165	push rdx
166	push rax
167	%ifndef HM_64_BIT_USE_NULL_SEL
168	push gs
169	%endif
170	%endmacro
171
172	; trashes, rax, rdx & rcx
173	%macro MYPOPSEGS64 2
174	; Note: do not step through this code with a debugger!
175	%ifndef HM_64_BIT_USE_NULL_SEL
176	xor eax, eax
177	mov ds, ax
178	mov es, ax
179	mov fs, ax
180	mov gs, ax
181	%endif
182
183	%ifndef HM_64_BIT_USE_NULL_SEL
184	pop gs
185	%endif
186	pop rax
187	pop rdx
188	mov ecx, MSR_K8_GS_BASE
189	wrmsr
190
191	%ifndef HM_64_BIT_USE_NULL_SEL
192	pop fs
193	%endif
194	pop rax
195	pop rdx
196	mov ecx, MSR_K8_FS_BASE
197	wrmsr
198	; Now it's safe to step again
199
200	%ifndef HM_64_BIT_USE_NULL_SEL
201	pop %1
202	mov ds, %2
203	pop %1
204	mov es, %2
205	%endif
206	%endmacro
207	%endif ; VBOX_SKIP_RESTORE_SEG
208
209	%macro MYPUSHAD32 0
210	pushad
211	%endmacro
212	%macro MYPOPAD32 0
213	popad
214	%endmacro
215
216	%macro MYPUSHSEGS32 2
217	push ds
218	push es
219	push fs
220	push gs
221	%endmacro
222	%macro MYPOPSEGS32 2
223	pop gs
224	pop fs
225	pop es
226	pop ds
227	%endmacro
228
229	%ifdef RT_ARCH_AMD64
230	%define MYPUSHAD MYPUSHAD64
231	%define MYPOPAD MYPOPAD64
232	%define MYPUSHSEGS MYPUSHSEGS64
233	%define MYPOPSEGS MYPOPSEGS64
234	%else
235	%define MYPUSHAD MYPUSHAD32
236	%define MYPOPAD MYPOPAD32
237	%define MYPUSHSEGS MYPUSHSEGS32
238	%define MYPOPSEGS MYPOPSEGS32
239	%endif
240
241	;;
242	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
243	; @clobbers eax, edx, ecx
244	; @param 1 How to address CPUMCTX.
245	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
246	%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2
247	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
248	jz %%no_indirect_branch_barrier
249	mov ecx, MSR_IA32_PRED_CMD
250	mov eax, MSR_IA32_PRED_CMD_F_IBPB
251	xor edx, edx
252	wrmsr
253	%%no_indirect_branch_barrier:
254	%endmacro
255
256
257	;*********************************************************************************************************************************
258	;* External Symbols *
259	;*********************************************************************************************************************************
260	%ifdef VBOX_WITH_KERNEL_USING_XMM
261	extern NAME(CPUMIsGuestFPUStateActive)
262	%endif
263
264
265	BEGINCODE
266
267
268	;/**
269	; * Restores host-state fields.
270	; *
271	; * @returns VBox status code
272	; * @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags.
273	; * @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct.
274	; */
275	ALIGNCODE(16)
276	BEGINPROC VMXRestoreHostState
277	%ifdef RT_ARCH_AMD64
278	%ifndef ASM_CALL64_GCC
279	; Use GCC's input registers since we'll be needing both rcx and rdx further
280	; down with the wrmsr instruction. Use the R10 and R11 register for saving
281	; RDI and RSI since MSC preserve the two latter registers.
282	mov r10, rdi
283	mov r11, rsi
284	mov rdi, rcx
285	mov rsi, rdx
286	%endif
287
288	test edi, VMX_RESTORE_HOST_GDTR
289	jz .test_idtr
290	lgdt [rsi + VMXRESTOREHOST.HostGdtr]
291
292	.test_idtr:
293	test edi, VMX_RESTORE_HOST_IDTR
294	jz .test_ds
295	lidt [rsi + VMXRESTOREHOST.HostIdtr]
296
297	.test_ds:
298	test edi, VMX_RESTORE_HOST_SEL_DS
299	jz .test_es
300	mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
301	mov ds, eax
302
303	.test_es:
304	test edi, VMX_RESTORE_HOST_SEL_ES
305	jz .test_tr
306	mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
307	mov es, eax
308
309	.test_tr:
310	test edi, VMX_RESTORE_HOST_SEL_TR
311	jz .test_fs
312	; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
313	mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
314	mov ax, dx
315	and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
316	test edi, VMX_RESTORE_HOST_GDT_READ_ONLY \| VMX_RESTORE_HOST_GDT_NEED_WRITABLE
317	jnz .gdt_readonly
318	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
319	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
320	ltr dx
321	jmp short .test_fs
322	.gdt_readonly:
323	test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
324	jnz .gdt_readonly_need_writable
325	mov rcx, cr0
326	mov r9, rcx
327	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
328	and rcx, ~X86_CR0_WP
329	mov cr0, rcx
330	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
331	ltr dx
332	mov cr0, r9
333	jmp short .test_fs
334	.gdt_readonly_need_writable:
335	add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw.
336	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
337	lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
338	ltr dx
339	lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; Load the original GDT
340
341	.test_fs:
342	;
343	; When restoring the selector values for FS and GS, we'll temporarily trash
344	; the base address (at least the high 32-bit bits, but quite possibly the
345	; whole base address), the wrmsr will restore it correctly. (VT-x actually
346	; restores the base correctly when leaving guest mode, but not the selector
347	; value, so there is little problem with interrupts being enabled prior to
348	; this restore job.)
349	; We'll disable ints once for both FS and GS as that's probably faster.
350	;
351	test edi, VMX_RESTORE_HOST_SEL_FS \| VMX_RESTORE_HOST_SEL_GS
352	jz .restore_success
353	pushfq
354	cli ; (see above)
355
356	test edi, VMX_RESTORE_HOST_SEL_FS
357	jz .test_gs
358	mov ax, word [rsi + VMXRESTOREHOST.uHostSelFS]
359	mov fs, eax
360	mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
361	mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
362	mov ecx, MSR_K8_FS_BASE
363	wrmsr
364
365	.test_gs:
366	test edi, VMX_RESTORE_HOST_SEL_GS
367	jz .restore_flags
368	mov ax, word [rsi + VMXRESTOREHOST.uHostSelGS]
369	mov gs, eax
370	mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
371	mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
372	mov ecx, MSR_K8_GS_BASE
373	wrmsr
374
375	.restore_flags:
376	popfq
377
378	.restore_success:
379	mov eax, VINF_SUCCESS
380	%ifndef ASM_CALL64_GCC
381	; Restore RDI and RSI on MSC.
382	mov rdi, r10
383	mov rsi, r11
384	%endif
385	%else ; RT_ARCH_X86
386	mov eax, VERR_NOT_IMPLEMENTED
387	%endif
388	ret
389	ENDPROC VMXRestoreHostState
390
391
392	;/**
393	; * Dispatches an NMI to the host.
394	; */
395	ALIGNCODE(16)
396	BEGINPROC VMXDispatchHostNmi
397	int 2 ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
398	ret
399	ENDPROC VMXDispatchHostNmi
400
401
402	;/**
403	; * Executes VMWRITE, 64-bit value.
404	; *
405	; * @returns VBox status code.
406	; * @param idxField x86: [ebp + 08h] msc: rcx gcc: rdi VMCS index.
407	; * @param u64Data x86: [ebp + 0ch] msc: rdx gcc: rsi VM field value.
408	; */
409	ALIGNCODE(16)
410	BEGINPROC VMXWriteVmcs64
411	%ifdef RT_ARCH_AMD64
412	%ifdef ASM_CALL64_GCC
413	and edi, 0ffffffffh
414	xor rax, rax
415	vmwrite rdi, rsi
416	%else
417	and ecx, 0ffffffffh
418	xor rax, rax
419	vmwrite rcx, rdx
420	%endif
421	%else ; RT_ARCH_X86
422	mov ecx, [esp + 4] ; idxField
423	lea edx, [esp + 8] ; &u64Data
424	vmwrite ecx, [edx] ; low dword
425	jz .done
426	jc .done
427	inc ecx
428	xor eax, eax
429	vmwrite ecx, [edx + 4] ; high dword
430	.done:
431	%endif ; RT_ARCH_X86
432	jnc .valid_vmcs
433	mov eax, VERR_VMX_INVALID_VMCS_PTR
434	ret
435	.valid_vmcs:
436	jnz .the_end
437	mov eax, VERR_VMX_INVALID_VMCS_FIELD
438	.the_end:
439	ret
440	ENDPROC VMXWriteVmcs64
441
442
443	;/**
444	; * Executes VMREAD, 64-bit value.
445	; *
446	; * @returns VBox status code.
447	; * @param idxField VMCS index.
448	; * @param pData Where to store VM field value.
449	; */
450	;DECLASM(int) VMXReadVmcs64(uint32_t idxField, uint64_t *pData);
451	ALIGNCODE(16)
452	BEGINPROC VMXReadVmcs64
453	%ifdef RT_ARCH_AMD64
454	%ifdef ASM_CALL64_GCC
455	and edi, 0ffffffffh
456	xor rax, rax
457	vmread [rsi], rdi
458	%else
459	and ecx, 0ffffffffh
460	xor rax, rax
461	vmread [rdx], rcx
462	%endif
463	%else ; RT_ARCH_X86
464	mov ecx, [esp + 4] ; idxField
465	mov edx, [esp + 8] ; pData
466	vmread [edx], ecx ; low dword
467	jz .done
468	jc .done
469	inc ecx
470	xor eax, eax
471	vmread [edx + 4], ecx ; high dword
472	.done:
473	%endif ; RT_ARCH_X86
474	jnc .valid_vmcs
475	mov eax, VERR_VMX_INVALID_VMCS_PTR
476	ret
477	.valid_vmcs:
478	jnz .the_end
479	mov eax, VERR_VMX_INVALID_VMCS_FIELD
480	.the_end:
481	ret
482	ENDPROC VMXReadVmcs64
483
484
485	;/**
486	; * Executes VMREAD, 32-bit value.
487	; *
488	; * @returns VBox status code.
489	; * @param idxField VMCS index.
490	; * @param pu32Data Where to store VM field value.
491	; */
492	;DECLASM(int) VMXReadVmcs32(uint32_t idxField, uint32_t *pu32Data);
493	ALIGNCODE(16)
494	BEGINPROC VMXReadVmcs32
495	%ifdef RT_ARCH_AMD64
496	%ifdef ASM_CALL64_GCC
497	and edi, 0ffffffffh
498	xor rax, rax
499	vmread r10, rdi
500	mov [rsi], r10d
501	%else
502	and ecx, 0ffffffffh
503	xor rax, rax
504	vmread r10, rcx
505	mov [rdx], r10d
506	%endif
507	%else ; RT_ARCH_X86
508	mov ecx, [esp + 4] ; idxField
509	mov edx, [esp + 8] ; pu32Data
510	xor eax, eax
511	vmread [edx], ecx
512	%endif ; RT_ARCH_X86
513	jnc .valid_vmcs
514	mov eax, VERR_VMX_INVALID_VMCS_PTR
515	ret
516	.valid_vmcs:
517	jnz .the_end
518	mov eax, VERR_VMX_INVALID_VMCS_FIELD
519	.the_end:
520	ret
521	ENDPROC VMXReadVmcs32
522
523
524	;/**
525	; * Executes VMWRITE, 32-bit value.
526	; *
527	; * @returns VBox status code.
528	; * @param idxField VMCS index.
529	; * @param u32Data Where to store VM field value.
530	; */
531	;DECLASM(int) VMXWriteVmcs32(uint32_t idxField, uint32_t u32Data);
532	ALIGNCODE(16)
533	BEGINPROC VMXWriteVmcs32
534	%ifdef RT_ARCH_AMD64
535	%ifdef ASM_CALL64_GCC
536	and edi, 0ffffffffh
537	and esi, 0ffffffffh
538	xor rax, rax
539	vmwrite rdi, rsi
540	%else
541	and ecx, 0ffffffffh
542	and edx, 0ffffffffh
543	xor rax, rax
544	vmwrite rcx, rdx
545	%endif
546	%else ; RT_ARCH_X86
547	mov ecx, [esp + 4] ; idxField
548	mov edx, [esp + 8] ; u32Data
549	xor eax, eax
550	vmwrite ecx, edx
551	%endif ; RT_ARCH_X86
552	jnc .valid_vmcs
553	mov eax, VERR_VMX_INVALID_VMCS_PTR
554	ret
555	.valid_vmcs:
556	jnz .the_end
557	mov eax, VERR_VMX_INVALID_VMCS_FIELD
558	.the_end:
559	ret
560	ENDPROC VMXWriteVmcs32
561
562
563	;/**
564	; * Executes VMXON.
565	; *
566	; * @returns VBox status code.
567	; * @param HCPhysVMXOn Physical address of VMXON structure.
568	; */
569	;DECLASM(int) VMXEnable(RTHCPHYS HCPhysVMXOn);
570	BEGINPROC VMXEnable
571	%ifdef RT_ARCH_AMD64
572	xor rax, rax
573	%ifdef ASM_CALL64_GCC
574	push rdi
575	%else
576	push rcx
577	%endif
578	vmxon [rsp]
579	%else ; RT_ARCH_X86
580	xor eax, eax
581	vmxon [esp + 4]
582	%endif ; RT_ARCH_X86
583	jnc .good
584	mov eax, VERR_VMX_INVALID_VMXON_PTR
585	jmp .the_end
586
587	.good:
588	jnz .the_end
589	mov eax, VERR_VMX_VMXON_FAILED
590
591	.the_end:
592	%ifdef RT_ARCH_AMD64
593	add rsp, 8
594	%endif
595	ret
596	ENDPROC VMXEnable
597
598
599	;/**
600	; * Executes VMXOFF.
601	; */
602	;DECLASM(void) VMXDisable(void);
603	BEGINPROC VMXDisable
604	vmxoff
605	.the_end:
606	ret
607	ENDPROC VMXDisable
608
609
610	;/**
611	; * Executes VMCLEAR.
612	; *
613	; * @returns VBox status code.
614	; * @param HCPhysVmcs Physical address of VM control structure.
615	; */
616	;DECLASM(int) VMXClearVmcs(RTHCPHYS HCPhysVmcs);
617	ALIGNCODE(16)
618	BEGINPROC VMXClearVmcs
619	%ifdef RT_ARCH_AMD64
620	xor rax, rax
621	%ifdef ASM_CALL64_GCC
622	push rdi
623	%else
624	push rcx
625	%endif
626	vmclear [rsp]
627	%else ; RT_ARCH_X86
628	xor eax, eax
629	vmclear [esp + 4]
630	%endif ; RT_ARCH_X86
631	jnc .the_end
632	mov eax, VERR_VMX_INVALID_VMCS_PTR
633	.the_end:
634	%ifdef RT_ARCH_AMD64
635	add rsp, 8
636	%endif
637	ret
638	ENDPROC VMXClearVmcs
639
640
641	;/**
642	; * Executes VMPTRLD.
643	; *
644	; * @returns VBox status code.
645	; * @param HCPhysVmcs Physical address of VMCS structure.
646	; */
647	;DECLASM(int) VMXActivateVmcs(RTHCPHYS HCPhysVmcs);
648	ALIGNCODE(16)
649	BEGINPROC VMXActivateVmcs
650	%ifdef RT_ARCH_AMD64
651	xor rax, rax
652	%ifdef ASM_CALL64_GCC
653	push rdi
654	%else
655	push rcx
656	%endif
657	vmptrld [rsp]
658	%else
659	xor eax, eax
660	vmptrld [esp + 4]
661	%endif
662	jnc .the_end
663	mov eax, VERR_VMX_INVALID_VMCS_PTR
664	.the_end:
665	%ifdef RT_ARCH_AMD64
666	add rsp, 8
667	%endif
668	ret
669	ENDPROC VMXActivateVmcs
670
671
672	;/**
673	; * Executes VMPTRST.
674	; *
675	; * @returns VBox status code.
676	; * @param [esp + 04h] gcc:rdi msc:rcx Param 1 - First parameter - Address that will receive the current pointer.
677	; */
678	;DECLASM(int) VMXGetActivatedVmcs(RTHCPHYS *pVMCS);
679	BEGINPROC VMXGetActivatedVmcs
680	%ifdef RT_OS_OS2
681	mov eax, VERR_NOT_SUPPORTED
682	ret
683	%else
684	%ifdef RT_ARCH_AMD64
685	%ifdef ASM_CALL64_GCC
686	vmptrst qword [rdi]
687	%else
688	vmptrst qword [rcx]
689	%endif
690	%else
691	vmptrst qword [esp+04h]
692	%endif
693	xor eax, eax
694	.the_end:
695	ret
696	%endif
697	ENDPROC VMXGetActivatedVmcs
698
699	;/**
700	; * Invalidate a page using INVEPT.
701	; @param enmTlbFlush msc:ecx gcc:edi x86:[esp+04] Type of flush.
702	; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer.
703	; */
704	;DECLASM(int) VMXR0InvEPT(VMXTLBFLUSHEPT enmTlbFlush, uint64_t *pDescriptor);
705	BEGINPROC VMXR0InvEPT
706	%ifdef RT_ARCH_AMD64
707	%ifdef ASM_CALL64_GCC
708	and edi, 0ffffffffh
709	xor rax, rax
710	; invept rdi, qword [rsi]
711	DB 0x66, 0x0F, 0x38, 0x80, 0x3E
712	%else
713	and ecx, 0ffffffffh
714	xor rax, rax
715	; invept rcx, qword [rdx]
716	DB 0x66, 0x0F, 0x38, 0x80, 0xA
717	%endif
718	%else
719	mov ecx, [esp + 4]
720	mov edx, [esp + 8]
721	xor eax, eax
722	; invept ecx, qword [edx]
723	DB 0x66, 0x0F, 0x38, 0x80, 0xA
724	%endif
725	jnc .valid_vmcs
726	mov eax, VERR_VMX_INVALID_VMCS_PTR
727	ret
728	.valid_vmcs:
729	jnz .the_end
730	mov eax, VERR_INVALID_PARAMETER
731	.the_end:
732	ret
733	ENDPROC VMXR0InvEPT
734
735
736	;/**
737	; * Invalidate a page using invvpid
738	; @param enmTlbFlush msc:ecx gcc:edi x86:[esp+04] Type of flush
739	; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer
740	; */
741	;DECLASM(int) VMXR0InvVPID(VMXTLBFLUSHVPID enmTlbFlush, uint64_t *pDescriptor);
742	BEGINPROC VMXR0InvVPID
743	%ifdef RT_ARCH_AMD64
744	%ifdef ASM_CALL64_GCC
745	and edi, 0ffffffffh
746	xor rax, rax
747	; invvpid rdi, qword [rsi]
748	DB 0x66, 0x0F, 0x38, 0x81, 0x3E
749	%else
750	and ecx, 0ffffffffh
751	xor rax, rax
752	; invvpid rcx, qword [rdx]
753	DB 0x66, 0x0F, 0x38, 0x81, 0xA
754	%endif
755	%else
756	mov ecx, [esp + 4]
757	mov edx, [esp + 8]
758	xor eax, eax
759	; invvpid ecx, qword [edx]
760	DB 0x66, 0x0F, 0x38, 0x81, 0xA
761	%endif
762	jnc .valid_vmcs
763	mov eax, VERR_VMX_INVALID_VMCS_PTR
764	ret
765	.valid_vmcs:
766	jnz .the_end
767	mov eax, VERR_INVALID_PARAMETER
768	.the_end:
769	ret
770	ENDPROC VMXR0InvVPID
771
772
773	%if GC_ARCH_BITS == 64
774	;;
775	; Executes INVLPGA
776	;
777	; @param pPageGC msc:rcx gcc:rdi x86:[esp+04] Virtual page to invalidate
778	; @param uASID msc:rdx gcc:rsi x86:[esp+0C] Tagged TLB id
779	;
780	;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
781	BEGINPROC SVMR0InvlpgA
782	%ifdef RT_ARCH_AMD64
783	%ifdef ASM_CALL64_GCC
784	mov rax, rdi
785	mov rcx, rsi
786	%else
787	mov rax, rcx
788	mov rcx, rdx
789	%endif
790	%else
791	mov eax, [esp + 4]
792	mov ecx, [esp + 0Ch]
793	%endif
794	invlpga [xAX], ecx
795	ret
796	ENDPROC SVMR0InvlpgA
797
798	%else ; GC_ARCH_BITS != 64
799	;;
800	; Executes INVLPGA
801	;
802	; @param pPageGC msc:ecx gcc:edi x86:[esp+04] Virtual page to invalidate
803	; @param uASID msc:edx gcc:esi x86:[esp+08] Tagged TLB id
804	;
805	;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
806	BEGINPROC SVMR0InvlpgA
807	%ifdef RT_ARCH_AMD64
808	%ifdef ASM_CALL64_GCC
809	movzx rax, edi
810	mov ecx, esi
811	%else
812	; from http://www.cs.cmu.edu/~fp/courses/15213-s06/misc/asm64-handout.pdf:
813	; ``Perhaps unexpectedly, instructions that move or generate 32-bit register
814	; values also set the upper 32 bits of the register to zero. Consequently
815	; there is no need for an instruction movzlq.''
816	mov eax, ecx
817	mov ecx, edx
818	%endif
819	%else
820	mov eax, [esp + 4]
821	mov ecx, [esp + 8]
822	%endif
823	invlpga [xAX], ecx
824	ret
825	ENDPROC SVMR0InvlpgA
826
827	%endif ; GC_ARCH_BITS != 64
828
829
830	%ifdef VBOX_WITH_KERNEL_USING_XMM
831
832	;;
833	; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
834	; load the guest ones when necessary.
835	;
836	; @cproto DECLASM(int) HMR0VMXStartVMhmR0DumpDescriptorM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM,
837	; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
838	;
839	; @returns eax
840	;
841	; @param fResumeVM msc:rcx
842	; @param pCtx msc:rdx
843	; @param pVMCSCache msc:r8
844	; @param pVM msc:r9
845	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
846	; @param pfnStartVM msc:[rbp+38h]
847	;
848	; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
849	;
850	; @remarks Drivers shouldn't use AVX registers without saving+loading:
851	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
852	; However the compiler docs have different idea:
853	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
854	; We'll go with the former for now.
855	;
856	; ASSUMING 64-bit and windows for now.
857	;
858	ALIGNCODE(16)
859	BEGINPROC hmR0VMXStartVMWrapXMM
860	push xBP
861	mov xBP, xSP
862	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
863
864	; spill input parameters.
865	mov [xBP + 010h], rcx ; fResumeVM
866	mov [xBP + 018h], rdx ; pCtx
867	mov [xBP + 020h], r8 ; pVMCSCache
868	mov [xBP + 028h], r9 ; pVM
869
870	; Ask CPUM whether we've started using the FPU yet.
871	mov rcx, [xBP + 30h] ; pVCpu
872	call NAME(CPUMIsGuestFPUStateActive)
873	test al, al
874	jnz .guest_fpu_state_active
875
876	; No need to mess with XMM registers just call the start routine and return.
877	mov r11, [xBP + 38h] ; pfnStartVM
878	mov r10, [xBP + 30h] ; pVCpu
879	mov [xSP + 020h], r10
880	mov rcx, [xBP + 010h] ; fResumeVM
881	mov rdx, [xBP + 018h] ; pCtx
882	mov r8, [xBP + 020h] ; pVMCSCache
883	mov r9, [xBP + 028h] ; pVM
884	call r11
885
886	leave
887	ret
888
889	ALIGNCODE(8)
890	.guest_fpu_state_active:
891	; Save the non-volatile host XMM registers.
892	movdqa [rsp + 040h + 000h], xmm6
893	movdqa [rsp + 040h + 010h], xmm7
894	movdqa [rsp + 040h + 020h], xmm8
895	movdqa [rsp + 040h + 030h], xmm9
896	movdqa [rsp + 040h + 040h], xmm10
897	movdqa [rsp + 040h + 050h], xmm11
898	movdqa [rsp + 040h + 060h], xmm12
899	movdqa [rsp + 040h + 070h], xmm13
900	movdqa [rsp + 040h + 080h], xmm14
901	movdqa [rsp + 040h + 090h], xmm15
902	stmxcsr [rsp + 040h + 0a0h]
903
904	mov r10, [xBP + 018h] ; pCtx
905	mov eax, [r10 + CPUMCTX.fXStateMask]
906	test eax, eax
907	jz .guest_fpu_state_manually
908
909	;
910	; Using XSAVE to load the guest XMM, YMM and ZMM registers.
911	;
912	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
913	xor edx, edx
914	mov r10, [r10 + CPUMCTX.pXStateR0]
915	xrstor [r10]
916
917	; Make the call (same as in the other case ).
918	mov r11, [xBP + 38h] ; pfnStartVM
919	mov r10, [xBP + 30h] ; pVCpu
920	mov [xSP + 020h], r10
921	mov rcx, [xBP + 010h] ; fResumeVM
922	mov rdx, [xBP + 018h] ; pCtx
923	mov r8, [xBP + 020h] ; pVMCSCache
924	mov r9, [xBP + 028h] ; pVM
925	call r11
926
927	mov r11d, eax ; save return value (xsave below uses eax)
928
929	; Save the guest XMM registers.
930	mov r10, [xBP + 018h] ; pCtx
931	mov eax, [r10 + CPUMCTX.fXStateMask]
932	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
933	xor edx, edx
934	mov r10, [r10 + CPUMCTX.pXStateR0]
935	xsave [r10]
936
937	mov eax, r11d ; restore return value.
938
939	.restore_non_volatile_host_xmm_regs:
940	; Load the non-volatile host XMM registers.
941	movdqa xmm6, [rsp + 040h + 000h]
942	movdqa xmm7, [rsp + 040h + 010h]
943	movdqa xmm8, [rsp + 040h + 020h]
944	movdqa xmm9, [rsp + 040h + 030h]
945	movdqa xmm10, [rsp + 040h + 040h]
946	movdqa xmm11, [rsp + 040h + 050h]
947	movdqa xmm12, [rsp + 040h + 060h]
948	movdqa xmm13, [rsp + 040h + 070h]
949	movdqa xmm14, [rsp + 040h + 080h]
950	movdqa xmm15, [rsp + 040h + 090h]
951	ldmxcsr [rsp + 040h + 0a0h]
952	leave
953	ret
954
955	;
956	; No XSAVE, load and save the guest XMM registers manually.
957	;
958	.guest_fpu_state_manually:
959	; Load the full guest XMM register state.
960	mov r10, [r10 + CPUMCTX.pXStateR0]
961	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
962	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
963	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
964	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
965	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
966	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
967	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
968	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
969	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
970	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
971	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
972	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
973	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
974	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
975	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
976	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
977	ldmxcsr [r10 + X86FXSTATE.MXCSR]
978
979	; Make the call (same as in the other case ).
980	mov r11, [xBP + 38h] ; pfnStartVM
981	mov r10, [xBP + 30h] ; pVCpu
982	mov [xSP + 020h], r10
983	mov rcx, [xBP + 010h] ; fResumeVM
984	mov rdx, [xBP + 018h] ; pCtx
985	mov r8, [xBP + 020h] ; pVMCSCache
986	mov r9, [xBP + 028h] ; pVM
987	call r11
988
989	; Save the guest XMM registers.
990	mov r10, [xBP + 018h] ; pCtx
991	mov r10, [r10 + CPUMCTX.pXStateR0]
992	stmxcsr [r10 + X86FXSTATE.MXCSR]
993	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
994	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
995	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
996	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
997	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
998	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
999	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1000	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1001	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1002	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1003	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1004	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1005	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1006	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1007	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1008	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1009	jmp .restore_non_volatile_host_xmm_regs
1010	ENDPROC hmR0VMXStartVMWrapXMM
1011
1012	;;
1013	; Wrapper around svm.pfnVMRun that preserves host XMM registers and
1014	; load the guest ones when necessary.
1015	;
1016	; @cproto DECLASM(int) hmR0SVMRunWrapXMM(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu,
1017	; PFNHMSVMVMRUN pfnVMRun);
1018	;
1019	; @returns eax
1020	;
1021	; @param HCPhysVmcbHost msc:rcx
1022	; @param HCPhysVmcb msc:rdx
1023	; @param pCtx msc:r8
1024	; @param pVM msc:r9
1025	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
1026	; @param pfnVMRun msc:[rbp+38h]
1027	;
1028	; @remarks This is essentially the same code as hmR0VMXStartVMWrapXMM, only the parameters differ a little bit.
1029	;
1030	; @remarks Drivers shouldn't use AVX registers without saving+loading:
1031	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
1032	; However the compiler docs have different idea:
1033	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
1034	; We'll go with the former for now.
1035	;
1036	; ASSUMING 64-bit and windows for now.
1037	ALIGNCODE(16)
1038	BEGINPROC hmR0SVMRunWrapXMM
1039	push xBP
1040	mov xBP, xSP
1041	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
1042
1043	; spill input parameters.
1044	mov [xBP + 010h], rcx ; HCPhysVmcbHost
1045	mov [xBP + 018h], rdx ; HCPhysVmcb
1046	mov [xBP + 020h], r8 ; pCtx
1047	mov [xBP + 028h], r9 ; pVM
1048
1049	; Ask CPUM whether we've started using the FPU yet.
1050	mov rcx, [xBP + 30h] ; pVCpu
1051	call NAME(CPUMIsGuestFPUStateActive)
1052	test al, al
1053	jnz .guest_fpu_state_active
1054
1055	; No need to mess with XMM registers just call the start routine and return.
1056	mov r11, [xBP + 38h] ; pfnVMRun
1057	mov r10, [xBP + 30h] ; pVCpu
1058	mov [xSP + 020h], r10
1059	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1060	mov rdx, [xBP + 018h] ; HCPhysVmcb
1061	mov r8, [xBP + 020h] ; pCtx
1062	mov r9, [xBP + 028h] ; pVM
1063	call r11
1064
1065	leave
1066	ret
1067
1068	ALIGNCODE(8)
1069	.guest_fpu_state_active:
1070	; Save the non-volatile host XMM registers.
1071	movdqa [rsp + 040h + 000h], xmm6
1072	movdqa [rsp + 040h + 010h], xmm7
1073	movdqa [rsp + 040h + 020h], xmm8
1074	movdqa [rsp + 040h + 030h], xmm9
1075	movdqa [rsp + 040h + 040h], xmm10
1076	movdqa [rsp + 040h + 050h], xmm11
1077	movdqa [rsp + 040h + 060h], xmm12
1078	movdqa [rsp + 040h + 070h], xmm13
1079	movdqa [rsp + 040h + 080h], xmm14
1080	movdqa [rsp + 040h + 090h], xmm15
1081	stmxcsr [rsp + 040h + 0a0h]
1082
1083	mov r10, [xBP + 020h] ; pCtx
1084	mov eax, [r10 + CPUMCTX.fXStateMask]
1085	test eax, eax
1086	jz .guest_fpu_state_manually
1087
1088	;
1089	; Using XSAVE.
1090	;
1091	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1092	xor edx, edx
1093	mov r10, [r10 + CPUMCTX.pXStateR0]
1094	xrstor [r10]
1095
1096	; Make the call (same as in the other case ).
1097	mov r11, [xBP + 38h] ; pfnVMRun
1098	mov r10, [xBP + 30h] ; pVCpu
1099	mov [xSP + 020h], r10
1100	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1101	mov rdx, [xBP + 018h] ; HCPhysVmcb
1102	mov r8, [xBP + 020h] ; pCtx
1103	mov r9, [xBP + 028h] ; pVM
1104	call r11
1105
1106	mov r11d, eax ; save return value (xsave below uses eax)
1107
1108	; Save the guest XMM registers.
1109	mov r10, [xBP + 020h] ; pCtx
1110	mov eax, [r10 + CPUMCTX.fXStateMask]
1111	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1112	xor edx, edx
1113	mov r10, [r10 + CPUMCTX.pXStateR0]
1114	xsave [r10]
1115
1116	mov eax, r11d ; restore return value.
1117
1118	.restore_non_volatile_host_xmm_regs:
1119	; Load the non-volatile host XMM registers.
1120	movdqa xmm6, [rsp + 040h + 000h]
1121	movdqa xmm7, [rsp + 040h + 010h]
1122	movdqa xmm8, [rsp + 040h + 020h]
1123	movdqa xmm9, [rsp + 040h + 030h]
1124	movdqa xmm10, [rsp + 040h + 040h]
1125	movdqa xmm11, [rsp + 040h + 050h]
1126	movdqa xmm12, [rsp + 040h + 060h]
1127	movdqa xmm13, [rsp + 040h + 070h]
1128	movdqa xmm14, [rsp + 040h + 080h]
1129	movdqa xmm15, [rsp + 040h + 090h]
1130	ldmxcsr [rsp + 040h + 0a0h]
1131	leave
1132	ret
1133
1134	;
1135	; No XSAVE, load and save the guest XMM registers manually.
1136	;
1137	.guest_fpu_state_manually:
1138	; Load the full guest XMM register state.
1139	mov r10, [r10 + CPUMCTX.pXStateR0]
1140	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
1141	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
1142	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
1143	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
1144	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
1145	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
1146	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
1147	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
1148	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
1149	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
1150	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
1151	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
1152	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
1153	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
1154	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
1155	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
1156	ldmxcsr [r10 + X86FXSTATE.MXCSR]
1157
1158	; Make the call (same as in the other case ).
1159	mov r11, [xBP + 38h] ; pfnVMRun
1160	mov r10, [xBP + 30h] ; pVCpu
1161	mov [xSP + 020h], r10
1162	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1163	mov rdx, [xBP + 018h] ; HCPhysVmcb
1164	mov r8, [xBP + 020h] ; pCtx
1165	mov r9, [xBP + 028h] ; pVM
1166	call r11
1167
1168	; Save the guest XMM registers.
1169	mov r10, [xBP + 020h] ; pCtx
1170	mov r10, [r10 + CPUMCTX.pXStateR0]
1171	stmxcsr [r10 + X86FXSTATE.MXCSR]
1172	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1173	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1174	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1175	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1176	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1177	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1178	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1179	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1180	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1181	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1182	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1183	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1184	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1185	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1186	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1187	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1188	jmp .restore_non_volatile_host_xmm_regs
1189	ENDPROC hmR0SVMRunWrapXMM
1190
1191	%endif ; VBOX_WITH_KERNEL_USING_XMM
1192
1193
1194	;; @def RESTORE_STATE_VM32
1195	; Macro restoring essential host state and updating guest state
1196	; for common host, 32-bit guest for VT-x.
1197	%macro RESTORE_STATE_VM32 0
1198	; Restore base and limit of the IDTR & GDTR.
1199	%ifndef VMX_SKIP_IDTR
1200	lidt [xSP]
1201	add xSP, xCB * 2
1202	%endif
1203	%ifndef VMX_SKIP_GDTR
1204	lgdt [xSP]
1205	add xSP, xCB * 2
1206	%endif
1207
1208	push xDI
1209	%ifndef VMX_SKIP_TR
1210	mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR).
1211	%else
1212	mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR).
1213	%endif
1214
1215	mov [ss:xDI + CPUMCTX.eax], eax
1216	mov xAX, SPECTRE_FILLER
1217	mov [ss:xDI + CPUMCTX.ebx], ebx
1218	mov xBX, xAX
1219	mov [ss:xDI + CPUMCTX.ecx], ecx
1220	mov xCX, xAX
1221	mov [ss:xDI + CPUMCTX.edx], edx
1222	mov xDX, xAX
1223	mov [ss:xDI + CPUMCTX.esi], esi
1224	mov xSI, xAX
1225	mov [ss:xDI + CPUMCTX.ebp], ebp
1226	mov xBP, xAX
1227	mov xAX, cr2
1228	mov [ss:xDI + CPUMCTX.cr2], xAX
1229
1230	%ifdef RT_ARCH_AMD64
1231	pop xAX ; The guest edi we pushed above.
1232	mov dword [ss:xDI + CPUMCTX.edi], eax
1233	%else
1234	pop dword [ss:xDI + CPUMCTX.edi] ; The guest edi we pushed above.
1235	%endif
1236
1237	; Fight spectre.
1238	INDIRECT_BRANCH_PREDICTION_BARRIER ss:xDI, CPUMCTX_WSF_IBPB_EXIT
1239
1240	%ifndef VMX_SKIP_TR
1241	; Restore TSS selector; must mark it as not busy before using ltr (!)
1242	; ASSUME that this is supposed to be 'BUSY'. (saves 20-30 ticks on the T42p)
1243	; @todo get rid of sgdt
1244	pop xBX ; Saved TR
1245	sub xSP, xCB * 2
1246	sgdt [xSP]
1247	mov xAX, xBX
1248	and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
1249	add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset.
1250	and dword [ss:xAX + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
1251	ltr bx
1252	add xSP, xCB * 2
1253	%endif
1254
1255	pop xAX ; Saved LDTR
1256	%ifdef RT_ARCH_AMD64
1257	cmp eax, 0
1258	je %%skip_ldt_write32
1259	%endif
1260	lldt ax
1261
1262	%%skip_ldt_write32:
1263	add xSP, xCB ; pCtx
1264
1265	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1266	pop xDX ; Saved pCache
1267
1268	; Note! If we get here as a result of invalid VMCS pointer, all the following
1269	; vmread's will fail (only eflags.cf=1 will be set) but that shouldn't cause any
1270	; trouble only just less efficient.
1271	mov ecx, [ss:xDX + VMCSCACHE.Read.cValidEntries]
1272	cmp ecx, 0 ; Can't happen
1273	je %%no_cached_read32
1274	jmp %%cached_read32
1275
1276	ALIGN(16)
1277	%%cached_read32:
1278	dec xCX
1279	mov eax, [ss:xDX + VMCSCACHE.Read.aField + xCX * 4]
1280	; Note! This leaves the high 32 bits of the cache entry unmodified!!
1281	vmread [ss:xDX + VMCSCACHE.Read.aFieldVal + xCX * 8], xAX
1282	cmp xCX, 0
1283	jnz %%cached_read32
1284	%%no_cached_read32:
1285	%endif
1286
1287	; Restore segment registers.
1288	MYPOPSEGS xAX, ax
1289
1290	; Restore the host XCR0 if necessary.
1291	pop xCX
1292	test ecx, ecx
1293	jnz %%xcr0_after_skip
1294	pop xAX
1295	pop xDX
1296	xsetbv ; ecx is already zero.
1297	%%xcr0_after_skip:
1298
1299	; Restore general purpose registers.
1300	MYPOPAD
1301	%endmacro
1302
1303
1304	;;
1305	; Prepares for and executes VMLAUNCH/VMRESUME (32 bits guest mode)
1306	;
1307	; @returns VBox status code
1308	; @param fResume x86:[ebp+8], msc:rcx,gcc:rdi Whether to use vmlauch/vmresume.
1309	; @param pCtx x86:[ebp+c], msc:rdx,gcc:rsi Pointer to the guest-CPU context.
1310	; @param pCache x86:[ebp+10],msc:r8, gcc:rdx Pointer to the VMCS cache.
1311	; @param pVM x86:[ebp+14],msc:r9, gcc:rcx The cross context VM structure.
1312	; @param pVCpu x86:[ebp+18],msc:[ebp+30],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1313	;
1314	ALIGNCODE(16)
1315	BEGINPROC VMXR0StartVM32
1316	push xBP
1317	mov xBP, xSP
1318
1319	pushf
1320	cli
1321
1322	;
1323	; Save all general purpose host registers.
1324	;
1325	MYPUSHAD
1326
1327	;
1328	; First we have to write some final guest CPU context registers.
1329	;
1330	mov eax, VMX_VMCS_HOST_RIP
1331	%ifdef RT_ARCH_AMD64
1332	lea r10, [.vmlaunch_done wrt rip]
1333	vmwrite rax, r10
1334	%else
1335	mov ecx, .vmlaunch_done
1336	vmwrite eax, ecx
1337	%endif
1338	; Note: assumes success!
1339
1340	;
1341	; Unify input parameter registers.
1342	;
1343	%ifdef RT_ARCH_AMD64
1344	%ifdef ASM_CALL64_GCC
1345	; fResume already in rdi
1346	; pCtx already in rsi
1347	mov rbx, rdx ; pCache
1348	%else
1349	mov rdi, rcx ; fResume
1350	mov rsi, rdx ; pCtx
1351	mov rbx, r8 ; pCache
1352	%endif
1353	%else
1354	mov edi, [ebp + 8] ; fResume
1355	mov esi, [ebp + 12] ; pCtx
1356	mov ebx, [ebp + 16] ; pCache
1357	%endif
1358
1359	;
1360	; Save the host XCR0 and load the guest one if necessary.
1361	; Note! Trashes rdx and rcx.
1362	;
1363	%ifdef ASM_CALL64_MSC
1364	mov rax, [xBP + 30h] ; pVCpu
1365	%elifdef ASM_CALL64_GCC
1366	mov rax, r8 ; pVCpu
1367	%else
1368	mov eax, [xBP + 18h] ; pVCpu
1369	%endif
1370	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1371	jz .xcr0_before_skip
1372
1373	xor ecx, ecx
1374	xgetbv ; Save the host one on the stack.
1375	push xDX
1376	push xAX
1377
1378	mov eax, [xSI + CPUMCTX.aXcr] ; Load the guest one.
1379	mov edx, [xSI + CPUMCTX.aXcr + 4]
1380	xor ecx, ecx ; paranoia
1381	xsetbv
1382
1383	push 0 ; Indicate that we must restore XCR0 (popped into ecx, thus 0).
1384	jmp .xcr0_before_done
1385
1386	.xcr0_before_skip:
1387	push 3fh ; indicate that we need not.
1388	.xcr0_before_done:
1389
1390	;
1391	; Save segment registers.
1392	; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
1393	;
1394	MYPUSHSEGS xAX, ax
1395
1396	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1397	mov ecx, [xBX + VMCSCACHE.Write.cValidEntries]
1398	cmp ecx, 0
1399	je .no_cached_writes
1400	mov edx, ecx
1401	mov ecx, 0
1402	jmp .cached_write
1403
1404	ALIGN(16)
1405	.cached_write:
1406	mov eax, [xBX + VMCSCACHE.Write.aField + xCX * 4]
1407	vmwrite xAX, [xBX + VMCSCACHE.Write.aFieldVal + xCX * 8]
1408	inc xCX
1409	cmp xCX, xDX
1410	jl .cached_write
1411
1412	mov dword [xBX + VMCSCACHE.Write.cValidEntries], 0
1413	.no_cached_writes:
1414
1415	; Save the pCache pointer.
1416	push xBX
1417	%endif
1418
1419	; Save the pCtx pointer.
1420	push xSI
1421
1422	; Save host LDTR.
1423	xor eax, eax
1424	sldt ax
1425	push xAX
1426
1427	%ifndef VMX_SKIP_TR
1428	; The host TR limit is reset to 0x67; save & restore it manually.
1429	str eax
1430	push xAX
1431	%endif
1432
1433	%ifndef VMX_SKIP_GDTR
1434	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
1435	sub xSP, xCB * 2
1436	sgdt [xSP]
1437	%endif
1438	%ifndef VMX_SKIP_IDTR
1439	sub xSP, xCB * 2
1440	sidt [xSP]
1441	%endif
1442
1443	; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
1444	mov xBX, [xSI + CPUMCTX.cr2]
1445	mov xDX, cr2
1446	cmp xBX, xDX
1447	je .skip_cr2_write32
1448	mov cr2, xBX
1449
1450	.skip_cr2_write32:
1451	mov eax, VMX_VMCS_HOST_RSP
1452	vmwrite xAX, xSP
1453	; Note: assumes success!
1454	; Don't mess with ESP anymore!!!
1455
1456	; Fight spectre.
1457	INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
1458
1459	; Load guest general purpose registers.
1460	mov eax, [xSI + CPUMCTX.eax]
1461	mov ebx, [xSI + CPUMCTX.ebx]
1462	mov ecx, [xSI + CPUMCTX.ecx]
1463	mov edx, [xSI + CPUMCTX.edx]
1464	mov ebp, [xSI + CPUMCTX.ebp]
1465
1466	; Resume or start VM?
1467	cmp xDI, 0 ; fResume
1468
1469	; Load guest edi & esi.
1470	mov edi, [xSI + CPUMCTX.edi]
1471	mov esi, [xSI + CPUMCTX.esi]
1472
1473	je .vmlaunch_launch
1474
1475	vmresume
1476	jc near .vmxstart_invalid_vmcs_ptr
1477	jz near .vmxstart_start_failed
1478	jmp .vmlaunch_done; ; Here if vmresume detected a failure.
1479
1480	.vmlaunch_launch:
1481	vmlaunch
1482	jc near .vmxstart_invalid_vmcs_ptr
1483	jz near .vmxstart_start_failed
1484	jmp .vmlaunch_done; ; Here if vmlaunch detected a failure.
1485
1486	ALIGNCODE(16) ;; @todo YASM BUG - this alignment is wrong on darwin, it's 1 byte off.
1487	.vmlaunch_done:
1488	RESTORE_STATE_VM32
1489	mov eax, VINF_SUCCESS
1490
1491	.vmstart_end:
1492	popf
1493	pop xBP
1494	ret
1495
1496	.vmxstart_invalid_vmcs_ptr:
1497	RESTORE_STATE_VM32
1498	mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1499	jmp .vmstart_end
1500
1501	.vmxstart_start_failed:
1502	RESTORE_STATE_VM32
1503	mov eax, VERR_VMX_UNABLE_TO_START_VM
1504	jmp .vmstart_end
1505
1506	ENDPROC VMXR0StartVM32
1507
1508
1509	%ifdef RT_ARCH_AMD64
1510	;; @def RESTORE_STATE_VM64
1511	; Macro restoring essential host state and updating guest state
1512	; for 64-bit host, 64-bit guest for VT-x.
1513	;
1514	%macro RESTORE_STATE_VM64 0
1515	; Restore base and limit of the IDTR & GDTR
1516	%ifndef VMX_SKIP_IDTR
1517	lidt [xSP]
1518	add xSP, xCB * 2
1519	%endif
1520	%ifndef VMX_SKIP_GDTR
1521	lgdt [xSP]
1522	add xSP, xCB * 2
1523	%endif
1524
1525	push xDI
1526	%ifndef VMX_SKIP_TR
1527	mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR)
1528	%else
1529	mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR)
1530	%endif
1531
1532	mov qword [xDI + CPUMCTX.eax], rax
1533	mov rax, SPECTRE_FILLER64
1534	mov qword [xDI + CPUMCTX.ebx], rbx
1535	mov rbx, rax
1536	mov qword [xDI + CPUMCTX.ecx], rcx
1537	mov rcx, rax
1538	mov qword [xDI + CPUMCTX.edx], rdx
1539	mov rdx, rax
1540	mov qword [xDI + CPUMCTX.esi], rsi
1541	mov rsi, rax
1542	mov qword [xDI + CPUMCTX.ebp], rbp
1543	mov rbp, rax
1544	mov qword [xDI + CPUMCTX.r8], r8
1545	mov r8, rax
1546	mov qword [xDI + CPUMCTX.r9], r9
1547	mov r9, rax
1548	mov qword [xDI + CPUMCTX.r10], r10
1549	mov r10, rax
1550	mov qword [xDI + CPUMCTX.r11], r11
1551	mov r11, rax
1552	mov qword [xDI + CPUMCTX.r12], r12
1553	mov r12, rax
1554	mov qword [xDI + CPUMCTX.r13], r13
1555	mov r13, rax
1556	mov qword [xDI + CPUMCTX.r14], r14
1557	mov r14, rax
1558	mov qword [xDI + CPUMCTX.r15], r15
1559	mov r15, rax
1560	mov rax, cr2
1561	mov qword [xDI + CPUMCTX.cr2], rax
1562
1563	pop xAX ; The guest rdi we pushed above
1564	mov qword [xDI + CPUMCTX.edi], rax
1565
1566	; Fight spectre.
1567	INDIRECT_BRANCH_PREDICTION_BARRIER xDI, CPUMCTX_WSF_IBPB_EXIT
1568
1569	%ifndef VMX_SKIP_TR
1570	; Restore TSS selector; must mark it as not busy before using ltr (!)
1571	; ASSUME that this is supposed to be 'BUSY'. (saves 20-30 ticks on the T42p).
1572	; @todo get rid of sgdt
1573	pop xBX ; Saved TR
1574	sub xSP, xCB * 2
1575	sgdt [xSP]
1576	mov xAX, xBX
1577	and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
1578	add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset.
1579	and dword [xAX + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
1580	ltr bx
1581	add xSP, xCB * 2
1582	%endif
1583
1584	pop xAX ; Saved LDTR
1585	cmp eax, 0
1586	je %%skip_ldt_write64
1587	lldt ax
1588
1589	%%skip_ldt_write64:
1590	pop xSI ; pCtx (needed in rsi by the macros below)
1591
1592	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1593	pop xDX ; Saved pCache
1594
1595	; Note! If we get here as a result of invalid VMCS pointer, all the following
1596	; vmread's will fail (only eflags.cf=1 will be set) but that shouldn't cause any
1597	; trouble only just less efficient.
1598	mov ecx, [xDX + VMCSCACHE.Read.cValidEntries]
1599	cmp ecx, 0 ; Can't happen
1600	je %%no_cached_read64
1601	jmp %%cached_read64
1602
1603	ALIGN(16)
1604	%%cached_read64:
1605	dec xCX
1606	mov eax, [xDX + VMCSCACHE.Read.aField + xCX * 4]
1607	vmread [xDX + VMCSCACHE.Read.aFieldVal + xCX * 8], xAX
1608	cmp xCX, 0
1609	jnz %%cached_read64
1610	%%no_cached_read64:
1611	%endif
1612
1613	; Restore segment registers.
1614	MYPOPSEGS xAX, ax
1615
1616	; Restore the host XCR0 if necessary.
1617	pop xCX
1618	test ecx, ecx
1619	jnz %%xcr0_after_skip
1620	pop xAX
1621	pop xDX
1622	xsetbv ; ecx is already zero.
1623	%%xcr0_after_skip:
1624
1625	; Restore general purpose registers.
1626	MYPOPAD
1627	%endmacro
1628
1629
1630	;;
1631	; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
1632	;
1633	; @returns VBox status code
1634	; @param fResume msc:rcx, gcc:rdi Whether to use vmlauch/vmresume.
1635	; @param pCtx msc:rdx, gcc:rsi Pointer to the guest-CPU context.
1636	; @param pCache msc:r8, gcc:rdx Pointer to the VMCS cache.
1637	; @param pVM msc:r9, gcc:rcx The cross context VM structure.
1638	; @param pVCpu msc:[ebp+30], gcc:r8 The cross context virtual CPU structure of the calling EMT.
1639	;
1640	ALIGNCODE(16)
1641	BEGINPROC VMXR0StartVM64
1642	push xBP
1643	mov xBP, xSP
1644
1645	pushf
1646	cli
1647
1648	; Save all general purpose host registers.
1649	MYPUSHAD
1650
1651	; First we have to save some final CPU context registers.
1652	lea r10, [.vmlaunch64_done wrt rip]
1653	mov rax, VMX_VMCS_HOST_RIP ; Return address (too difficult to continue after VMLAUNCH?).
1654	vmwrite rax, r10
1655	; Note: assumes success!
1656
1657	;
1658	; Unify the input parameter registers.
1659	;
1660	%ifdef ASM_CALL64_GCC
1661	; fResume already in rdi
1662	; pCtx already in rsi
1663	mov rbx, rdx ; pCache
1664	%else
1665	mov rdi, rcx ; fResume
1666	mov rsi, rdx ; pCtx
1667	mov rbx, r8 ; pCache
1668	%endif
1669
1670	;
1671	; Save the host XCR0 and load the guest one if necessary.
1672	; Note! Trashes rdx and rcx.
1673	;
1674	%ifdef ASM_CALL64_MSC
1675	mov rax, [xBP + 30h] ; pVCpu
1676	%else
1677	mov rax, r8 ; pVCpu
1678	%endif
1679	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1680	jz .xcr0_before_skip
1681
1682	xor ecx, ecx
1683	xgetbv ; Save the host one on the stack.
1684	push xDX
1685	push xAX
1686
1687	mov eax, [xSI + CPUMCTX.aXcr] ; Load the guest one.
1688	mov edx, [xSI + CPUMCTX.aXcr + 4]
1689	xor ecx, ecx ; paranoia
1690	xsetbv
1691
1692	push 0 ; Indicate that we must restore XCR0 (popped into ecx, thus 0).
1693	jmp .xcr0_before_done
1694
1695	.xcr0_before_skip:
1696	push 3fh ; indicate that we need not.
1697	.xcr0_before_done:
1698
1699	;
1700	; Save segment registers.
1701	; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
1702	;
1703	MYPUSHSEGS xAX, ax
1704
1705	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1706	mov ecx, [xBX + VMCSCACHE.Write.cValidEntries]
1707	cmp ecx, 0
1708	je .no_cached_writes
1709	mov edx, ecx
1710	mov ecx, 0
1711	jmp .cached_write
1712
1713	ALIGN(16)
1714	.cached_write:
1715	mov eax, [xBX + VMCSCACHE.Write.aField + xCX * 4]
1716	vmwrite xAX, [xBX + VMCSCACHE.Write.aFieldVal + xCX * 8]
1717	inc xCX
1718	cmp xCX, xDX
1719	jl .cached_write
1720
1721	mov dword [xBX + VMCSCACHE.Write.cValidEntries], 0
1722	.no_cached_writes:
1723
1724	; Save the pCache pointer.
1725	push xBX
1726	%endif
1727
1728	; Save the pCtx pointer.
1729	push xSI
1730
1731	; Save host LDTR.
1732	xor eax, eax
1733	sldt ax
1734	push xAX
1735
1736	%ifndef VMX_SKIP_TR
1737	; The host TR limit is reset to 0x67; save & restore it manually.
1738	str eax
1739	push xAX
1740	%endif
1741
1742	%ifndef VMX_SKIP_GDTR
1743	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
1744	sub xSP, xCB * 2
1745	sgdt [xSP]
1746	%endif
1747	%ifndef VMX_SKIP_IDTR
1748	sub xSP, xCB * 2
1749	sidt [xSP]
1750	%endif
1751
1752	; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
1753	mov rbx, qword [xSI + CPUMCTX.cr2]
1754	mov rdx, cr2
1755	cmp rbx, rdx
1756	je .skip_cr2_write
1757	mov cr2, rbx
1758
1759	.skip_cr2_write:
1760	mov eax, VMX_VMCS_HOST_RSP
1761	vmwrite xAX, xSP
1762	; Note: assumes success!
1763	; Don't mess with ESP anymore!!!
1764
1765	; Fight spectre.
1766	INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
1767
1768	; Load guest general purpose registers.
1769	mov rax, qword [xSI + CPUMCTX.eax]
1770	mov rbx, qword [xSI + CPUMCTX.ebx]
1771	mov rcx, qword [xSI + CPUMCTX.ecx]
1772	mov rdx, qword [xSI + CPUMCTX.edx]
1773	mov rbp, qword [xSI + CPUMCTX.ebp]
1774	mov r8, qword [xSI + CPUMCTX.r8]
1775	mov r9, qword [xSI + CPUMCTX.r9]
1776	mov r10, qword [xSI + CPUMCTX.r10]
1777	mov r11, qword [xSI + CPUMCTX.r11]
1778	mov r12, qword [xSI + CPUMCTX.r12]
1779	mov r13, qword [xSI + CPUMCTX.r13]
1780	mov r14, qword [xSI + CPUMCTX.r14]
1781	mov r15, qword [xSI + CPUMCTX.r15]
1782
1783	; Resume or start VM?
1784	cmp xDI, 0 ; fResume
1785
1786	; Load guest rdi & rsi.
1787	mov rdi, qword [xSI + CPUMCTX.edi]
1788	mov rsi, qword [xSI + CPUMCTX.esi]
1789
1790	je .vmlaunch64_launch
1791
1792	vmresume
1793	jc near .vmxstart64_invalid_vmcs_ptr
1794	jz near .vmxstart64_start_failed
1795	jmp .vmlaunch64_done; ; Here if vmresume detected a failure.
1796
1797	.vmlaunch64_launch:
1798	vmlaunch
1799	jc near .vmxstart64_invalid_vmcs_ptr
1800	jz near .vmxstart64_start_failed
1801	jmp .vmlaunch64_done; ; Here if vmlaunch detected a failure.
1802
1803	ALIGNCODE(16)
1804	.vmlaunch64_done:
1805	RESTORE_STATE_VM64
1806	mov eax, VINF_SUCCESS
1807
1808	.vmstart64_end:
1809	popf
1810	pop xBP
1811	ret
1812
1813	.vmxstart64_invalid_vmcs_ptr:
1814	RESTORE_STATE_VM64
1815	mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1816	jmp .vmstart64_end
1817
1818	.vmxstart64_start_failed:
1819	RESTORE_STATE_VM64
1820	mov eax, VERR_VMX_UNABLE_TO_START_VM
1821	jmp .vmstart64_end
1822	ENDPROC VMXR0StartVM64
1823	%endif ; RT_ARCH_AMD64
1824
1825
1826	;;
1827	; Prepares for and executes VMRUN (32 bits guests)
1828	;
1829	; @returns VBox status code
1830	; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB.
1831	; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB.
1832	; @param pCtx msc:r8,gcc:rdx Pointer to the guest CPU-context.
1833	; @param pVM msc:r9,gcc:rcx The cross context VM structure.
1834	; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1835	;
1836	ALIGNCODE(16)
1837	BEGINPROC SVMR0VMRun
1838	%ifdef RT_ARCH_AMD64 ; fake a cdecl stack frame
1839	%ifdef ASM_CALL64_GCC
1840	push r8 ; pVCpu
1841	push rcx ; pVM
1842	push rdx ; pCtx
1843	push rsi ; HCPhysVmcb
1844	push rdi ; HCPhysVmcbHost
1845	%else
1846	mov rax, [rsp + 28h]
1847	push rax ; pVCpu
1848	push r9 ; pVM
1849	push r8 ; pCtx
1850	push rdx ; HCPhysVmcb
1851	push rcx ; HCPhysVmcbHost
1852	%endif
1853	push 0
1854	%endif
1855	push xBP
1856	mov xBP, xSP
1857	pushf
1858
1859	; Save all general purpose host registers.
1860	MYPUSHAD
1861
1862	; Load pCtx into xSI.
1863	mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1864
1865	; Save the host XCR0 and load the guest one if necessary.
1866	mov xAX, [xBP + xCB * 2 + RTHCPHYS_CB * 2 + xCB * 2] ; pVCpu
1867	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1868	jz .xcr0_before_skip
1869
1870	xor ecx, ecx
1871	xgetbv ; Save the host XCR0 on the stack
1872	push xDX
1873	push xAX
1874
1875	mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1876	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0
1877	mov edx, [xSI + CPUMCTX.aXcr + 4]
1878	xor ecx, ecx ; paranoia
1879	xsetbv
1880
1881	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
1882	jmp .xcr0_before_done
1883
1884	.xcr0_before_skip:
1885	push 3fh ; indicate that we need not restore XCR0
1886	.xcr0_before_done:
1887
1888	; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
1889	push xSI
1890
1891	; Save host fs, gs, sysenter msr etc.
1892	mov xAX, [xBP + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
1893	push xAX ; save for the vmload after vmrun
1894	vmsave
1895
1896	; Fight spectre.
1897	INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
1898
1899	; Setup xAX for VMLOAD.
1900	mov xAX, [xBP + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; x86: take low dword only)
1901
1902	; Load guest general purpose registers.
1903	; eax is loaded from the VMCB by VMRUN.
1904	mov ebx, [xSI + CPUMCTX.ebx]
1905	mov ecx, [xSI + CPUMCTX.ecx]
1906	mov edx, [xSI + CPUMCTX.edx]
1907	mov edi, [xSI + CPUMCTX.edi]
1908	mov ebp, [xSI + CPUMCTX.ebp]
1909	mov esi, [xSI + CPUMCTX.esi]
1910
1911	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1912	clgi
1913	sti
1914
1915	; Load guest fs, gs, sysenter msr etc.
1916	vmload
1917
1918	; Run the VM.
1919	vmrun
1920
1921	; Save guest fs, gs, sysenter msr etc.
1922	vmsave
1923
1924	; Load host fs, gs, sysenter msr etc.
1925	pop xAX ; load HCPhysVmcbHost (pushed above)
1926	vmload
1927
1928	; Set the global interrupt flag again, but execute cli to make sure IF=0.
1929	cli
1930	stgi
1931
1932	; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
1933	pop xAX
1934
1935	mov [ss:xAX + CPUMCTX.ebx], ebx
1936	mov xBX, SPECTRE_FILLER
1937	mov [ss:xAX + CPUMCTX.ecx], ecx
1938	mov xCX, xBX
1939	mov [ss:xAX + CPUMCTX.edx], edx
1940	mov xDX, xBX
1941	mov [ss:xAX + CPUMCTX.esi], esi
1942	mov xSI, xBX
1943	mov [ss:xAX + CPUMCTX.edi], edi
1944	mov xDI, xBX
1945	mov [ss:xAX + CPUMCTX.ebp], ebp
1946	mov xBP, xBX
1947
1948	; Fight spectre. Note! Trashes xAX!
1949	INDIRECT_BRANCH_PREDICTION_BARRIER ss:xAX, CPUMCTX_WSF_IBPB_EXIT
1950
1951	; Restore the host xcr0 if necessary.
1952	pop xCX
1953	test ecx, ecx
1954	jnz .xcr0_after_skip
1955	pop xAX
1956	pop xDX
1957	xsetbv ; ecx is already zero
1958	.xcr0_after_skip:
1959
1960	; Restore host general purpose registers.
1961	MYPOPAD
1962
1963	mov eax, VINF_SUCCESS
1964
1965	popf
1966	pop xBP
1967	%ifdef RT_ARCH_AMD64
1968	add xSP, 6*xCB
1969	%endif
1970	ret
1971	ENDPROC SVMR0VMRun
1972
1973
1974	%ifdef RT_ARCH_AMD64
1975	;;
1976	; Prepares for and executes VMRUN (64 bits guests)
1977	;
1978	; @returns VBox status code
1979	; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB.
1980	; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB.
1981	; @param pCtx msc:r8,gcc:rdx Pointer to the guest-CPU context.
1982	; @param pVM msc:r9,gcc:rcx The cross context VM structure.
1983	; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1984	;
1985	ALIGNCODE(16)
1986	BEGINPROC SVMR0VMRun64
1987	; Fake a cdecl stack frame
1988	%ifdef ASM_CALL64_GCC
1989	push r8 ;pVCpu
1990	push rcx ;pVM
1991	push rdx ;pCtx
1992	push rsi ;HCPhysVmcb
1993	push rdi ;HCPhysVmcbHost
1994	%else
1995	mov rax, [rsp + 28h]
1996	push rax ; rbp + 30h pVCpu
1997	push r9 ; rbp + 28h pVM
1998	push r8 ; rbp + 20h pCtx
1999	push rdx ; rbp + 18h HCPhysVmcb
2000	push rcx ; rbp + 10h HCPhysVmcbHost
2001	%endif
2002	push 0 ; rbp + 08h "fake ret addr"
2003	push rbp ; rbp + 00h
2004	mov rbp, rsp
2005	pushf
2006
2007	; Manual save and restore:
2008	; - General purpose registers except RIP, RSP, RAX
2009	;
2010	; Trashed:
2011	; - CR2 (we don't care)
2012	; - LDTR (reset to 0)
2013	; - DRx (presumably not changed at all)
2014	; - DR7 (reset to 0x400)
2015
2016	; Save all general purpose host registers.
2017	MYPUSHAD
2018
2019	; Load pCtx into xSI.
2020	mov xSI, [rbp + xCB * 2 + RTHCPHYS_CB * 2]
2021
2022	; Save the host XCR0 and load the guest one if necessary.
2023	mov rax, [xBP + 30h] ; pVCpu
2024	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
2025	jz .xcr0_before_skip
2026
2027	xor ecx, ecx
2028	xgetbv ; save the host XCR0 on the stack.
2029	push xDX
2030	push xAX
2031
2032	mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
2033	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0
2034	mov edx, [xSI + CPUMCTX.aXcr + 4]
2035	xor ecx, ecx ; paranoia
2036	xsetbv
2037
2038	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
2039	jmp .xcr0_before_done
2040
2041	.xcr0_before_skip:
2042	push 3fh ; indicate that we need not restore XCR0
2043	.xcr0_before_done:
2044
2045	; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
2046	push rsi
2047
2048	; Save host fs, gs, sysenter msr etc.
2049	mov rax, [rbp + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
2050	push rax ; save for the vmload after vmrun
2051	vmsave
2052
2053	; Fight spectre.
2054	INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
2055
2056	; Setup rax for VMLOAD.
2057	mov rax, [rbp + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; take low dword only)
2058
2059	; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
2060	mov rbx, qword [xSI + CPUMCTX.ebx]
2061	mov rcx, qword [xSI + CPUMCTX.ecx]
2062	mov rdx, qword [xSI + CPUMCTX.edx]
2063	mov rdi, qword [xSI + CPUMCTX.edi]
2064	mov rbp, qword [xSI + CPUMCTX.ebp]
2065	mov r8, qword [xSI + CPUMCTX.r8]
2066	mov r9, qword [xSI + CPUMCTX.r9]
2067	mov r10, qword [xSI + CPUMCTX.r10]
2068	mov r11, qword [xSI + CPUMCTX.r11]
2069	mov r12, qword [xSI + CPUMCTX.r12]
2070	mov r13, qword [xSI + CPUMCTX.r13]
2071	mov r14, qword [xSI + CPUMCTX.r14]
2072	mov r15, qword [xSI + CPUMCTX.r15]
2073	mov rsi, qword [xSI + CPUMCTX.esi]
2074
2075	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
2076	clgi
2077	sti
2078
2079	; Load guest FS, GS, Sysenter MSRs etc.
2080	vmload
2081
2082	; Run the VM.
2083	vmrun
2084
2085	; Save guest fs, gs, sysenter msr etc.
2086	vmsave
2087
2088	; Load host fs, gs, sysenter msr etc.
2089	pop rax ; load HCPhysVmcbHost (pushed above)
2090	vmload
2091
2092	; Set the global interrupt flag again, but execute cli to make sure IF=0.
2093	cli
2094	stgi
2095
2096	; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
2097	pop rax
2098
2099	mov qword [rax + CPUMCTX.ebx], rbx
2100	mov rbx, SPECTRE_FILLER64
2101	mov qword [rax + CPUMCTX.ecx], rcx
2102	mov rcx, rbx
2103	mov qword [rax + CPUMCTX.edx], rdx
2104	mov rdx, rbx
2105	mov qword [rax + CPUMCTX.esi], rsi
2106	mov rsi, rbx
2107	mov qword [rax + CPUMCTX.edi], rdi
2108	mov rdi, rbx
2109	mov qword [rax + CPUMCTX.ebp], rbp
2110	mov rbp, rbx
2111	mov qword [rax + CPUMCTX.r8], r8
2112	mov r8, rbx
2113	mov qword [rax + CPUMCTX.r9], r9
2114	mov r9, rbx
2115	mov qword [rax + CPUMCTX.r10], r10
2116	mov r10, rbx
2117	mov qword [rax + CPUMCTX.r11], r11
2118	mov r11, rbx
2119	mov qword [rax + CPUMCTX.r12], r12
2120	mov r12, rbx
2121	mov qword [rax + CPUMCTX.r13], r13
2122	mov r13, rbx
2123	mov qword [rax + CPUMCTX.r14], r14
2124	mov r14, rbx
2125	mov qword [rax + CPUMCTX.r15], r15
2126	mov r15, rbx
2127
2128	; Fight spectre. Note! Trashes rax!
2129	INDIRECT_BRANCH_PREDICTION_BARRIER rax, CPUMCTX_WSF_IBPB_EXIT
2130
2131	; Restore the host xcr0 if necessary.
2132	pop xCX
2133	test ecx, ecx
2134	jnz .xcr0_after_skip
2135	pop xAX
2136	pop xDX
2137	xsetbv ; ecx is already zero
2138	.xcr0_after_skip:
2139
2140	; Restore host general purpose registers.
2141	MYPOPAD
2142
2143	mov eax, VINF_SUCCESS
2144
2145	popf
2146	pop rbp
2147	add rsp, 6 * xCB
2148	ret
2149	ENDPROC SVMR0VMRun64
2150	%endif ; RT_ARCH_AMD64
2151

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 72893

Download in other formats: