HMR0A.asm@ 70262

Last change on this file since 70262 was 69221, checked in by vboxsync, 7 years ago
VMM: scm cleanups
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 61.0 KB

Line
1	; $Id: HMR0A.asm 69221 2017-10-24 15:07:46Z vboxsync $
2	;; @file
3	; HM - Ring-0 VMX, SVM world-switch and helper routines
4	;
5
6	;
7	; Copyright (C) 2006-2017 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.virtualbox.org. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17
18	;*********************************************************************************************************************************
19	;* Header Files *
20	;*********************************************************************************************************************************
21	%include "VBox/asmdefs.mac"
22	%include "VBox/err.mac"
23	%include "VBox/vmm/hm_vmx.mac"
24	%include "VBox/vmm/cpum.mac"
25	%include "VBox/vmm/vm.mac"
26	%include "iprt/x86.mac"
27	%include "HMInternal.mac"
28
29	%ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely.
30	%macro vmwrite 2,
31	int3
32	%endmacro
33	%define vmlaunch int3
34	%define vmresume int3
35	%define vmsave int3
36	%define vmload int3
37	%define vmrun int3
38	%define clgi int3
39	%define stgi int3
40	%macro invlpga 2,
41	int3
42	%endmacro
43	%endif
44
45	;*********************************************************************************************************************************
46	;* Defined Constants And Macros *
47	;*********************************************************************************************************************************
48	;; The offset of the XMM registers in X86FXSTATE.
49	; Use define because I'm too lazy to convert the struct.
50	%define XMM_OFF_IN_X86FXSTATE 160
51
52	;;
53	; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation
54	;
55	%ifdef RT_ARCH_AMD64
56	%define VMX_SKIP_GDTR
57	%define VMX_SKIP_TR
58	%define VBOX_SKIP_RESTORE_SEG
59	%ifdef RT_OS_DARWIN
60	; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
61	; risk loading a stale LDT value or something invalid.
62	%define HM_64_BIT_USE_NULL_SEL
63	; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always.
64	; See @bugref{6875}.
65	%else
66	%define VMX_SKIP_IDTR
67	%endif
68	%endif
69
70	;; @def MYPUSHAD
71	; Macro generating an equivalent to pushad
72
73	;; @def MYPOPAD
74	; Macro generating an equivalent to popad
75
76	;; @def MYPUSHSEGS
77	; Macro saving all segment registers on the stack.
78	; @param 1 full width register name
79	; @param 2 16-bit register name for \a 1.
80
81	;; @def MYPOPSEGS
82	; Macro restoring all segment registers on the stack
83	; @param 1 full width register name
84	; @param 2 16-bit register name for \a 1.
85
86	%ifdef ASM_CALL64_GCC
87	%macro MYPUSHAD64 0
88	push r15
89	push r14
90	push r13
91	push r12
92	push rbx
93	%endmacro
94	%macro MYPOPAD64 0
95	pop rbx
96	pop r12
97	pop r13
98	pop r14
99	pop r15
100	%endmacro
101
102	%else ; ASM_CALL64_MSC
103	%macro MYPUSHAD64 0
104	push r15
105	push r14
106	push r13
107	push r12
108	push rbx
109	push rsi
110	push rdi
111	%endmacro
112	%macro MYPOPAD64 0
113	pop rdi
114	pop rsi
115	pop rbx
116	pop r12
117	pop r13
118	pop r14
119	pop r15
120	%endmacro
121	%endif
122
123	%ifdef VBOX_SKIP_RESTORE_SEG
124	%macro MYPUSHSEGS64 2
125	%endmacro
126
127	%macro MYPOPSEGS64 2
128	%endmacro
129	%else ; !VBOX_SKIP_RESTORE_SEG
130	; trashes, rax, rdx & rcx
131	%macro MYPUSHSEGS64 2
132	%ifndef HM_64_BIT_USE_NULL_SEL
133	mov %2, es
134	push %1
135	mov %2, ds
136	push %1
137	%endif
138
139	; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode, Solaris OTOH doesn't and we must save it.
140	mov ecx, MSR_K8_FS_BASE
141	rdmsr
142	push rdx
143	push rax
144	%ifndef HM_64_BIT_USE_NULL_SEL
145	push fs
146	%endif
147
148	; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel. The same happens on exit
149	mov ecx, MSR_K8_GS_BASE
150	rdmsr
151	push rdx
152	push rax
153	%ifndef HM_64_BIT_USE_NULL_SEL
154	push gs
155	%endif
156	%endmacro
157
158	; trashes, rax, rdx & rcx
159	%macro MYPOPSEGS64 2
160	; Note: do not step through this code with a debugger!
161	%ifndef HM_64_BIT_USE_NULL_SEL
162	xor eax, eax
163	mov ds, ax
164	mov es, ax
165	mov fs, ax
166	mov gs, ax
167	%endif
168
169	%ifndef HM_64_BIT_USE_NULL_SEL
170	pop gs
171	%endif
172	pop rax
173	pop rdx
174	mov ecx, MSR_K8_GS_BASE
175	wrmsr
176
177	%ifndef HM_64_BIT_USE_NULL_SEL
178	pop fs
179	%endif
180	pop rax
181	pop rdx
182	mov ecx, MSR_K8_FS_BASE
183	wrmsr
184	; Now it's safe to step again
185
186	%ifndef HM_64_BIT_USE_NULL_SEL
187	pop %1
188	mov ds, %2
189	pop %1
190	mov es, %2
191	%endif
192	%endmacro
193	%endif ; VBOX_SKIP_RESTORE_SEG
194
195	%macro MYPUSHAD32 0
196	pushad
197	%endmacro
198	%macro MYPOPAD32 0
199	popad
200	%endmacro
201
202	%macro MYPUSHSEGS32 2
203	push ds
204	push es
205	push fs
206	push gs
207	%endmacro
208	%macro MYPOPSEGS32 2
209	pop gs
210	pop fs
211	pop es
212	pop ds
213	%endmacro
214
215	%ifdef RT_ARCH_AMD64
216	%define MYPUSHAD MYPUSHAD64
217	%define MYPOPAD MYPOPAD64
218	%define MYPUSHSEGS MYPUSHSEGS64
219	%define MYPOPSEGS MYPOPSEGS64
220	%else
221	%define MYPUSHAD MYPUSHAD32
222	%define MYPOPAD MYPOPAD32
223	%define MYPUSHSEGS MYPUSHSEGS32
224	%define MYPOPSEGS MYPOPSEGS32
225	%endif
226
227
228	;*********************************************************************************************************************************
229	;* External Symbols *
230	;*********************************************************************************************************************************
231	%ifdef VBOX_WITH_KERNEL_USING_XMM
232	extern NAME(CPUMIsGuestFPUStateActive)
233	%endif
234
235
236	BEGINCODE
237
238
239	;/**
240	; * Restores host-state fields.
241	; *
242	; * @returns VBox status code
243	; * @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags.
244	; * @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct.
245	; */
246	ALIGNCODE(16)
247	BEGINPROC VMXRestoreHostState
248	%ifdef RT_ARCH_AMD64
249	%ifndef ASM_CALL64_GCC
250	; Use GCC's input registers since we'll be needing both rcx and rdx further
251	; down with the wrmsr instruction. Use the R10 and R11 register for saving
252	; RDI and RSI since MSC preserve the two latter registers.
253	mov r10, rdi
254	mov r11, rsi
255	mov rdi, rcx
256	mov rsi, rdx
257	%endif
258
259	test edi, VMX_RESTORE_HOST_GDTR
260	jz .test_idtr
261	lgdt [rsi + VMXRESTOREHOST.HostGdtr]
262
263	.test_idtr:
264	test edi, VMX_RESTORE_HOST_IDTR
265	jz .test_ds
266	lidt [rsi + VMXRESTOREHOST.HostIdtr]
267
268	.test_ds:
269	test edi, VMX_RESTORE_HOST_SEL_DS
270	jz .test_es
271	mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
272	mov ds, eax
273
274	.test_es:
275	test edi, VMX_RESTORE_HOST_SEL_ES
276	jz .test_tr
277	mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
278	mov es, eax
279
280	.test_tr:
281	test edi, VMX_RESTORE_HOST_SEL_TR
282	jz .test_fs
283	; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
284	mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
285	mov ax, dx
286	and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
287	test edi, VMX_RESTORE_HOST_GDT_READ_ONLY \| VMX_RESTORE_HOST_GDT_NEED_WRITABLE
288	jnz .gdt_readonly
289	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
290	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
291	ltr dx
292	jmp short .test_fs
293	.gdt_readonly:
294	test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
295	jnz .gdt_readonly_need_writable
296	mov rcx, cr0
297	mov r9, rcx
298	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
299	and rcx, ~X86_CR0_WP
300	mov cr0, rcx
301	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
302	ltr dx
303	mov cr0, r9
304	jmp short .test_fs
305	.gdt_readonly_need_writable:
306	add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw.
307	and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
308	lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
309	ltr dx
310	lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; Load the original GDT
311
312	.test_fs:
313	;
314	; When restoring the selector values for FS and GS, we'll temporarily trash
315	; the base address (at least the high 32-bit bits, but quite possibly the
316	; whole base address), the wrmsr will restore it correctly. (VT-x actually
317	; restores the base correctly when leaving guest mode, but not the selector
318	; value, so there is little problem with interrupts being enabled prior to
319	; this restore job.)
320	; We'll disable ints once for both FS and GS as that's probably faster.
321	;
322	test edi, VMX_RESTORE_HOST_SEL_FS \| VMX_RESTORE_HOST_SEL_GS
323	jz .restore_success
324	pushfq
325	cli ; (see above)
326
327	test edi, VMX_RESTORE_HOST_SEL_FS
328	jz .test_gs
329	mov ax, word [rsi + VMXRESTOREHOST.uHostSelFS]
330	mov fs, eax
331	mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
332	mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
333	mov ecx, MSR_K8_FS_BASE
334	wrmsr
335
336	.test_gs:
337	test edi, VMX_RESTORE_HOST_SEL_GS
338	jz .restore_flags
339	mov ax, word [rsi + VMXRESTOREHOST.uHostSelGS]
340	mov gs, eax
341	mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
342	mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
343	mov ecx, MSR_K8_GS_BASE
344	wrmsr
345
346	.restore_flags:
347	popfq
348
349	.restore_success:
350	mov eax, VINF_SUCCESS
351	%ifndef ASM_CALL64_GCC
352	; Restore RDI and RSI on MSC.
353	mov rdi, r10
354	mov rsi, r11
355	%endif
356	%else ; RT_ARCH_X86
357	mov eax, VERR_NOT_IMPLEMENTED
358	%endif
359	ret
360	ENDPROC VMXRestoreHostState
361
362
363	;/**
364	; * Dispatches an NMI to the host.
365	; */
366	ALIGNCODE(16)
367	BEGINPROC VMXDispatchHostNmi
368	int 2 ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
369	ret
370	ENDPROC VMXDispatchHostNmi
371
372
373	;/**
374	; * Executes VMWRITE, 64-bit value.
375	; *
376	; * @returns VBox status code.
377	; * @param idxField x86: [ebp + 08h] msc: rcx gcc: rdi VMCS index.
378	; * @param u64Data x86: [ebp + 0ch] msc: rdx gcc: rsi VM field value.
379	; */
380	ALIGNCODE(16)
381	BEGINPROC VMXWriteVmcs64
382	%ifdef RT_ARCH_AMD64
383	%ifdef ASM_CALL64_GCC
384	and edi, 0ffffffffh
385	xor rax, rax
386	vmwrite rdi, rsi
387	%else
388	and ecx, 0ffffffffh
389	xor rax, rax
390	vmwrite rcx, rdx
391	%endif
392	%else ; RT_ARCH_X86
393	mov ecx, [esp + 4] ; idxField
394	lea edx, [esp + 8] ; &u64Data
395	vmwrite ecx, [edx] ; low dword
396	jz .done
397	jc .done
398	inc ecx
399	xor eax, eax
400	vmwrite ecx, [edx + 4] ; high dword
401	.done:
402	%endif ; RT_ARCH_X86
403	jnc .valid_vmcs
404	mov eax, VERR_VMX_INVALID_VMCS_PTR
405	ret
406	.valid_vmcs:
407	jnz .the_end
408	mov eax, VERR_VMX_INVALID_VMCS_FIELD
409	.the_end:
410	ret
411	ENDPROC VMXWriteVmcs64
412
413
414	;/**
415	; * Executes VMREAD, 64-bit value.
416	; *
417	; * @returns VBox status code.
418	; * @param idxField VMCS index.
419	; * @param pData Where to store VM field value.
420	; */
421	;DECLASM(int) VMXReadVmcs64(uint32_t idxField, uint64_t *pData);
422	ALIGNCODE(16)
423	BEGINPROC VMXReadVmcs64
424	%ifdef RT_ARCH_AMD64
425	%ifdef ASM_CALL64_GCC
426	and edi, 0ffffffffh
427	xor rax, rax
428	vmread [rsi], rdi
429	%else
430	and ecx, 0ffffffffh
431	xor rax, rax
432	vmread [rdx], rcx
433	%endif
434	%else ; RT_ARCH_X86
435	mov ecx, [esp + 4] ; idxField
436	mov edx, [esp + 8] ; pData
437	vmread [edx], ecx ; low dword
438	jz .done
439	jc .done
440	inc ecx
441	xor eax, eax
442	vmread [edx + 4], ecx ; high dword
443	.done:
444	%endif ; RT_ARCH_X86
445	jnc .valid_vmcs
446	mov eax, VERR_VMX_INVALID_VMCS_PTR
447	ret
448	.valid_vmcs:
449	jnz .the_end
450	mov eax, VERR_VMX_INVALID_VMCS_FIELD
451	.the_end:
452	ret
453	ENDPROC VMXReadVmcs64
454
455
456	;/**
457	; * Executes VMREAD, 32-bit value.
458	; *
459	; * @returns VBox status code.
460	; * @param idxField VMCS index.
461	; * @param pu32Data Where to store VM field value.
462	; */
463	;DECLASM(int) VMXReadVmcs32(uint32_t idxField, uint32_t *pu32Data);
464	ALIGNCODE(16)
465	BEGINPROC VMXReadVmcs32
466	%ifdef RT_ARCH_AMD64
467	%ifdef ASM_CALL64_GCC
468	and edi, 0ffffffffh
469	xor rax, rax
470	vmread r10, rdi
471	mov [rsi], r10d
472	%else
473	and ecx, 0ffffffffh
474	xor rax, rax
475	vmread r10, rcx
476	mov [rdx], r10d
477	%endif
478	%else ; RT_ARCH_X86
479	mov ecx, [esp + 4] ; idxField
480	mov edx, [esp + 8] ; pu32Data
481	xor eax, eax
482	vmread [edx], ecx
483	%endif ; RT_ARCH_X86
484	jnc .valid_vmcs
485	mov eax, VERR_VMX_INVALID_VMCS_PTR
486	ret
487	.valid_vmcs:
488	jnz .the_end
489	mov eax, VERR_VMX_INVALID_VMCS_FIELD
490	.the_end:
491	ret
492	ENDPROC VMXReadVmcs32
493
494
495	;/**
496	; * Executes VMWRITE, 32-bit value.
497	; *
498	; * @returns VBox status code.
499	; * @param idxField VMCS index.
500	; * @param u32Data Where to store VM field value.
501	; */
502	;DECLASM(int) VMXWriteVmcs32(uint32_t idxField, uint32_t u32Data);
503	ALIGNCODE(16)
504	BEGINPROC VMXWriteVmcs32
505	%ifdef RT_ARCH_AMD64
506	%ifdef ASM_CALL64_GCC
507	and edi, 0ffffffffh
508	and esi, 0ffffffffh
509	xor rax, rax
510	vmwrite rdi, rsi
511	%else
512	and ecx, 0ffffffffh
513	and edx, 0ffffffffh
514	xor rax, rax
515	vmwrite rcx, rdx
516	%endif
517	%else ; RT_ARCH_X86
518	mov ecx, [esp + 4] ; idxField
519	mov edx, [esp + 8] ; u32Data
520	xor eax, eax
521	vmwrite ecx, edx
522	%endif ; RT_ARCH_X86
523	jnc .valid_vmcs
524	mov eax, VERR_VMX_INVALID_VMCS_PTR
525	ret
526	.valid_vmcs:
527	jnz .the_end
528	mov eax, VERR_VMX_INVALID_VMCS_FIELD
529	.the_end:
530	ret
531	ENDPROC VMXWriteVmcs32
532
533
534	;/**
535	; * Executes VMXON.
536	; *
537	; * @returns VBox status code.
538	; * @param HCPhysVMXOn Physical address of VMXON structure.
539	; */
540	;DECLASM(int) VMXEnable(RTHCPHYS HCPhysVMXOn);
541	BEGINPROC VMXEnable
542	%ifdef RT_ARCH_AMD64
543	xor rax, rax
544	%ifdef ASM_CALL64_GCC
545	push rdi
546	%else
547	push rcx
548	%endif
549	vmxon [rsp]
550	%else ; RT_ARCH_X86
551	xor eax, eax
552	vmxon [esp + 4]
553	%endif ; RT_ARCH_X86
554	jnc .good
555	mov eax, VERR_VMX_INVALID_VMXON_PTR
556	jmp .the_end
557
558	.good:
559	jnz .the_end
560	mov eax, VERR_VMX_VMXON_FAILED
561
562	.the_end:
563	%ifdef RT_ARCH_AMD64
564	add rsp, 8
565	%endif
566	ret
567	ENDPROC VMXEnable
568
569
570	;/**
571	; * Executes VMXOFF.
572	; */
573	;DECLASM(void) VMXDisable(void);
574	BEGINPROC VMXDisable
575	vmxoff
576	.the_end:
577	ret
578	ENDPROC VMXDisable
579
580
581	;/**
582	; * Executes VMCLEAR.
583	; *
584	; * @returns VBox status code.
585	; * @param HCPhysVmcs Physical address of VM control structure.
586	; */
587	;DECLASM(int) VMXClearVmcs(RTHCPHYS HCPhysVmcs);
588	ALIGNCODE(16)
589	BEGINPROC VMXClearVmcs
590	%ifdef RT_ARCH_AMD64
591	xor rax, rax
592	%ifdef ASM_CALL64_GCC
593	push rdi
594	%else
595	push rcx
596	%endif
597	vmclear [rsp]
598	%else ; RT_ARCH_X86
599	xor eax, eax
600	vmclear [esp + 4]
601	%endif ; RT_ARCH_X86
602	jnc .the_end
603	mov eax, VERR_VMX_INVALID_VMCS_PTR
604	.the_end:
605	%ifdef RT_ARCH_AMD64
606	add rsp, 8
607	%endif
608	ret
609	ENDPROC VMXClearVmcs
610
611
612	;/**
613	; * Executes VMPTRLD.
614	; *
615	; * @returns VBox status code.
616	; * @param HCPhysVmcs Physical address of VMCS structure.
617	; */
618	;DECLASM(int) VMXActivateVmcs(RTHCPHYS HCPhysVmcs);
619	ALIGNCODE(16)
620	BEGINPROC VMXActivateVmcs
621	%ifdef RT_ARCH_AMD64
622	xor rax, rax
623	%ifdef ASM_CALL64_GCC
624	push rdi
625	%else
626	push rcx
627	%endif
628	vmptrld [rsp]
629	%else
630	xor eax, eax
631	vmptrld [esp + 4]
632	%endif
633	jnc .the_end
634	mov eax, VERR_VMX_INVALID_VMCS_PTR
635	.the_end:
636	%ifdef RT_ARCH_AMD64
637	add rsp, 8
638	%endif
639	ret
640	ENDPROC VMXActivateVmcs
641
642
643	;/**
644	; * Executes VMPTRST.
645	; *
646	; * @returns VBox status code.
647	; * @param [esp + 04h] gcc:rdi msc:rcx Param 1 - First parameter - Address that will receive the current pointer.
648	; */
649	;DECLASM(int) VMXGetActivatedVmcs(RTHCPHYS *pVMCS);
650	BEGINPROC VMXGetActivatedVmcs
651	%ifdef RT_OS_OS2
652	mov eax, VERR_NOT_SUPPORTED
653	ret
654	%else
655	%ifdef RT_ARCH_AMD64
656	%ifdef ASM_CALL64_GCC
657	vmptrst qword [rdi]
658	%else
659	vmptrst qword [rcx]
660	%endif
661	%else
662	vmptrst qword [esp+04h]
663	%endif
664	xor eax, eax
665	.the_end:
666	ret
667	%endif
668	ENDPROC VMXGetActivatedVmcs
669
670	;/**
671	; * Invalidate a page using INVEPT.
672	; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush.
673	; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer.
674	; */
675	;DECLASM(int) VMXR0InvEPT(VMX_FLUSH enmFlush, uint64_t *pDescriptor);
676	BEGINPROC VMXR0InvEPT
677	%ifdef RT_ARCH_AMD64
678	%ifdef ASM_CALL64_GCC
679	and edi, 0ffffffffh
680	xor rax, rax
681	; invept rdi, qword [rsi]
682	DB 0x66, 0x0F, 0x38, 0x80, 0x3E
683	%else
684	and ecx, 0ffffffffh
685	xor rax, rax
686	; invept rcx, qword [rdx]
687	DB 0x66, 0x0F, 0x38, 0x80, 0xA
688	%endif
689	%else
690	mov ecx, [esp + 4]
691	mov edx, [esp + 8]
692	xor eax, eax
693	; invept ecx, qword [edx]
694	DB 0x66, 0x0F, 0x38, 0x80, 0xA
695	%endif
696	jnc .valid_vmcs
697	mov eax, VERR_VMX_INVALID_VMCS_PTR
698	ret
699	.valid_vmcs:
700	jnz .the_end
701	mov eax, VERR_INVALID_PARAMETER
702	.the_end:
703	ret
704	ENDPROC VMXR0InvEPT
705
706
707	;/**
708	; * Invalidate a page using invvpid
709	; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush
710	; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer
711	; */
712	;DECLASM(int) VMXR0InvVPID(VMX_FLUSH enmFlush, uint64_t *pDescriptor);
713	BEGINPROC VMXR0InvVPID
714	%ifdef RT_ARCH_AMD64
715	%ifdef ASM_CALL64_GCC
716	and edi, 0ffffffffh
717	xor rax, rax
718	; invvpid rdi, qword [rsi]
719	DB 0x66, 0x0F, 0x38, 0x81, 0x3E
720	%else
721	and ecx, 0ffffffffh
722	xor rax, rax
723	; invvpid rcx, qword [rdx]
724	DB 0x66, 0x0F, 0x38, 0x81, 0xA
725	%endif
726	%else
727	mov ecx, [esp + 4]
728	mov edx, [esp + 8]
729	xor eax, eax
730	; invvpid ecx, qword [edx]
731	DB 0x66, 0x0F, 0x38, 0x81, 0xA
732	%endif
733	jnc .valid_vmcs
734	mov eax, VERR_VMX_INVALID_VMCS_PTR
735	ret
736	.valid_vmcs:
737	jnz .the_end
738	mov eax, VERR_INVALID_PARAMETER
739	.the_end:
740	ret
741	ENDPROC VMXR0InvVPID
742
743
744	%if GC_ARCH_BITS == 64
745	;;
746	; Executes INVLPGA
747	;
748	; @param pPageGC msc:rcx gcc:rdi x86:[esp+04] Virtual page to invalidate
749	; @param uASID msc:rdx gcc:rsi x86:[esp+0C] Tagged TLB id
750	;
751	;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
752	BEGINPROC SVMR0InvlpgA
753	%ifdef RT_ARCH_AMD64
754	%ifdef ASM_CALL64_GCC
755	mov rax, rdi
756	mov rcx, rsi
757	%else
758	mov rax, rcx
759	mov rcx, rdx
760	%endif
761	%else
762	mov eax, [esp + 4]
763	mov ecx, [esp + 0Ch]
764	%endif
765	invlpga [xAX], ecx
766	ret
767	ENDPROC SVMR0InvlpgA
768
769	%else ; GC_ARCH_BITS != 64
770	;;
771	; Executes INVLPGA
772	;
773	; @param pPageGC msc:ecx gcc:edi x86:[esp+04] Virtual page to invalidate
774	; @param uASID msc:edx gcc:esi x86:[esp+08] Tagged TLB id
775	;
776	;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
777	BEGINPROC SVMR0InvlpgA
778	%ifdef RT_ARCH_AMD64
779	%ifdef ASM_CALL64_GCC
780	movzx rax, edi
781	mov ecx, esi
782	%else
783	; from http://www.cs.cmu.edu/~fp/courses/15213-s06/misc/asm64-handout.pdf:
784	; ``Perhaps unexpectedly, instructions that move or generate 32-bit register
785	; values also set the upper 32 bits of the register to zero. Consequently
786	; there is no need for an instruction movzlq.''
787	mov eax, ecx
788	mov ecx, edx
789	%endif
790	%else
791	mov eax, [esp + 4]
792	mov ecx, [esp + 8]
793	%endif
794	invlpga [xAX], ecx
795	ret
796	ENDPROC SVMR0InvlpgA
797
798	%endif ; GC_ARCH_BITS != 64
799
800
801	%ifdef VBOX_WITH_KERNEL_USING_XMM
802
803	;;
804	; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
805	; load the guest ones when necessary.
806	;
807	; @cproto DECLASM(int) HMR0VMXStartVMhmR0DumpDescriptorM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM,
808	; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
809	;
810	; @returns eax
811	;
812	; @param fResumeVM msc:rcx
813	; @param pCtx msc:rdx
814	; @param pVMCSCache msc:r8
815	; @param pVM msc:r9
816	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
817	; @param pfnStartVM msc:[rbp+38h]
818	;
819	; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
820	;
821	; @remarks Drivers shouldn't use AVX registers without saving+loading:
822	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
823	; However the compiler docs have different idea:
824	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
825	; We'll go with the former for now.
826	;
827	; ASSUMING 64-bit and windows for now.
828	;
829	ALIGNCODE(16)
830	BEGINPROC hmR0VMXStartVMWrapXMM
831	push xBP
832	mov xBP, xSP
833	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
834
835	; spill input parameters.
836	mov [xBP + 010h], rcx ; fResumeVM
837	mov [xBP + 018h], rdx ; pCtx
838	mov [xBP + 020h], r8 ; pVMCSCache
839	mov [xBP + 028h], r9 ; pVM
840
841	; Ask CPUM whether we've started using the FPU yet.
842	mov rcx, [xBP + 30h] ; pVCpu
843	call NAME(CPUMIsGuestFPUStateActive)
844	test al, al
845	jnz .guest_fpu_state_active
846
847	; No need to mess with XMM registers just call the start routine and return.
848	mov r11, [xBP + 38h] ; pfnStartVM
849	mov r10, [xBP + 30h] ; pVCpu
850	mov [xSP + 020h], r10
851	mov rcx, [xBP + 010h] ; fResumeVM
852	mov rdx, [xBP + 018h] ; pCtx
853	mov r8, [xBP + 020h] ; pVMCSCache
854	mov r9, [xBP + 028h] ; pVM
855	call r11
856
857	leave
858	ret
859
860	ALIGNCODE(8)
861	.guest_fpu_state_active:
862	; Save the non-volatile host XMM registers.
863	movdqa [rsp + 040h + 000h], xmm6
864	movdqa [rsp + 040h + 010h], xmm7
865	movdqa [rsp + 040h + 020h], xmm8
866	movdqa [rsp + 040h + 030h], xmm9
867	movdqa [rsp + 040h + 040h], xmm10
868	movdqa [rsp + 040h + 050h], xmm11
869	movdqa [rsp + 040h + 060h], xmm12
870	movdqa [rsp + 040h + 070h], xmm13
871	movdqa [rsp + 040h + 080h], xmm14
872	movdqa [rsp + 040h + 090h], xmm15
873	stmxcsr [rsp + 040h + 0a0h]
874
875	mov r10, [xBP + 018h] ; pCtx
876	mov eax, [r10 + CPUMCTX.fXStateMask]
877	test eax, eax
878	jz .guest_fpu_state_manually
879
880	;
881	; Using XSAVE to load the guest XMM, YMM and ZMM registers.
882	;
883	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
884	xor edx, edx
885	mov r10, [r10 + CPUMCTX.pXStateR0]
886	xrstor [r10]
887
888	; Make the call (same as in the other case ).
889	mov r11, [xBP + 38h] ; pfnStartVM
890	mov r10, [xBP + 30h] ; pVCpu
891	mov [xSP + 020h], r10
892	mov rcx, [xBP + 010h] ; fResumeVM
893	mov rdx, [xBP + 018h] ; pCtx
894	mov r8, [xBP + 020h] ; pVMCSCache
895	mov r9, [xBP + 028h] ; pVM
896	call r11
897
898	mov r11d, eax ; save return value (xsave below uses eax)
899
900	; Save the guest XMM registers.
901	mov r10, [xBP + 018h] ; pCtx
902	mov eax, [r10 + CPUMCTX.fXStateMask]
903	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
904	xor edx, edx
905	mov r10, [r10 + CPUMCTX.pXStateR0]
906	xsave [r10]
907
908	mov eax, r11d ; restore return value.
909
910	.restore_non_volatile_host_xmm_regs:
911	; Load the non-volatile host XMM registers.
912	movdqa xmm6, [rsp + 040h + 000h]
913	movdqa xmm7, [rsp + 040h + 010h]
914	movdqa xmm8, [rsp + 040h + 020h]
915	movdqa xmm9, [rsp + 040h + 030h]
916	movdqa xmm10, [rsp + 040h + 040h]
917	movdqa xmm11, [rsp + 040h + 050h]
918	movdqa xmm12, [rsp + 040h + 060h]
919	movdqa xmm13, [rsp + 040h + 070h]
920	movdqa xmm14, [rsp + 040h + 080h]
921	movdqa xmm15, [rsp + 040h + 090h]
922	ldmxcsr [rsp + 040h + 0a0h]
923	leave
924	ret
925
926	;
927	; No XSAVE, load and save the guest XMM registers manually.
928	;
929	.guest_fpu_state_manually:
930	; Load the full guest XMM register state.
931	mov r10, [r10 + CPUMCTX.pXStateR0]
932	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
933	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
934	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
935	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
936	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
937	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
938	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
939	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
940	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
941	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
942	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
943	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
944	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
945	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
946	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
947	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
948	ldmxcsr [r10 + X86FXSTATE.MXCSR]
949
950	; Make the call (same as in the other case ).
951	mov r11, [xBP + 38h] ; pfnStartVM
952	mov r10, [xBP + 30h] ; pVCpu
953	mov [xSP + 020h], r10
954	mov rcx, [xBP + 010h] ; fResumeVM
955	mov rdx, [xBP + 018h] ; pCtx
956	mov r8, [xBP + 020h] ; pVMCSCache
957	mov r9, [xBP + 028h] ; pVM
958	call r11
959
960	; Save the guest XMM registers.
961	mov r10, [xBP + 018h] ; pCtx
962	mov r10, [r10 + CPUMCTX.pXStateR0]
963	stmxcsr [r10 + X86FXSTATE.MXCSR]
964	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
965	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
966	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
967	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
968	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
969	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
970	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
971	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
972	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
973	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
974	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
975	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
976	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
977	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
978	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
979	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
980	jmp .restore_non_volatile_host_xmm_regs
981	ENDPROC hmR0VMXStartVMWrapXMM
982
983	;;
984	; Wrapper around svm.pfnVMRun that preserves host XMM registers and
985	; load the guest ones when necessary.
986	;
987	; @cproto DECLASM(int) hmR0SVMRunWrapXMM(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu,
988	; PFNHMSVMVMRUN pfnVMRun);
989	;
990	; @returns eax
991	;
992	; @param HCPhysVmcbHost msc:rcx
993	; @param HCPhysVmcb msc:rdx
994	; @param pCtx msc:r8
995	; @param pVM msc:r9
996	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
997	; @param pfnVMRun msc:[rbp+38h]
998	;
999	; @remarks This is essentially the same code as hmR0VMXStartVMWrapXMM, only the parameters differ a little bit.
1000	;
1001	; @remarks Drivers shouldn't use AVX registers without saving+loading:
1002	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
1003	; However the compiler docs have different idea:
1004	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
1005	; We'll go with the former for now.
1006	;
1007	; ASSUMING 64-bit and windows for now.
1008	ALIGNCODE(16)
1009	BEGINPROC hmR0SVMRunWrapXMM
1010	push xBP
1011	mov xBP, xSP
1012	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
1013
1014	; spill input parameters.
1015	mov [xBP + 010h], rcx ; HCPhysVmcbHost
1016	mov [xBP + 018h], rdx ; HCPhysVmcb
1017	mov [xBP + 020h], r8 ; pCtx
1018	mov [xBP + 028h], r9 ; pVM
1019
1020	; Ask CPUM whether we've started using the FPU yet.
1021	mov rcx, [xBP + 30h] ; pVCpu
1022	call NAME(CPUMIsGuestFPUStateActive)
1023	test al, al
1024	jnz .guest_fpu_state_active
1025
1026	; No need to mess with XMM registers just call the start routine and return.
1027	mov r11, [xBP + 38h] ; pfnVMRun
1028	mov r10, [xBP + 30h] ; pVCpu
1029	mov [xSP + 020h], r10
1030	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1031	mov rdx, [xBP + 018h] ; HCPhysVmcb
1032	mov r8, [xBP + 020h] ; pCtx
1033	mov r9, [xBP + 028h] ; pVM
1034	call r11
1035
1036	leave
1037	ret
1038
1039	ALIGNCODE(8)
1040	.guest_fpu_state_active:
1041	; Save the non-volatile host XMM registers.
1042	movdqa [rsp + 040h + 000h], xmm6
1043	movdqa [rsp + 040h + 010h], xmm7
1044	movdqa [rsp + 040h + 020h], xmm8
1045	movdqa [rsp + 040h + 030h], xmm9
1046	movdqa [rsp + 040h + 040h], xmm10
1047	movdqa [rsp + 040h + 050h], xmm11
1048	movdqa [rsp + 040h + 060h], xmm12
1049	movdqa [rsp + 040h + 070h], xmm13
1050	movdqa [rsp + 040h + 080h], xmm14
1051	movdqa [rsp + 040h + 090h], xmm15
1052	stmxcsr [rsp + 040h + 0a0h]
1053
1054	mov r10, [xBP + 020h] ; pCtx
1055	mov eax, [r10 + CPUMCTX.fXStateMask]
1056	test eax, eax
1057	jz .guest_fpu_state_manually
1058
1059	;
1060	; Using XSAVE.
1061	;
1062	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1063	xor edx, edx
1064	mov r10, [r10 + CPUMCTX.pXStateR0]
1065	xrstor [r10]
1066
1067	; Make the call (same as in the other case ).
1068	mov r11, [xBP + 38h] ; pfnVMRun
1069	mov r10, [xBP + 30h] ; pVCpu
1070	mov [xSP + 020h], r10
1071	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1072	mov rdx, [xBP + 018h] ; HCPhysVmcb
1073	mov r8, [xBP + 020h] ; pCtx
1074	mov r9, [xBP + 028h] ; pVM
1075	call r11
1076
1077	mov r11d, eax ; save return value (xsave below uses eax)
1078
1079	; Save the guest XMM registers.
1080	mov r10, [xBP + 020h] ; pCtx
1081	mov eax, [r10 + CPUMCTX.fXStateMask]
1082	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1083	xor edx, edx
1084	mov r10, [r10 + CPUMCTX.pXStateR0]
1085	xsave [r10]
1086
1087	mov eax, r11d ; restore return value.
1088
1089	.restore_non_volatile_host_xmm_regs:
1090	; Load the non-volatile host XMM registers.
1091	movdqa xmm6, [rsp + 040h + 000h]
1092	movdqa xmm7, [rsp + 040h + 010h]
1093	movdqa xmm8, [rsp + 040h + 020h]
1094	movdqa xmm9, [rsp + 040h + 030h]
1095	movdqa xmm10, [rsp + 040h + 040h]
1096	movdqa xmm11, [rsp + 040h + 050h]
1097	movdqa xmm12, [rsp + 040h + 060h]
1098	movdqa xmm13, [rsp + 040h + 070h]
1099	movdqa xmm14, [rsp + 040h + 080h]
1100	movdqa xmm15, [rsp + 040h + 090h]
1101	ldmxcsr [rsp + 040h + 0a0h]
1102	leave
1103	ret
1104
1105	;
1106	; No XSAVE, load and save the guest XMM registers manually.
1107	;
1108	.guest_fpu_state_manually:
1109	; Load the full guest XMM register state.
1110	mov r10, [r10 + CPUMCTX.pXStateR0]
1111	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
1112	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
1113	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
1114	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
1115	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
1116	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
1117	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
1118	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
1119	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
1120	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
1121	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
1122	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
1123	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
1124	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
1125	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
1126	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
1127	ldmxcsr [r10 + X86FXSTATE.MXCSR]
1128
1129	; Make the call (same as in the other case ).
1130	mov r11, [xBP + 38h] ; pfnVMRun
1131	mov r10, [xBP + 30h] ; pVCpu
1132	mov [xSP + 020h], r10
1133	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1134	mov rdx, [xBP + 018h] ; HCPhysVmcb
1135	mov r8, [xBP + 020h] ; pCtx
1136	mov r9, [xBP + 028h] ; pVM
1137	call r11
1138
1139	; Save the guest XMM registers.
1140	mov r10, [xBP + 020h] ; pCtx
1141	mov r10, [r10 + CPUMCTX.pXStateR0]
1142	stmxcsr [r10 + X86FXSTATE.MXCSR]
1143	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1144	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1145	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1146	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1147	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1148	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1149	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1150	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1151	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1152	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1153	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1154	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1155	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1156	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1157	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1158	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1159	jmp .restore_non_volatile_host_xmm_regs
1160	ENDPROC hmR0SVMRunWrapXMM
1161
1162	%endif ; VBOX_WITH_KERNEL_USING_XMM
1163
1164
1165	;; @def RESTORE_STATE_VM32
1166	; Macro restoring essential host state and updating guest state
1167	; for common host, 32-bit guest for VT-x.
1168	%macro RESTORE_STATE_VM32 0
1169	; Restore base and limit of the IDTR & GDTR.
1170	%ifndef VMX_SKIP_IDTR
1171	lidt [xSP]
1172	add xSP, xCB * 2
1173	%endif
1174	%ifndef VMX_SKIP_GDTR
1175	lgdt [xSP]
1176	add xSP, xCB * 2
1177	%endif
1178
1179	push xDI
1180	%ifndef VMX_SKIP_TR
1181	mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR).
1182	%else
1183	mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR).
1184	%endif
1185
1186	mov [ss:xDI + CPUMCTX.eax], eax
1187	mov [ss:xDI + CPUMCTX.ebx], ebx
1188	mov [ss:xDI + CPUMCTX.ecx], ecx
1189	mov [ss:xDI + CPUMCTX.edx], edx
1190	mov [ss:xDI + CPUMCTX.esi], esi
1191	mov [ss:xDI + CPUMCTX.ebp], ebp
1192	mov xAX, cr2
1193	mov [ss:xDI + CPUMCTX.cr2], xAX
1194
1195	%ifdef RT_ARCH_AMD64
1196	pop xAX ; The guest edi we pushed above.
1197	mov dword [ss:xDI + CPUMCTX.edi], eax
1198	%else
1199	pop dword [ss:xDI + CPUMCTX.edi] ; The guest edi we pushed above.
1200	%endif
1201
1202	%ifndef VMX_SKIP_TR
1203	; Restore TSS selector; must mark it as not busy before using ltr (!)
1204	; ASSUME that this is supposed to be 'BUSY'. (saves 20-30 ticks on the T42p)
1205	; @todo get rid of sgdt
1206	pop xBX ; Saved TR
1207	sub xSP, xCB * 2
1208	sgdt [xSP]
1209	mov xAX, xBX
1210	and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
1211	add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset.
1212	and dword [ss:xAX + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
1213	ltr bx
1214	add xSP, xCB * 2
1215	%endif
1216
1217	pop xAX ; Saved LDTR
1218	%ifdef RT_ARCH_AMD64
1219	cmp eax, 0
1220	je %%skip_ldt_write32
1221	%endif
1222	lldt ax
1223
1224	%%skip_ldt_write32:
1225	add xSP, xCB ; pCtx
1226
1227	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1228	pop xDX ; Saved pCache
1229
1230	; Note! If we get here as a result of invalid VMCS pointer, all the following
1231	; vmread's will fail (only eflags.cf=1 will be set) but that shouldn't cause any
1232	; trouble only just less efficient.
1233	mov ecx, [ss:xDX + VMCSCACHE.Read.cValidEntries]
1234	cmp ecx, 0 ; Can't happen
1235	je %%no_cached_read32
1236	jmp %%cached_read32
1237
1238	ALIGN(16)
1239	%%cached_read32:
1240	dec xCX
1241	mov eax, [ss:xDX + VMCSCACHE.Read.aField + xCX * 4]
1242	; Note! This leaves the high 32 bits of the cache entry unmodified!!
1243	vmread [ss:xDX + VMCSCACHE.Read.aFieldVal + xCX * 8], xAX
1244	cmp xCX, 0
1245	jnz %%cached_read32
1246	%%no_cached_read32:
1247	%endif
1248
1249	; Restore segment registers.
1250	MYPOPSEGS xAX, ax
1251
1252	; Restore the host XCR0 if necessary.
1253	pop xCX
1254	test ecx, ecx
1255	jnz %%xcr0_after_skip
1256	pop xAX
1257	pop xDX
1258	xsetbv ; ecx is already zero.
1259	%%xcr0_after_skip:
1260
1261	; Restore general purpose registers.
1262	MYPOPAD
1263	%endmacro
1264
1265
1266	;;
1267	; Prepares for and executes VMLAUNCH/VMRESUME (32 bits guest mode)
1268	;
1269	; @returns VBox status code
1270	; @param fResume x86:[ebp+8], msc:rcx,gcc:rdi Whether to use vmlauch/vmresume.
1271	; @param pCtx x86:[ebp+c], msc:rdx,gcc:rsi Pointer to the guest-CPU context.
1272	; @param pCache x86:[ebp+10],msc:r8, gcc:rdx Pointer to the VMCS cache.
1273	; @param pVM x86:[ebp+14],msc:r9, gcc:rcx The cross context VM structure.
1274	; @param pVCpu x86:[ebp+18],msc:[ebp+30],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1275	;
1276	ALIGNCODE(16)
1277	BEGINPROC VMXR0StartVM32
1278	push xBP
1279	mov xBP, xSP
1280
1281	pushf
1282	cli
1283
1284	;
1285	; Save all general purpose host registers.
1286	;
1287	MYPUSHAD
1288
1289	;
1290	; First we have to write some final guest CPU context registers.
1291	;
1292	mov eax, VMX_VMCS_HOST_RIP
1293	%ifdef RT_ARCH_AMD64
1294	lea r10, [.vmlaunch_done wrt rip]
1295	vmwrite rax, r10
1296	%else
1297	mov ecx, .vmlaunch_done
1298	vmwrite eax, ecx
1299	%endif
1300	; Note: assumes success!
1301
1302	;
1303	; Unify input parameter registers.
1304	;
1305	%ifdef RT_ARCH_AMD64
1306	%ifdef ASM_CALL64_GCC
1307	; fResume already in rdi
1308	; pCtx already in rsi
1309	mov rbx, rdx ; pCache
1310	%else
1311	mov rdi, rcx ; fResume
1312	mov rsi, rdx ; pCtx
1313	mov rbx, r8 ; pCache
1314	%endif
1315	%else
1316	mov edi, [ebp + 8] ; fResume
1317	mov esi, [ebp + 12] ; pCtx
1318	mov ebx, [ebp + 16] ; pCache
1319	%endif
1320
1321	;
1322	; Save the host XCR0 and load the guest one if necessary.
1323	; Note! Trashes rdx and rcx.
1324	;
1325	%ifdef ASM_CALL64_MSC
1326	mov rax, [xBP + 30h] ; pVCpu
1327	%elifdef ASM_CALL64_GCC
1328	mov rax, r8 ; pVCpu
1329	%else
1330	mov eax, [xBP + 18h] ; pVCpu
1331	%endif
1332	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1333	jz .xcr0_before_skip
1334
1335	xor ecx, ecx
1336	xgetbv ; Save the host one on the stack.
1337	push xDX
1338	push xAX
1339
1340	mov eax, [xSI + CPUMCTX.aXcr] ; Load the guest one.
1341	mov edx, [xSI + CPUMCTX.aXcr + 4]
1342	xor ecx, ecx ; paranoia
1343	xsetbv
1344
1345	push 0 ; Indicate that we must restore XCR0 (popped into ecx, thus 0).
1346	jmp .xcr0_before_done
1347
1348	.xcr0_before_skip:
1349	push 3fh ; indicate that we need not.
1350	.xcr0_before_done:
1351
1352	;
1353	; Save segment registers.
1354	; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
1355	;
1356	MYPUSHSEGS xAX, ax
1357
1358	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1359	mov ecx, [xBX + VMCSCACHE.Write.cValidEntries]
1360	cmp ecx, 0
1361	je .no_cached_writes
1362	mov edx, ecx
1363	mov ecx, 0
1364	jmp .cached_write
1365
1366	ALIGN(16)
1367	.cached_write:
1368	mov eax, [xBX + VMCSCACHE.Write.aField + xCX * 4]
1369	vmwrite xAX, [xBX + VMCSCACHE.Write.aFieldVal + xCX * 8]
1370	inc xCX
1371	cmp xCX, xDX
1372	jl .cached_write
1373
1374	mov dword [xBX + VMCSCACHE.Write.cValidEntries], 0
1375	.no_cached_writes:
1376
1377	; Save the pCache pointer.
1378	push xBX
1379	%endif
1380
1381	; Save the pCtx pointer.
1382	push xSI
1383
1384	; Save host LDTR.
1385	xor eax, eax
1386	sldt ax
1387	push xAX
1388
1389	%ifndef VMX_SKIP_TR
1390	; The host TR limit is reset to 0x67; save & restore it manually.
1391	str eax
1392	push xAX
1393	%endif
1394
1395	%ifndef VMX_SKIP_GDTR
1396	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
1397	sub xSP, xCB * 2
1398	sgdt [xSP]
1399	%endif
1400	%ifndef VMX_SKIP_IDTR
1401	sub xSP, xCB * 2
1402	sidt [xSP]
1403	%endif
1404
1405	; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
1406	mov xBX, [xSI + CPUMCTX.cr2]
1407	mov xDX, cr2
1408	cmp xBX, xDX
1409	je .skip_cr2_write32
1410	mov cr2, xBX
1411
1412	.skip_cr2_write32:
1413	mov eax, VMX_VMCS_HOST_RSP
1414	vmwrite xAX, xSP
1415	; Note: assumes success!
1416	; Don't mess with ESP anymore!!!
1417
1418	; Load guest general purpose registers.
1419	mov eax, [xSI + CPUMCTX.eax]
1420	mov ebx, [xSI + CPUMCTX.ebx]
1421	mov ecx, [xSI + CPUMCTX.ecx]
1422	mov edx, [xSI + CPUMCTX.edx]
1423	mov ebp, [xSI + CPUMCTX.ebp]
1424
1425	; Resume or start VM?
1426	cmp xDI, 0 ; fResume
1427
1428	; Load guest edi & esi.
1429	mov edi, [xSI + CPUMCTX.edi]
1430	mov esi, [xSI + CPUMCTX.esi]
1431
1432	je .vmlaunch_launch
1433
1434	vmresume
1435	jc near .vmxstart_invalid_vmcs_ptr
1436	jz near .vmxstart_start_failed
1437	jmp .vmlaunch_done; ; Here if vmresume detected a failure.
1438
1439	.vmlaunch_launch:
1440	vmlaunch
1441	jc near .vmxstart_invalid_vmcs_ptr
1442	jz near .vmxstart_start_failed
1443	jmp .vmlaunch_done; ; Here if vmlaunch detected a failure.
1444
1445	ALIGNCODE(16) ;; @todo YASM BUG - this alignment is wrong on darwin, it's 1 byte off.
1446	.vmlaunch_done:
1447	RESTORE_STATE_VM32
1448	mov eax, VINF_SUCCESS
1449
1450	.vmstart_end:
1451	popf
1452	pop xBP
1453	ret
1454
1455	.vmxstart_invalid_vmcs_ptr:
1456	RESTORE_STATE_VM32
1457	mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1458	jmp .vmstart_end
1459
1460	.vmxstart_start_failed:
1461	RESTORE_STATE_VM32
1462	mov eax, VERR_VMX_UNABLE_TO_START_VM
1463	jmp .vmstart_end
1464
1465	ENDPROC VMXR0StartVM32
1466
1467
1468	%ifdef RT_ARCH_AMD64
1469	;; @def RESTORE_STATE_VM64
1470	; Macro restoring essential host state and updating guest state
1471	; for 64-bit host, 64-bit guest for VT-x.
1472	;
1473	%macro RESTORE_STATE_VM64 0
1474	; Restore base and limit of the IDTR & GDTR
1475	%ifndef VMX_SKIP_IDTR
1476	lidt [xSP]
1477	add xSP, xCB * 2
1478	%endif
1479	%ifndef VMX_SKIP_GDTR
1480	lgdt [xSP]
1481	add xSP, xCB * 2
1482	%endif
1483
1484	push xDI
1485	%ifndef VMX_SKIP_TR
1486	mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR)
1487	%else
1488	mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR)
1489	%endif
1490
1491	mov qword [xDI + CPUMCTX.eax], rax
1492	mov qword [xDI + CPUMCTX.ebx], rbx
1493	mov qword [xDI + CPUMCTX.ecx], rcx
1494	mov qword [xDI + CPUMCTX.edx], rdx
1495	mov qword [xDI + CPUMCTX.esi], rsi
1496	mov qword [xDI + CPUMCTX.ebp], rbp
1497	mov qword [xDI + CPUMCTX.r8], r8
1498	mov qword [xDI + CPUMCTX.r9], r9
1499	mov qword [xDI + CPUMCTX.r10], r10
1500	mov qword [xDI + CPUMCTX.r11], r11
1501	mov qword [xDI + CPUMCTX.r12], r12
1502	mov qword [xDI + CPUMCTX.r13], r13
1503	mov qword [xDI + CPUMCTX.r14], r14
1504	mov qword [xDI + CPUMCTX.r15], r15
1505	mov rax, cr2
1506	mov qword [xDI + CPUMCTX.cr2], rax
1507
1508	pop xAX ; The guest rdi we pushed above
1509	mov qword [xDI + CPUMCTX.edi], rax
1510
1511	%ifndef VMX_SKIP_TR
1512	; Restore TSS selector; must mark it as not busy before using ltr (!)
1513	; ASSUME that this is supposed to be 'BUSY'. (saves 20-30 ticks on the T42p).
1514	; @todo get rid of sgdt
1515	pop xBX ; Saved TR
1516	sub xSP, xCB * 2
1517	sgdt [xSP]
1518	mov xAX, xBX
1519	and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
1520	add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset.
1521	and dword [xAX + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
1522	ltr bx
1523	add xSP, xCB * 2
1524	%endif
1525
1526	pop xAX ; Saved LDTR
1527	cmp eax, 0
1528	je %%skip_ldt_write64
1529	lldt ax
1530
1531	%%skip_ldt_write64:
1532	pop xSI ; pCtx (needed in rsi by the macros below)
1533
1534	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1535	pop xDX ; Saved pCache
1536
1537	; Note! If we get here as a result of invalid VMCS pointer, all the following
1538	; vmread's will fail (only eflags.cf=1 will be set) but that shouldn't cause any
1539	; trouble only just less efficient.
1540	mov ecx, [xDX + VMCSCACHE.Read.cValidEntries]
1541	cmp ecx, 0 ; Can't happen
1542	je %%no_cached_read64
1543	jmp %%cached_read64
1544
1545	ALIGN(16)
1546	%%cached_read64:
1547	dec xCX
1548	mov eax, [xDX + VMCSCACHE.Read.aField + xCX * 4]
1549	vmread [xDX + VMCSCACHE.Read.aFieldVal + xCX * 8], xAX
1550	cmp xCX, 0
1551	jnz %%cached_read64
1552	%%no_cached_read64:
1553	%endif
1554
1555	; Restore segment registers.
1556	MYPOPSEGS xAX, ax
1557
1558	; Restore the host XCR0 if necessary.
1559	pop xCX
1560	test ecx, ecx
1561	jnz %%xcr0_after_skip
1562	pop xAX
1563	pop xDX
1564	xsetbv ; ecx is already zero.
1565	%%xcr0_after_skip:
1566
1567	; Restore general purpose registers.
1568	MYPOPAD
1569	%endmacro
1570
1571
1572	;;
1573	; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
1574	;
1575	; @returns VBox status code
1576	; @param fResume msc:rcx, gcc:rdi Whether to use vmlauch/vmresume.
1577	; @param pCtx msc:rdx, gcc:rsi Pointer to the guest-CPU context.
1578	; @param pCache msc:r8, gcc:rdx Pointer to the VMCS cache.
1579	; @param pVM msc:r9, gcc:rcx The cross context VM structure.
1580	; @param pVCpu msc:[ebp+30], gcc:r8 The cross context virtual CPU structure of the calling EMT.
1581	;
1582	ALIGNCODE(16)
1583	BEGINPROC VMXR0StartVM64
1584	push xBP
1585	mov xBP, xSP
1586
1587	pushf
1588	cli
1589
1590	; Save all general purpose host registers.
1591	MYPUSHAD
1592
1593	; First we have to save some final CPU context registers.
1594	lea r10, [.vmlaunch64_done wrt rip]
1595	mov rax, VMX_VMCS_HOST_RIP ; Return address (too difficult to continue after VMLAUNCH?).
1596	vmwrite rax, r10
1597	; Note: assumes success!
1598
1599	;
1600	; Unify the input parameter registers.
1601	;
1602	%ifdef ASM_CALL64_GCC
1603	; fResume already in rdi
1604	; pCtx already in rsi
1605	mov rbx, rdx ; pCache
1606	%else
1607	mov rdi, rcx ; fResume
1608	mov rsi, rdx ; pCtx
1609	mov rbx, r8 ; pCache
1610	%endif
1611
1612	;
1613	; Save the host XCR0 and load the guest one if necessary.
1614	; Note! Trashes rdx and rcx.
1615	;
1616	%ifdef ASM_CALL64_MSC
1617	mov rax, [xBP + 30h] ; pVCpu
1618	%else
1619	mov rax, r8 ; pVCpu
1620	%endif
1621	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1622	jz .xcr0_before_skip
1623
1624	xor ecx, ecx
1625	xgetbv ; Save the host one on the stack.
1626	push xDX
1627	push xAX
1628
1629	mov eax, [xSI + CPUMCTX.aXcr] ; Load the guest one.
1630	mov edx, [xSI + CPUMCTX.aXcr + 4]
1631	xor ecx, ecx ; paranoia
1632	xsetbv
1633
1634	push 0 ; Indicate that we must restore XCR0 (popped into ecx, thus 0).
1635	jmp .xcr0_before_done
1636
1637	.xcr0_before_skip:
1638	push 3fh ; indicate that we need not.
1639	.xcr0_before_done:
1640
1641	;
1642	; Save segment registers.
1643	; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
1644	;
1645	MYPUSHSEGS xAX, ax
1646
1647	%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1648	mov ecx, [xBX + VMCSCACHE.Write.cValidEntries]
1649	cmp ecx, 0
1650	je .no_cached_writes
1651	mov edx, ecx
1652	mov ecx, 0
1653	jmp .cached_write
1654
1655	ALIGN(16)
1656	.cached_write:
1657	mov eax, [xBX + VMCSCACHE.Write.aField + xCX * 4]
1658	vmwrite xAX, [xBX + VMCSCACHE.Write.aFieldVal + xCX * 8]
1659	inc xCX
1660	cmp xCX, xDX
1661	jl .cached_write
1662
1663	mov dword [xBX + VMCSCACHE.Write.cValidEntries], 0
1664	.no_cached_writes:
1665
1666	; Save the pCache pointer.
1667	push xBX
1668	%endif
1669
1670	; Save the pCtx pointer.
1671	push xSI
1672
1673	; Save host LDTR.
1674	xor eax, eax
1675	sldt ax
1676	push xAX
1677
1678	%ifndef VMX_SKIP_TR
1679	; The host TR limit is reset to 0x67; save & restore it manually.
1680	str eax
1681	push xAX
1682	%endif
1683
1684	%ifndef VMX_SKIP_GDTR
1685	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
1686	sub xSP, xCB * 2
1687	sgdt [xSP]
1688	%endif
1689	%ifndef VMX_SKIP_IDTR
1690	sub xSP, xCB * 2
1691	sidt [xSP]
1692	%endif
1693
1694	; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
1695	mov rbx, qword [xSI + CPUMCTX.cr2]
1696	mov rdx, cr2
1697	cmp rbx, rdx
1698	je .skip_cr2_write
1699	mov cr2, rbx
1700
1701	.skip_cr2_write:
1702	mov eax, VMX_VMCS_HOST_RSP
1703	vmwrite xAX, xSP
1704	; Note: assumes success!
1705	; Don't mess with ESP anymore!!!
1706
1707	; Load guest general purpose registers.
1708	mov rax, qword [xSI + CPUMCTX.eax]
1709	mov rbx, qword [xSI + CPUMCTX.ebx]
1710	mov rcx, qword [xSI + CPUMCTX.ecx]
1711	mov rdx, qword [xSI + CPUMCTX.edx]
1712	mov rbp, qword [xSI + CPUMCTX.ebp]
1713	mov r8, qword [xSI + CPUMCTX.r8]
1714	mov r9, qword [xSI + CPUMCTX.r9]
1715	mov r10, qword [xSI + CPUMCTX.r10]
1716	mov r11, qword [xSI + CPUMCTX.r11]
1717	mov r12, qword [xSI + CPUMCTX.r12]
1718	mov r13, qword [xSI + CPUMCTX.r13]
1719	mov r14, qword [xSI + CPUMCTX.r14]
1720	mov r15, qword [xSI + CPUMCTX.r15]
1721
1722	; Resume or start VM?
1723	cmp xDI, 0 ; fResume
1724
1725	; Load guest rdi & rsi.
1726	mov rdi, qword [xSI + CPUMCTX.edi]
1727	mov rsi, qword [xSI + CPUMCTX.esi]
1728
1729	je .vmlaunch64_launch
1730
1731	vmresume
1732	jc near .vmxstart64_invalid_vmcs_ptr
1733	jz near .vmxstart64_start_failed
1734	jmp .vmlaunch64_done; ; Here if vmresume detected a failure.
1735
1736	.vmlaunch64_launch:
1737	vmlaunch
1738	jc near .vmxstart64_invalid_vmcs_ptr
1739	jz near .vmxstart64_start_failed
1740	jmp .vmlaunch64_done; ; Here if vmlaunch detected a failure.
1741
1742	ALIGNCODE(16)
1743	.vmlaunch64_done:
1744	RESTORE_STATE_VM64
1745	mov eax, VINF_SUCCESS
1746
1747	.vmstart64_end:
1748	popf
1749	pop xBP
1750	ret
1751
1752	.vmxstart64_invalid_vmcs_ptr:
1753	RESTORE_STATE_VM64
1754	mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1755	jmp .vmstart64_end
1756
1757	.vmxstart64_start_failed:
1758	RESTORE_STATE_VM64
1759	mov eax, VERR_VMX_UNABLE_TO_START_VM
1760	jmp .vmstart64_end
1761	ENDPROC VMXR0StartVM64
1762	%endif ; RT_ARCH_AMD64
1763
1764
1765	;;
1766	; Prepares for and executes VMRUN (32 bits guests)
1767	;
1768	; @returns VBox status code
1769	; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB.
1770	; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB.
1771	; @param pCtx msc:r8,gcc:rdx Pointer to the guest CPU-context.
1772	; @param pVM msc:r9,gcc:rcx The cross context VM structure.
1773	; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1774	;
1775	ALIGNCODE(16)
1776	BEGINPROC SVMR0VMRun
1777	%ifdef RT_ARCH_AMD64 ; fake a cdecl stack frame
1778	%ifdef ASM_CALL64_GCC
1779	push r8 ; pVCpu
1780	push rcx ; pVM
1781	push rdx ; pCtx
1782	push rsi ; HCPhysVmcb
1783	push rdi ; HCPhysVmcbHost
1784	%else
1785	mov rax, [rsp + 28h]
1786	push rax ; pVCpu
1787	push r9 ; pVM
1788	push r8 ; pCtx
1789	push rdx ; HCPhysVmcb
1790	push rcx ; HCPhysVmcbHost
1791	%endif
1792	push 0
1793	%endif
1794	push xBP
1795	mov xBP, xSP
1796	pushf
1797
1798	; Save all general purpose host registers.
1799	MYPUSHAD
1800
1801	; Load pCtx into xSI.
1802	mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1803
1804	; Save the host XCR0 and load the guest one if necessary.
1805	mov xAX, [xBP + xCB * 2 + RTHCPHYS_CB * 2 + xCB * 2] ; pVCpu
1806	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1807	jz .xcr0_before_skip
1808
1809	xor ecx, ecx
1810	xgetbv ; Save the host XCR0 on the stack
1811	push xDX
1812	push xAX
1813
1814	mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1815	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0
1816	mov edx, [xSI + CPUMCTX.aXcr + 4]
1817	xor ecx, ecx ; paranoia
1818	xsetbv
1819
1820	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
1821	jmp .xcr0_before_done
1822
1823	.xcr0_before_skip:
1824	push 3fh ; indicate that we need not restore XCR0
1825	.xcr0_before_done:
1826
1827	; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
1828	push xSI
1829
1830	; Save host fs, gs, sysenter msr etc.
1831	mov xAX, [xBP + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
1832	push xAX ; save for the vmload after vmrun
1833	vmsave
1834
1835	; Setup xAX for VMLOAD.
1836	mov xAX, [xBP + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; x86: take low dword only)
1837
1838	; Load guest general purpose registers.
1839	; eax is loaded from the VMCB by VMRUN.
1840	mov ebx, [xSI + CPUMCTX.ebx]
1841	mov ecx, [xSI + CPUMCTX.ecx]
1842	mov edx, [xSI + CPUMCTX.edx]
1843	mov edi, [xSI + CPUMCTX.edi]
1844	mov ebp, [xSI + CPUMCTX.ebp]
1845	mov esi, [xSI + CPUMCTX.esi]
1846
1847	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1848	clgi
1849	sti
1850
1851	; Load guest fs, gs, sysenter msr etc.
1852	vmload
1853
1854	; Run the VM.
1855	vmrun
1856
1857	; Save guest fs, gs, sysenter msr etc.
1858	vmsave
1859
1860	; Load host fs, gs, sysenter msr etc.
1861	pop xAX ; load HCPhysVmcbHost (pushed above)
1862	vmload
1863
1864	; Set the global interrupt flag again, but execute cli to make sure IF=0.
1865	cli
1866	stgi
1867
1868	; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
1869	pop xAX
1870
1871	mov [ss:xAX + CPUMCTX.ebx], ebx
1872	mov [ss:xAX + CPUMCTX.ecx], ecx
1873	mov [ss:xAX + CPUMCTX.edx], edx
1874	mov [ss:xAX + CPUMCTX.esi], esi
1875	mov [ss:xAX + CPUMCTX.edi], edi
1876	mov [ss:xAX + CPUMCTX.ebp], ebp
1877
1878	; Restore the host xcr0 if necessary.
1879	pop xCX
1880	test ecx, ecx
1881	jnz .xcr0_after_skip
1882	pop xAX
1883	pop xDX
1884	xsetbv ; ecx is already zero
1885	.xcr0_after_skip:
1886
1887	; Restore host general purpose registers.
1888	MYPOPAD
1889
1890	mov eax, VINF_SUCCESS
1891
1892	popf
1893	pop xBP
1894	%ifdef RT_ARCH_AMD64
1895	add xSP, 6*xCB
1896	%endif
1897	ret
1898	ENDPROC SVMR0VMRun
1899
1900
1901	%ifdef RT_ARCH_AMD64
1902	;;
1903	; Prepares for and executes VMRUN (64 bits guests)
1904	;
1905	; @returns VBox status code
1906	; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB.
1907	; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB.
1908	; @param pCtx msc:r8,gcc:rdx Pointer to the guest-CPU context.
1909	; @param pVM msc:r9,gcc:rcx The cross context VM structure.
1910	; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1911	;
1912	ALIGNCODE(16)
1913	BEGINPROC SVMR0VMRun64
1914	; Fake a cdecl stack frame
1915	%ifdef ASM_CALL64_GCC
1916	push r8 ;pVCpu
1917	push rcx ;pVM
1918	push rdx ;pCtx
1919	push rsi ;HCPhysVmcb
1920	push rdi ;HCPhysVmcbHost
1921	%else
1922	mov rax, [rsp + 28h]
1923	push rax ; rbp + 30h pVCpu
1924	push r9 ; rbp + 28h pVM
1925	push r8 ; rbp + 20h pCtx
1926	push rdx ; rbp + 18h HCPhysVmcb
1927	push rcx ; rbp + 10h HCPhysVmcbHost
1928	%endif
1929	push 0 ; rbp + 08h "fake ret addr"
1930	push rbp ; rbp + 00h
1931	mov rbp, rsp
1932	pushf
1933
1934	; Manual save and restore:
1935	; - General purpose registers except RIP, RSP, RAX
1936	;
1937	; Trashed:
1938	; - CR2 (we don't care)
1939	; - LDTR (reset to 0)
1940	; - DRx (presumably not changed at all)
1941	; - DR7 (reset to 0x400)
1942
1943	; Save all general purpose host registers.
1944	MYPUSHAD
1945
1946	; Load pCtx into xSI.
1947	mov xSI, [rbp + xCB * 2 + RTHCPHYS_CB * 2]
1948
1949	; Save the host XCR0 and load the guest one if necessary.
1950	mov rax, [xBP + 30h] ; pVCpu
1951	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1952	jz .xcr0_before_skip
1953
1954	xor ecx, ecx
1955	xgetbv ; save the host XCR0 on the stack.
1956	push xDX
1957	push xAX
1958
1959	mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1960	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0
1961	mov edx, [xSI + CPUMCTX.aXcr + 4]
1962	xor ecx, ecx ; paranoia
1963	xsetbv
1964
1965	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
1966	jmp .xcr0_before_done
1967
1968	.xcr0_before_skip:
1969	push 3fh ; indicate that we need not restore XCR0
1970	.xcr0_before_done:
1971
1972	; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
1973	push rsi
1974
1975	; Save host fs, gs, sysenter msr etc.
1976	mov rax, [rbp + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
1977	push rax ; save for the vmload after vmrun
1978	vmsave
1979
1980	; Setup rax for VMLOAD.
1981	mov rax, [rbp + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; take low dword only)
1982
1983	; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
1984	mov rbx, qword [xSI + CPUMCTX.ebx]
1985	mov rcx, qword [xSI + CPUMCTX.ecx]
1986	mov rdx, qword [xSI + CPUMCTX.edx]
1987	mov rdi, qword [xSI + CPUMCTX.edi]
1988	mov rbp, qword [xSI + CPUMCTX.ebp]
1989	mov r8, qword [xSI + CPUMCTX.r8]
1990	mov r9, qword [xSI + CPUMCTX.r9]
1991	mov r10, qword [xSI + CPUMCTX.r10]
1992	mov r11, qword [xSI + CPUMCTX.r11]
1993	mov r12, qword [xSI + CPUMCTX.r12]
1994	mov r13, qword [xSI + CPUMCTX.r13]
1995	mov r14, qword [xSI + CPUMCTX.r14]
1996	mov r15, qword [xSI + CPUMCTX.r15]
1997	mov rsi, qword [xSI + CPUMCTX.esi]
1998
1999	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
2000	clgi
2001	sti
2002
2003	; Load guest FS, GS, Sysenter MSRs etc.
2004	vmload
2005
2006	; Run the VM.
2007	vmrun
2008
2009	; Save guest fs, gs, sysenter msr etc.
2010	vmsave
2011
2012	; Load host fs, gs, sysenter msr etc.
2013	pop rax ; load HCPhysVmcbHost (pushed above)
2014	vmload
2015
2016	; Set the global interrupt flag again, but execute cli to make sure IF=0.
2017	cli
2018	stgi
2019
2020	; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
2021	pop rax
2022
2023	mov qword [rax + CPUMCTX.ebx], rbx
2024	mov qword [rax + CPUMCTX.ecx], rcx
2025	mov qword [rax + CPUMCTX.edx], rdx
2026	mov qword [rax + CPUMCTX.esi], rsi
2027	mov qword [rax + CPUMCTX.edi], rdi
2028	mov qword [rax + CPUMCTX.ebp], rbp
2029	mov qword [rax + CPUMCTX.r8], r8
2030	mov qword [rax + CPUMCTX.r9], r9
2031	mov qword [rax + CPUMCTX.r10], r10
2032	mov qword [rax + CPUMCTX.r11], r11
2033	mov qword [rax + CPUMCTX.r12], r12
2034	mov qword [rax + CPUMCTX.r13], r13
2035	mov qword [rax + CPUMCTX.r14], r14
2036	mov qword [rax + CPUMCTX.r15], r15
2037
2038	; Restore the host xcr0 if necessary.
2039	pop xCX
2040	test ecx, ecx
2041	jnz .xcr0_after_skip
2042	pop xAX
2043	pop xDX
2044	xsetbv ; ecx is already zero
2045	.xcr0_after_skip:
2046
2047	; Restore host general purpose registers.
2048	MYPOPAD
2049
2050	mov eax, VINF_SUCCESS
2051
2052	popf
2053	pop rbp
2054	add rsp, 6 * xCB
2055	ret
2056	ENDPROC SVMR0VMRun64
2057	%endif ; RT_ARCH_AMD64
2058

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 70262

Download in other formats: