VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 72372

Last change on this file since 72372 was 70606, checked in by vboxsync, 7 years ago

updates (bugref:9087)
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 63.6 KB
Line 
1; $Id: HMR0A.asm 70606 2018-01-16 19:05:36Z vboxsync $
2;; @file
3; HM - Ring-0 VMX, SVM world-switch and helper routines
4;
5
6;
7; Copyright (C) 2006-2017 Oracle Corporation
8;
9; This file is part of VirtualBox Open Source Edition (OSE), as
10; available from http://www.virtualbox.org. This file is free software;
11; you can redistribute it and/or modify it under the terms of the GNU
12; General Public License (GPL) as published by the Free Software
13; Foundation, in version 2 as it comes in the "COPYING" file of the
14; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16;
17
18;*********************************************************************************************************************************
19;* Header Files *
20;*********************************************************************************************************************************
21%include "VBox/asmdefs.mac"
22%include "VBox/err.mac"
23%include "VBox/vmm/hm_vmx.mac"
24%include "VBox/vmm/cpum.mac"
25%include "VBox/vmm/vm.mac"
26%include "iprt/x86.mac"
27%include "HMInternal.mac"
28
29%ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely.
30 %macro vmwrite 2,
31 int3
32 %endmacro
33 %define vmlaunch int3
34 %define vmresume int3
35 %define vmsave int3
36 %define vmload int3
37 %define vmrun int3
38 %define clgi int3
39 %define stgi int3
40 %macro invlpga 2,
41 int3
42 %endmacro
43%endif
44
45;*********************************************************************************************************************************
46;* Defined Constants And Macros *
47;*********************************************************************************************************************************
48;; The offset of the XMM registers in X86FXSTATE.
49; Use define because I'm too lazy to convert the struct.
50%define XMM_OFF_IN_X86FXSTATE 160
51
52;; Spectre filler for 32-bit mode.
53; Some user space address that points to a 4MB page boundrary in hope that it
54; will somehow make it less useful.
55%define SPECTRE_FILLER32 0x227fffff
56;; Spectre filler for 64-bit mode.
57; Choosen to be an invalid address (also with 5 level paging).
58%define SPECTRE_FILLER64 0x02204204207fffff
59;; Spectre filler for the current CPU mode.
60%ifdef RT_ARCH_AMD64
61 %define SPECTRE_FILLER SPECTRE_FILLER64
62%else
63 %define SPECTRE_FILLER SPECTRE_FILLER32
64%endif
65
66;;
67; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation
68;
69%ifdef RT_ARCH_AMD64
70 %define VMX_SKIP_GDTR
71 %define VMX_SKIP_TR
72 %define VBOX_SKIP_RESTORE_SEG
73 %ifdef RT_OS_DARWIN
74 ; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
75 ; risk loading a stale LDT value or something invalid.
76 %define HM_64_BIT_USE_NULL_SEL
77 ; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always.
78 ; See @bugref{6875}.
79 %else
80 %define VMX_SKIP_IDTR
81 %endif
82%endif
83
84;; @def MYPUSHAD
85; Macro generating an equivalent to pushad
86
87;; @def MYPOPAD
88; Macro generating an equivalent to popad
89
90;; @def MYPUSHSEGS
91; Macro saving all segment registers on the stack.
92; @param 1 full width register name
93; @param 2 16-bit register name for \a 1.
94
95;; @def MYPOPSEGS
96; Macro restoring all segment registers on the stack
97; @param 1 full width register name
98; @param 2 16-bit register name for \a 1.
99
100%ifdef ASM_CALL64_GCC
101 %macro MYPUSHAD64 0
102 push r15
103 push r14
104 push r13
105 push r12
106 push rbx
107 %endmacro
108 %macro MYPOPAD64 0
109 pop rbx
110 pop r12
111 pop r13
112 pop r14
113 pop r15
114 %endmacro
115
116%else ; ASM_CALL64_MSC
117 %macro MYPUSHAD64 0
118 push r15
119 push r14
120 push r13
121 push r12
122 push rbx
123 push rsi
124 push rdi
125 %endmacro
126 %macro MYPOPAD64 0
127 pop rdi
128 pop rsi
129 pop rbx
130 pop r12
131 pop r13
132 pop r14
133 pop r15
134 %endmacro
135%endif
136
137%ifdef VBOX_SKIP_RESTORE_SEG
138 %macro MYPUSHSEGS64 2
139 %endmacro
140
141 %macro MYPOPSEGS64 2
142 %endmacro
143%else ; !VBOX_SKIP_RESTORE_SEG
144 ; trashes, rax, rdx & rcx
145 %macro MYPUSHSEGS64 2
146 %ifndef HM_64_BIT_USE_NULL_SEL
147 mov %2, es
148 push %1
149 mov %2, ds
150 push %1
151 %endif
152
153 ; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode, Solaris OTOH doesn't and we must save it.
154 mov ecx, MSR_K8_FS_BASE
155 rdmsr
156 push rdx
157 push rax
158 %ifndef HM_64_BIT_USE_NULL_SEL
159 push fs
160 %endif
161
162 ; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel. The same happens on exit
163 mov ecx, MSR_K8_GS_BASE
164 rdmsr
165 push rdx
166 push rax
167 %ifndef HM_64_BIT_USE_NULL_SEL
168 push gs
169 %endif
170 %endmacro
171
172 ; trashes, rax, rdx & rcx
173 %macro MYPOPSEGS64 2
174 ; Note: do not step through this code with a debugger!
175 %ifndef HM_64_BIT_USE_NULL_SEL
176 xor eax, eax
177 mov ds, ax
178 mov es, ax
179 mov fs, ax
180 mov gs, ax
181 %endif
182
183 %ifndef HM_64_BIT_USE_NULL_SEL
184 pop gs
185 %endif
186 pop rax
187 pop rdx
188 mov ecx, MSR_K8_GS_BASE
189 wrmsr
190
191 %ifndef HM_64_BIT_USE_NULL_SEL
192 pop fs
193 %endif
194 pop rax
195 pop rdx
196 mov ecx, MSR_K8_FS_BASE
197 wrmsr
198 ; Now it's safe to step again
199
200 %ifndef HM_64_BIT_USE_NULL_SEL
201 pop %1
202 mov ds, %2
203 pop %1
204 mov es, %2
205 %endif
206 %endmacro
207%endif ; VBOX_SKIP_RESTORE_SEG
208
209%macro MYPUSHAD32 0
210 pushad
211%endmacro
212%macro MYPOPAD32 0
213 popad
214%endmacro
215
216%macro MYPUSHSEGS32 2
217 push ds
218 push es
219 push fs
220 push gs
221%endmacro
222%macro MYPOPSEGS32 2
223 pop gs
224 pop fs
225 pop es
226 pop ds
227%endmacro
228
229%ifdef RT_ARCH_AMD64
230 %define MYPUSHAD MYPUSHAD64
231 %define MYPOPAD MYPOPAD64
232 %define MYPUSHSEGS MYPUSHSEGS64
233 %define MYPOPSEGS MYPOPSEGS64
234%else
235 %define MYPUSHAD MYPUSHAD32
236 %define MYPOPAD MYPOPAD32
237 %define MYPUSHSEGS MYPUSHSEGS32
238 %define MYPOPSEGS MYPOPSEGS32
239%endif
240
241;;
242; Creates an indirect branch prediction barrier on CPUs that need and supports that.
243; @clobbers eax, edx, ecx
244; @param 1 How to address CPUMCTX.
245; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
246%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2
247 test byte [%1 + CPUMCTX.fWorldSwitcher], %2
248 jz %%no_indirect_branch_barrier
249 mov ecx, MSR_IA32_PRED_CMD
250 mov eax, MSR_IA32_PRED_CMD_F_IBPB
251 xor edx, edx
252 wrmsr
253%%no_indirect_branch_barrier:
254%endmacro
255
256
257;*********************************************************************************************************************************
258;* External Symbols *
259;*********************************************************************************************************************************
260%ifdef VBOX_WITH_KERNEL_USING_XMM
261extern NAME(CPUMIsGuestFPUStateActive)
262%endif
263
264
265BEGINCODE
266
267
268;/**
269; * Restores host-state fields.
270; *
271; * @returns VBox status code
272; * @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags.
273; * @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct.
274; */
275ALIGNCODE(16)
276BEGINPROC VMXRestoreHostState
277%ifdef RT_ARCH_AMD64
278 %ifndef ASM_CALL64_GCC
279 ; Use GCC's input registers since we'll be needing both rcx and rdx further
280 ; down with the wrmsr instruction. Use the R10 and R11 register for saving
281 ; RDI and RSI since MSC preserve the two latter registers.
282 mov r10, rdi
283 mov r11, rsi
284 mov rdi, rcx
285 mov rsi, rdx
286 %endif
287
288 test edi, VMX_RESTORE_HOST_GDTR
289 jz .test_idtr
290 lgdt [rsi + VMXRESTOREHOST.HostGdtr]
291
292.test_idtr:
293 test edi, VMX_RESTORE_HOST_IDTR
294 jz .test_ds
295 lidt [rsi + VMXRESTOREHOST.HostIdtr]
296
297.test_ds:
298 test edi, VMX_RESTORE_HOST_SEL_DS
299 jz .test_es
300 mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
301 mov ds, eax
302
303.test_es:
304 test edi, VMX_RESTORE_HOST_SEL_ES
305 jz .test_tr
306 mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
307 mov es, eax
308
309.test_tr:
310 test edi, VMX_RESTORE_HOST_SEL_TR
311 jz .test_fs
312 ; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
313 mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
314 mov ax, dx
315 and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
316 test edi, VMX_RESTORE_HOST_GDT_READ_ONLY | VMX_RESTORE_HOST_GDT_NEED_WRITABLE
317 jnz .gdt_readonly
318 add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
319 and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
320 ltr dx
321 jmp short .test_fs
322.gdt_readonly:
323 test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
324 jnz .gdt_readonly_need_writable
325 mov rcx, cr0
326 mov r9, rcx
327 add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
328 and rcx, ~X86_CR0_WP
329 mov cr0, rcx
330 and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
331 ltr dx
332 mov cr0, r9
333 jmp short .test_fs
334.gdt_readonly_need_writable:
335 add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw.
336 and dword [rax + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
337 lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
338 ltr dx
339 lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; Load the original GDT
340
341.test_fs:
342 ;
343 ; When restoring the selector values for FS and GS, we'll temporarily trash
344 ; the base address (at least the high 32-bit bits, but quite possibly the
345 ; whole base address), the wrmsr will restore it correctly. (VT-x actually
346 ; restores the base correctly when leaving guest mode, but not the selector
347 ; value, so there is little problem with interrupts being enabled prior to
348 ; this restore job.)
349 ; We'll disable ints once for both FS and GS as that's probably faster.
350 ;
351 test edi, VMX_RESTORE_HOST_SEL_FS | VMX_RESTORE_HOST_SEL_GS
352 jz .restore_success
353 pushfq
354 cli ; (see above)
355
356 test edi, VMX_RESTORE_HOST_SEL_FS
357 jz .test_gs
358 mov ax, word [rsi + VMXRESTOREHOST.uHostSelFS]
359 mov fs, eax
360 mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
361 mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
362 mov ecx, MSR_K8_FS_BASE
363 wrmsr
364
365.test_gs:
366 test edi, VMX_RESTORE_HOST_SEL_GS
367 jz .restore_flags
368 mov ax, word [rsi + VMXRESTOREHOST.uHostSelGS]
369 mov gs, eax
370 mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
371 mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
372 mov ecx, MSR_K8_GS_BASE
373 wrmsr
374
375.restore_flags:
376 popfq
377
378.restore_success:
379 mov eax, VINF_SUCCESS
380 %ifndef ASM_CALL64_GCC
381 ; Restore RDI and RSI on MSC.
382 mov rdi, r10
383 mov rsi, r11
384 %endif
385%else ; RT_ARCH_X86
386 mov eax, VERR_NOT_IMPLEMENTED
387%endif
388 ret
389ENDPROC VMXRestoreHostState
390
391
392;/**
393; * Dispatches an NMI to the host.
394; */
395ALIGNCODE(16)
396BEGINPROC VMXDispatchHostNmi
397 int 2 ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
398 ret
399ENDPROC VMXDispatchHostNmi
400
401
402;/**
403; * Executes VMWRITE, 64-bit value.
404; *
405; * @returns VBox status code.
406; * @param idxField x86: [ebp + 08h] msc: rcx gcc: rdi VMCS index.
407; * @param u64Data x86: [ebp + 0ch] msc: rdx gcc: rsi VM field value.
408; */
409ALIGNCODE(16)
410BEGINPROC VMXWriteVmcs64
411%ifdef RT_ARCH_AMD64
412 %ifdef ASM_CALL64_GCC
413 and edi, 0ffffffffh
414 xor rax, rax
415 vmwrite rdi, rsi
416 %else
417 and ecx, 0ffffffffh
418 xor rax, rax
419 vmwrite rcx, rdx
420 %endif
421%else ; RT_ARCH_X86
422 mov ecx, [esp + 4] ; idxField
423 lea edx, [esp + 8] ; &u64Data
424 vmwrite ecx, [edx] ; low dword
425 jz .done
426 jc .done
427 inc ecx
428 xor eax, eax
429 vmwrite ecx, [edx + 4] ; high dword
430.done:
431%endif ; RT_ARCH_X86
432 jnc .valid_vmcs
433 mov eax, VERR_VMX_INVALID_VMCS_PTR
434 ret
435.valid_vmcs:
436 jnz .the_end
437 mov eax, VERR_VMX_INVALID_VMCS_FIELD
438.the_end:
439 ret
440ENDPROC VMXWriteVmcs64
441
442
443;/**
444; * Executes VMREAD, 64-bit value.
445; *
446; * @returns VBox status code.
447; * @param idxField VMCS index.
448; * @param pData Where to store VM field value.
449; */
450;DECLASM(int) VMXReadVmcs64(uint32_t idxField, uint64_t *pData);
451ALIGNCODE(16)
452BEGINPROC VMXReadVmcs64
453%ifdef RT_ARCH_AMD64
454 %ifdef ASM_CALL64_GCC
455 and edi, 0ffffffffh
456 xor rax, rax
457 vmread [rsi], rdi
458 %else
459 and ecx, 0ffffffffh
460 xor rax, rax
461 vmread [rdx], rcx
462 %endif
463%else ; RT_ARCH_X86
464 mov ecx, [esp + 4] ; idxField
465 mov edx, [esp + 8] ; pData
466 vmread [edx], ecx ; low dword
467 jz .done
468 jc .done
469 inc ecx
470 xor eax, eax
471 vmread [edx + 4], ecx ; high dword
472.done:
473%endif ; RT_ARCH_X86
474 jnc .valid_vmcs
475 mov eax, VERR_VMX_INVALID_VMCS_PTR
476 ret
477.valid_vmcs:
478 jnz .the_end
479 mov eax, VERR_VMX_INVALID_VMCS_FIELD
480.the_end:
481 ret
482ENDPROC VMXReadVmcs64
483
484
485;/**
486; * Executes VMREAD, 32-bit value.
487; *
488; * @returns VBox status code.
489; * @param idxField VMCS index.
490; * @param pu32Data Where to store VM field value.
491; */
492;DECLASM(int) VMXReadVmcs32(uint32_t idxField, uint32_t *pu32Data);
493ALIGNCODE(16)
494BEGINPROC VMXReadVmcs32
495%ifdef RT_ARCH_AMD64
496 %ifdef ASM_CALL64_GCC
497 and edi, 0ffffffffh
498 xor rax, rax
499 vmread r10, rdi
500 mov [rsi], r10d
501 %else
502 and ecx, 0ffffffffh
503 xor rax, rax
504 vmread r10, rcx
505 mov [rdx], r10d
506 %endif
507%else ; RT_ARCH_X86
508 mov ecx, [esp + 4] ; idxField
509 mov edx, [esp + 8] ; pu32Data
510 xor eax, eax
511 vmread [edx], ecx
512%endif ; RT_ARCH_X86
513 jnc .valid_vmcs
514 mov eax, VERR_VMX_INVALID_VMCS_PTR
515 ret
516.valid_vmcs:
517 jnz .the_end
518 mov eax, VERR_VMX_INVALID_VMCS_FIELD
519.the_end:
520 ret
521ENDPROC VMXReadVmcs32
522
523
524;/**
525; * Executes VMWRITE, 32-bit value.
526; *
527; * @returns VBox status code.
528; * @param idxField VMCS index.
529; * @param u32Data Where to store VM field value.
530; */
531;DECLASM(int) VMXWriteVmcs32(uint32_t idxField, uint32_t u32Data);
532ALIGNCODE(16)
533BEGINPROC VMXWriteVmcs32
534%ifdef RT_ARCH_AMD64
535 %ifdef ASM_CALL64_GCC
536 and edi, 0ffffffffh
537 and esi, 0ffffffffh
538 xor rax, rax
539 vmwrite rdi, rsi
540 %else
541 and ecx, 0ffffffffh
542 and edx, 0ffffffffh
543 xor rax, rax
544 vmwrite rcx, rdx
545 %endif
546%else ; RT_ARCH_X86
547 mov ecx, [esp + 4] ; idxField
548 mov edx, [esp + 8] ; u32Data
549 xor eax, eax
550 vmwrite ecx, edx
551%endif ; RT_ARCH_X86
552 jnc .valid_vmcs
553 mov eax, VERR_VMX_INVALID_VMCS_PTR
554 ret
555.valid_vmcs:
556 jnz .the_end
557 mov eax, VERR_VMX_INVALID_VMCS_FIELD
558.the_end:
559 ret
560ENDPROC VMXWriteVmcs32
561
562
563;/**
564; * Executes VMXON.
565; *
566; * @returns VBox status code.
567; * @param HCPhysVMXOn Physical address of VMXON structure.
568; */
569;DECLASM(int) VMXEnable(RTHCPHYS HCPhysVMXOn);
570BEGINPROC VMXEnable
571%ifdef RT_ARCH_AMD64
572 xor rax, rax
573 %ifdef ASM_CALL64_GCC
574 push rdi
575 %else
576 push rcx
577 %endif
578 vmxon [rsp]
579%else ; RT_ARCH_X86
580 xor eax, eax
581 vmxon [esp + 4]
582%endif ; RT_ARCH_X86
583 jnc .good
584 mov eax, VERR_VMX_INVALID_VMXON_PTR
585 jmp .the_end
586
587.good:
588 jnz .the_end
589 mov eax, VERR_VMX_VMXON_FAILED
590
591.the_end:
592%ifdef RT_ARCH_AMD64
593 add rsp, 8
594%endif
595 ret
596ENDPROC VMXEnable
597
598
599;/**
600; * Executes VMXOFF.
601; */
602;DECLASM(void) VMXDisable(void);
603BEGINPROC VMXDisable
604 vmxoff
605.the_end:
606 ret
607ENDPROC VMXDisable
608
609
610;/**
611; * Executes VMCLEAR.
612; *
613; * @returns VBox status code.
614; * @param HCPhysVmcs Physical address of VM control structure.
615; */
616;DECLASM(int) VMXClearVmcs(RTHCPHYS HCPhysVmcs);
617ALIGNCODE(16)
618BEGINPROC VMXClearVmcs
619%ifdef RT_ARCH_AMD64
620 xor rax, rax
621 %ifdef ASM_CALL64_GCC
622 push rdi
623 %else
624 push rcx
625 %endif
626 vmclear [rsp]
627%else ; RT_ARCH_X86
628 xor eax, eax
629 vmclear [esp + 4]
630%endif ; RT_ARCH_X86
631 jnc .the_end
632 mov eax, VERR_VMX_INVALID_VMCS_PTR
633.the_end:
634%ifdef RT_ARCH_AMD64
635 add rsp, 8
636%endif
637 ret
638ENDPROC VMXClearVmcs
639
640
641;/**
642; * Executes VMPTRLD.
643; *
644; * @returns VBox status code.
645; * @param HCPhysVmcs Physical address of VMCS structure.
646; */
647;DECLASM(int) VMXActivateVmcs(RTHCPHYS HCPhysVmcs);
648ALIGNCODE(16)
649BEGINPROC VMXActivateVmcs
650%ifdef RT_ARCH_AMD64
651 xor rax, rax
652 %ifdef ASM_CALL64_GCC
653 push rdi
654 %else
655 push rcx
656 %endif
657 vmptrld [rsp]
658%else
659 xor eax, eax
660 vmptrld [esp + 4]
661%endif
662 jnc .the_end
663 mov eax, VERR_VMX_INVALID_VMCS_PTR
664.the_end:
665%ifdef RT_ARCH_AMD64
666 add rsp, 8
667%endif
668 ret
669ENDPROC VMXActivateVmcs
670
671
672;/**
673; * Executes VMPTRST.
674; *
675; * @returns VBox status code.
676; * @param [esp + 04h] gcc:rdi msc:rcx Param 1 - First parameter - Address that will receive the current pointer.
677; */
678;DECLASM(int) VMXGetActivatedVmcs(RTHCPHYS *pVMCS);
679BEGINPROC VMXGetActivatedVmcs
680%ifdef RT_OS_OS2
681 mov eax, VERR_NOT_SUPPORTED
682 ret
683%else
684 %ifdef RT_ARCH_AMD64
685 %ifdef ASM_CALL64_GCC
686 vmptrst qword [rdi]
687 %else
688 vmptrst qword [rcx]
689 %endif
690 %else
691 vmptrst qword [esp+04h]
692 %endif
693 xor eax, eax
694.the_end:
695 ret
696%endif
697ENDPROC VMXGetActivatedVmcs
698
699;/**
700; * Invalidate a page using INVEPT.
701; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush.
702; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer.
703; */
704;DECLASM(int) VMXR0InvEPT(VMX_FLUSH enmFlush, uint64_t *pDescriptor);
705BEGINPROC VMXR0InvEPT
706%ifdef RT_ARCH_AMD64
707 %ifdef ASM_CALL64_GCC
708 and edi, 0ffffffffh
709 xor rax, rax
710; invept rdi, qword [rsi]
711 DB 0x66, 0x0F, 0x38, 0x80, 0x3E
712 %else
713 and ecx, 0ffffffffh
714 xor rax, rax
715; invept rcx, qword [rdx]
716 DB 0x66, 0x0F, 0x38, 0x80, 0xA
717 %endif
718%else
719 mov ecx, [esp + 4]
720 mov edx, [esp + 8]
721 xor eax, eax
722; invept ecx, qword [edx]
723 DB 0x66, 0x0F, 0x38, 0x80, 0xA
724%endif
725 jnc .valid_vmcs
726 mov eax, VERR_VMX_INVALID_VMCS_PTR
727 ret
728.valid_vmcs:
729 jnz .the_end
730 mov eax, VERR_INVALID_PARAMETER
731.the_end:
732 ret
733ENDPROC VMXR0InvEPT
734
735
736;/**
737; * Invalidate a page using invvpid
738; @param enmFlush msc:ecx gcc:edi x86:[esp+04] Type of flush
739; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer
740; */
741;DECLASM(int) VMXR0InvVPID(VMX_FLUSH enmFlush, uint64_t *pDescriptor);
742BEGINPROC VMXR0InvVPID
743%ifdef RT_ARCH_AMD64
744 %ifdef ASM_CALL64_GCC
745 and edi, 0ffffffffh
746 xor rax, rax
747; invvpid rdi, qword [rsi]
748 DB 0x66, 0x0F, 0x38, 0x81, 0x3E
749 %else
750 and ecx, 0ffffffffh
751 xor rax, rax
752; invvpid rcx, qword [rdx]
753 DB 0x66, 0x0F, 0x38, 0x81, 0xA
754 %endif
755%else
756 mov ecx, [esp + 4]
757 mov edx, [esp + 8]
758 xor eax, eax
759; invvpid ecx, qword [edx]
760 DB 0x66, 0x0F, 0x38, 0x81, 0xA
761%endif
762 jnc .valid_vmcs
763 mov eax, VERR_VMX_INVALID_VMCS_PTR
764 ret
765.valid_vmcs:
766 jnz .the_end
767 mov eax, VERR_INVALID_PARAMETER
768.the_end:
769 ret
770ENDPROC VMXR0InvVPID
771
772
773%if GC_ARCH_BITS == 64
774;;
775; Executes INVLPGA
776;
777; @param pPageGC msc:rcx gcc:rdi x86:[esp+04] Virtual page to invalidate
778; @param uASID msc:rdx gcc:rsi x86:[esp+0C] Tagged TLB id
779;
780;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
781BEGINPROC SVMR0InvlpgA
782%ifdef RT_ARCH_AMD64
783 %ifdef ASM_CALL64_GCC
784 mov rax, rdi
785 mov rcx, rsi
786 %else
787 mov rax, rcx
788 mov rcx, rdx
789 %endif
790%else
791 mov eax, [esp + 4]
792 mov ecx, [esp + 0Ch]
793%endif
794 invlpga [xAX], ecx
795 ret
796ENDPROC SVMR0InvlpgA
797
798%else ; GC_ARCH_BITS != 64
799;;
800; Executes INVLPGA
801;
802; @param pPageGC msc:ecx gcc:edi x86:[esp+04] Virtual page to invalidate
803; @param uASID msc:edx gcc:esi x86:[esp+08] Tagged TLB id
804;
805;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID);
806BEGINPROC SVMR0InvlpgA
807%ifdef RT_ARCH_AMD64
808 %ifdef ASM_CALL64_GCC
809 movzx rax, edi
810 mov ecx, esi
811 %else
812 ; from http://www.cs.cmu.edu/~fp/courses/15213-s06/misc/asm64-handout.pdf:
813 ; ``Perhaps unexpectedly, instructions that move or generate 32-bit register
814 ; values also set the upper 32 bits of the register to zero. Consequently
815 ; there is no need for an instruction movzlq.''
816 mov eax, ecx
817 mov ecx, edx
818 %endif
819%else
820 mov eax, [esp + 4]
821 mov ecx, [esp + 8]
822%endif
823 invlpga [xAX], ecx
824 ret
825ENDPROC SVMR0InvlpgA
826
827%endif ; GC_ARCH_BITS != 64
828
829
830%ifdef VBOX_WITH_KERNEL_USING_XMM
831
832;;
833; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
834; load the guest ones when necessary.
835;
836; @cproto DECLASM(int) HMR0VMXStartVMhmR0DumpDescriptorM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM,
837; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
838;
839; @returns eax
840;
841; @param fResumeVM msc:rcx
842; @param pCtx msc:rdx
843; @param pVMCSCache msc:r8
844; @param pVM msc:r9
845; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
846; @param pfnStartVM msc:[rbp+38h]
847;
848; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
849;
850; @remarks Drivers shouldn't use AVX registers without saving+loading:
851; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
852; However the compiler docs have different idea:
853; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
854; We'll go with the former for now.
855;
856; ASSUMING 64-bit and windows for now.
857;
858ALIGNCODE(16)
859BEGINPROC hmR0VMXStartVMWrapXMM
860 push xBP
861 mov xBP, xSP
862 sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
863
864 ; spill input parameters.
865 mov [xBP + 010h], rcx ; fResumeVM
866 mov [xBP + 018h], rdx ; pCtx
867 mov [xBP + 020h], r8 ; pVMCSCache
868 mov [xBP + 028h], r9 ; pVM
869
870 ; Ask CPUM whether we've started using the FPU yet.
871 mov rcx, [xBP + 30h] ; pVCpu
872 call NAME(CPUMIsGuestFPUStateActive)
873 test al, al
874 jnz .guest_fpu_state_active
875
876 ; No need to mess with XMM registers just call the start routine and return.
877 mov r11, [xBP + 38h] ; pfnStartVM
878 mov r10, [xBP + 30h] ; pVCpu
879 mov [xSP + 020h], r10
880 mov rcx, [xBP + 010h] ; fResumeVM
881 mov rdx, [xBP + 018h] ; pCtx
882 mov r8, [xBP + 020h] ; pVMCSCache
883 mov r9, [xBP + 028h] ; pVM
884 call r11
885
886 leave
887 ret
888
889ALIGNCODE(8)
890.guest_fpu_state_active:
891 ; Save the non-volatile host XMM registers.
892 movdqa [rsp + 040h + 000h], xmm6
893 movdqa [rsp + 040h + 010h], xmm7
894 movdqa [rsp + 040h + 020h], xmm8
895 movdqa [rsp + 040h + 030h], xmm9
896 movdqa [rsp + 040h + 040h], xmm10
897 movdqa [rsp + 040h + 050h], xmm11
898 movdqa [rsp + 040h + 060h], xmm12
899 movdqa [rsp + 040h + 070h], xmm13
900 movdqa [rsp + 040h + 080h], xmm14
901 movdqa [rsp + 040h + 090h], xmm15
902 stmxcsr [rsp + 040h + 0a0h]
903
904 mov r10, [xBP + 018h] ; pCtx
905 mov eax, [r10 + CPUMCTX.fXStateMask]
906 test eax, eax
907 jz .guest_fpu_state_manually
908
909 ;
910 ; Using XSAVE to load the guest XMM, YMM and ZMM registers.
911 ;
912 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
913 xor edx, edx
914 mov r10, [r10 + CPUMCTX.pXStateR0]
915 xrstor [r10]
916
917 ; Make the call (same as in the other case ).
918 mov r11, [xBP + 38h] ; pfnStartVM
919 mov r10, [xBP + 30h] ; pVCpu
920 mov [xSP + 020h], r10
921 mov rcx, [xBP + 010h] ; fResumeVM
922 mov rdx, [xBP + 018h] ; pCtx
923 mov r8, [xBP + 020h] ; pVMCSCache
924 mov r9, [xBP + 028h] ; pVM
925 call r11
926
927 mov r11d, eax ; save return value (xsave below uses eax)
928
929 ; Save the guest XMM registers.
930 mov r10, [xBP + 018h] ; pCtx
931 mov eax, [r10 + CPUMCTX.fXStateMask]
932 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
933 xor edx, edx
934 mov r10, [r10 + CPUMCTX.pXStateR0]
935 xsave [r10]
936
937 mov eax, r11d ; restore return value.
938
939.restore_non_volatile_host_xmm_regs:
940 ; Load the non-volatile host XMM registers.
941 movdqa xmm6, [rsp + 040h + 000h]
942 movdqa xmm7, [rsp + 040h + 010h]
943 movdqa xmm8, [rsp + 040h + 020h]
944 movdqa xmm9, [rsp + 040h + 030h]
945 movdqa xmm10, [rsp + 040h + 040h]
946 movdqa xmm11, [rsp + 040h + 050h]
947 movdqa xmm12, [rsp + 040h + 060h]
948 movdqa xmm13, [rsp + 040h + 070h]
949 movdqa xmm14, [rsp + 040h + 080h]
950 movdqa xmm15, [rsp + 040h + 090h]
951 ldmxcsr [rsp + 040h + 0a0h]
952 leave
953 ret
954
955 ;
956 ; No XSAVE, load and save the guest XMM registers manually.
957 ;
958.guest_fpu_state_manually:
959 ; Load the full guest XMM register state.
960 mov r10, [r10 + CPUMCTX.pXStateR0]
961 movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
962 movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
963 movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
964 movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
965 movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
966 movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
967 movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
968 movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
969 movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
970 movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
971 movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
972 movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
973 movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
974 movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
975 movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
976 movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
977 ldmxcsr [r10 + X86FXSTATE.MXCSR]
978
979 ; Make the call (same as in the other case ).
980 mov r11, [xBP + 38h] ; pfnStartVM
981 mov r10, [xBP + 30h] ; pVCpu
982 mov [xSP + 020h], r10
983 mov rcx, [xBP + 010h] ; fResumeVM
984 mov rdx, [xBP + 018h] ; pCtx
985 mov r8, [xBP + 020h] ; pVMCSCache
986 mov r9, [xBP + 028h] ; pVM
987 call r11
988
989 ; Save the guest XMM registers.
990 mov r10, [xBP + 018h] ; pCtx
991 mov r10, [r10 + CPUMCTX.pXStateR0]
992 stmxcsr [r10 + X86FXSTATE.MXCSR]
993 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
994 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
995 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
996 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
997 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
998 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
999 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1000 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1001 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1002 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1003 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1004 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1005 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1006 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1007 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1008 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1009 jmp .restore_non_volatile_host_xmm_regs
1010ENDPROC hmR0VMXStartVMWrapXMM
1011
1012;;
1013; Wrapper around svm.pfnVMRun that preserves host XMM registers and
1014; load the guest ones when necessary.
1015;
1016; @cproto DECLASM(int) hmR0SVMRunWrapXMM(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu,
1017; PFNHMSVMVMRUN pfnVMRun);
1018;
1019; @returns eax
1020;
1021; @param HCPhysVmcbHost msc:rcx
1022; @param HCPhysVmcb msc:rdx
1023; @param pCtx msc:r8
1024; @param pVM msc:r9
1025; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
1026; @param pfnVMRun msc:[rbp+38h]
1027;
1028; @remarks This is essentially the same code as hmR0VMXStartVMWrapXMM, only the parameters differ a little bit.
1029;
1030; @remarks Drivers shouldn't use AVX registers without saving+loading:
1031; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
1032; However the compiler docs have different idea:
1033; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
1034; We'll go with the former for now.
1035;
1036; ASSUMING 64-bit and windows for now.
1037ALIGNCODE(16)
1038BEGINPROC hmR0SVMRunWrapXMM
1039 push xBP
1040 mov xBP, xSP
1041 sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
1042
1043 ; spill input parameters.
1044 mov [xBP + 010h], rcx ; HCPhysVmcbHost
1045 mov [xBP + 018h], rdx ; HCPhysVmcb
1046 mov [xBP + 020h], r8 ; pCtx
1047 mov [xBP + 028h], r9 ; pVM
1048
1049 ; Ask CPUM whether we've started using the FPU yet.
1050 mov rcx, [xBP + 30h] ; pVCpu
1051 call NAME(CPUMIsGuestFPUStateActive)
1052 test al, al
1053 jnz .guest_fpu_state_active
1054
1055 ; No need to mess with XMM registers just call the start routine and return.
1056 mov r11, [xBP + 38h] ; pfnVMRun
1057 mov r10, [xBP + 30h] ; pVCpu
1058 mov [xSP + 020h], r10
1059 mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1060 mov rdx, [xBP + 018h] ; HCPhysVmcb
1061 mov r8, [xBP + 020h] ; pCtx
1062 mov r9, [xBP + 028h] ; pVM
1063 call r11
1064
1065 leave
1066 ret
1067
1068ALIGNCODE(8)
1069.guest_fpu_state_active:
1070 ; Save the non-volatile host XMM registers.
1071 movdqa [rsp + 040h + 000h], xmm6
1072 movdqa [rsp + 040h + 010h], xmm7
1073 movdqa [rsp + 040h + 020h], xmm8
1074 movdqa [rsp + 040h + 030h], xmm9
1075 movdqa [rsp + 040h + 040h], xmm10
1076 movdqa [rsp + 040h + 050h], xmm11
1077 movdqa [rsp + 040h + 060h], xmm12
1078 movdqa [rsp + 040h + 070h], xmm13
1079 movdqa [rsp + 040h + 080h], xmm14
1080 movdqa [rsp + 040h + 090h], xmm15
1081 stmxcsr [rsp + 040h + 0a0h]
1082
1083 mov r10, [xBP + 020h] ; pCtx
1084 mov eax, [r10 + CPUMCTX.fXStateMask]
1085 test eax, eax
1086 jz .guest_fpu_state_manually
1087
1088 ;
1089 ; Using XSAVE.
1090 ;
1091 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1092 xor edx, edx
1093 mov r10, [r10 + CPUMCTX.pXStateR0]
1094 xrstor [r10]
1095
1096 ; Make the call (same as in the other case ).
1097 mov r11, [xBP + 38h] ; pfnVMRun
1098 mov r10, [xBP + 30h] ; pVCpu
1099 mov [xSP + 020h], r10
1100 mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1101 mov rdx, [xBP + 018h] ; HCPhysVmcb
1102 mov r8, [xBP + 020h] ; pCtx
1103 mov r9, [xBP + 028h] ; pVM
1104 call r11
1105
1106 mov r11d, eax ; save return value (xsave below uses eax)
1107
1108 ; Save the guest XMM registers.
1109 mov r10, [xBP + 020h] ; pCtx
1110 mov eax, [r10 + CPUMCTX.fXStateMask]
1111 and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
1112 xor edx, edx
1113 mov r10, [r10 + CPUMCTX.pXStateR0]
1114 xsave [r10]
1115
1116 mov eax, r11d ; restore return value.
1117
1118.restore_non_volatile_host_xmm_regs:
1119 ; Load the non-volatile host XMM registers.
1120 movdqa xmm6, [rsp + 040h + 000h]
1121 movdqa xmm7, [rsp + 040h + 010h]
1122 movdqa xmm8, [rsp + 040h + 020h]
1123 movdqa xmm9, [rsp + 040h + 030h]
1124 movdqa xmm10, [rsp + 040h + 040h]
1125 movdqa xmm11, [rsp + 040h + 050h]
1126 movdqa xmm12, [rsp + 040h + 060h]
1127 movdqa xmm13, [rsp + 040h + 070h]
1128 movdqa xmm14, [rsp + 040h + 080h]
1129 movdqa xmm15, [rsp + 040h + 090h]
1130 ldmxcsr [rsp + 040h + 0a0h]
1131 leave
1132 ret
1133
1134 ;
1135 ; No XSAVE, load and save the guest XMM registers manually.
1136 ;
1137.guest_fpu_state_manually:
1138 ; Load the full guest XMM register state.
1139 mov r10, [r10 + CPUMCTX.pXStateR0]
1140 movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
1141 movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
1142 movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
1143 movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
1144 movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
1145 movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
1146 movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
1147 movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
1148 movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
1149 movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
1150 movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
1151 movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
1152 movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
1153 movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
1154 movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
1155 movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
1156 ldmxcsr [r10 + X86FXSTATE.MXCSR]
1157
1158 ; Make the call (same as in the other case ).
1159 mov r11, [xBP + 38h] ; pfnVMRun
1160 mov r10, [xBP + 30h] ; pVCpu
1161 mov [xSP + 020h], r10
1162 mov rcx, [xBP + 010h] ; HCPhysVmcbHost
1163 mov rdx, [xBP + 018h] ; HCPhysVmcb
1164 mov r8, [xBP + 020h] ; pCtx
1165 mov r9, [xBP + 028h] ; pVM
1166 call r11
1167
1168 ; Save the guest XMM registers.
1169 mov r10, [xBP + 020h] ; pCtx
1170 mov r10, [r10 + CPUMCTX.pXStateR0]
1171 stmxcsr [r10 + X86FXSTATE.MXCSR]
1172 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
1173 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
1174 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
1175 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
1176 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
1177 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
1178 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
1179 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
1180 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
1181 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
1182 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
1183 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
1184 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
1185 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
1186 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
1187 movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
1188 jmp .restore_non_volatile_host_xmm_regs
1189ENDPROC hmR0SVMRunWrapXMM
1190
1191%endif ; VBOX_WITH_KERNEL_USING_XMM
1192
1193
1194;; @def RESTORE_STATE_VM32
1195; Macro restoring essential host state and updating guest state
1196; for common host, 32-bit guest for VT-x.
1197%macro RESTORE_STATE_VM32 0
1198 ; Restore base and limit of the IDTR & GDTR.
1199 %ifndef VMX_SKIP_IDTR
1200 lidt [xSP]
1201 add xSP, xCB * 2
1202 %endif
1203 %ifndef VMX_SKIP_GDTR
1204 lgdt [xSP]
1205 add xSP, xCB * 2
1206 %endif
1207
1208 push xDI
1209 %ifndef VMX_SKIP_TR
1210 mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR).
1211 %else
1212 mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR).
1213 %endif
1214
1215 mov [ss:xDI + CPUMCTX.eax], eax
1216 mov xAX, SPECTRE_FILLER
1217 mov [ss:xDI + CPUMCTX.ebx], ebx
1218 mov xBX, xAX
1219 mov [ss:xDI + CPUMCTX.ecx], ecx
1220 mov xCX, xAX
1221 mov [ss:xDI + CPUMCTX.edx], edx
1222 mov xDX, xAX
1223 mov [ss:xDI + CPUMCTX.esi], esi
1224 mov xSI, xAX
1225 mov [ss:xDI + CPUMCTX.ebp], ebp
1226 mov xBP, xAX
1227 mov xAX, cr2
1228 mov [ss:xDI + CPUMCTX.cr2], xAX
1229
1230 %ifdef RT_ARCH_AMD64
1231 pop xAX ; The guest edi we pushed above.
1232 mov dword [ss:xDI + CPUMCTX.edi], eax
1233 %else
1234 pop dword [ss:xDI + CPUMCTX.edi] ; The guest edi we pushed above.
1235 %endif
1236
1237 ; Fight spectre.
1238 INDIRECT_BRANCH_PREDICTION_BARRIER ss:xDI, CPUMCTX_WSF_IBPB_EXIT
1239
1240 %ifndef VMX_SKIP_TR
1241 ; Restore TSS selector; must mark it as not busy before using ltr (!)
1242 ; ASSUME that this is supposed to be 'BUSY'. (saves 20-30 ticks on the T42p)
1243 ; @todo get rid of sgdt
1244 pop xBX ; Saved TR
1245 sub xSP, xCB * 2
1246 sgdt [xSP]
1247 mov xAX, xBX
1248 and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
1249 add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset.
1250 and dword [ss:xAX + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
1251 ltr bx
1252 add xSP, xCB * 2
1253 %endif
1254
1255 pop xAX ; Saved LDTR
1256 %ifdef RT_ARCH_AMD64
1257 cmp eax, 0
1258 je %%skip_ldt_write32
1259 %endif
1260 lldt ax
1261
1262%%skip_ldt_write32:
1263 add xSP, xCB ; pCtx
1264
1265 %ifdef VMX_USE_CACHED_VMCS_ACCESSES
1266 pop xDX ; Saved pCache
1267
1268 ; Note! If we get here as a result of invalid VMCS pointer, all the following
1269 ; vmread's will fail (only eflags.cf=1 will be set) but that shouldn't cause any
1270 ; trouble only just less efficient.
1271 mov ecx, [ss:xDX + VMCSCACHE.Read.cValidEntries]
1272 cmp ecx, 0 ; Can't happen
1273 je %%no_cached_read32
1274 jmp %%cached_read32
1275
1276ALIGN(16)
1277%%cached_read32:
1278 dec xCX
1279 mov eax, [ss:xDX + VMCSCACHE.Read.aField + xCX * 4]
1280 ; Note! This leaves the high 32 bits of the cache entry unmodified!!
1281 vmread [ss:xDX + VMCSCACHE.Read.aFieldVal + xCX * 8], xAX
1282 cmp xCX, 0
1283 jnz %%cached_read32
1284%%no_cached_read32:
1285 %endif
1286
1287 ; Restore segment registers.
1288 MYPOPSEGS xAX, ax
1289
1290 ; Restore the host XCR0 if necessary.
1291 pop xCX
1292 test ecx, ecx
1293 jnz %%xcr0_after_skip
1294 pop xAX
1295 pop xDX
1296 xsetbv ; ecx is already zero.
1297%%xcr0_after_skip:
1298
1299 ; Restore general purpose registers.
1300 MYPOPAD
1301%endmacro
1302
1303
1304;;
1305; Prepares for and executes VMLAUNCH/VMRESUME (32 bits guest mode)
1306;
1307; @returns VBox status code
1308; @param fResume x86:[ebp+8], msc:rcx,gcc:rdi Whether to use vmlauch/vmresume.
1309; @param pCtx x86:[ebp+c], msc:rdx,gcc:rsi Pointer to the guest-CPU context.
1310; @param pCache x86:[ebp+10],msc:r8, gcc:rdx Pointer to the VMCS cache.
1311; @param pVM x86:[ebp+14],msc:r9, gcc:rcx The cross context VM structure.
1312; @param pVCpu x86:[ebp+18],msc:[ebp+30],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1313;
1314ALIGNCODE(16)
1315BEGINPROC VMXR0StartVM32
1316 push xBP
1317 mov xBP, xSP
1318
1319 pushf
1320 cli
1321
1322 ;
1323 ; Save all general purpose host registers.
1324 ;
1325 MYPUSHAD
1326
1327 ;
1328 ; First we have to write some final guest CPU context registers.
1329 ;
1330 mov eax, VMX_VMCS_HOST_RIP
1331%ifdef RT_ARCH_AMD64
1332 lea r10, [.vmlaunch_done wrt rip]
1333 vmwrite rax, r10
1334%else
1335 mov ecx, .vmlaunch_done
1336 vmwrite eax, ecx
1337%endif
1338 ; Note: assumes success!
1339
1340 ;
1341 ; Unify input parameter registers.
1342 ;
1343%ifdef RT_ARCH_AMD64
1344 %ifdef ASM_CALL64_GCC
1345 ; fResume already in rdi
1346 ; pCtx already in rsi
1347 mov rbx, rdx ; pCache
1348 %else
1349 mov rdi, rcx ; fResume
1350 mov rsi, rdx ; pCtx
1351 mov rbx, r8 ; pCache
1352 %endif
1353%else
1354 mov edi, [ebp + 8] ; fResume
1355 mov esi, [ebp + 12] ; pCtx
1356 mov ebx, [ebp + 16] ; pCache
1357%endif
1358
1359 ;
1360 ; Save the host XCR0 and load the guest one if necessary.
1361 ; Note! Trashes rdx and rcx.
1362 ;
1363%ifdef ASM_CALL64_MSC
1364 mov rax, [xBP + 30h] ; pVCpu
1365%elifdef ASM_CALL64_GCC
1366 mov rax, r8 ; pVCpu
1367%else
1368 mov eax, [xBP + 18h] ; pVCpu
1369%endif
1370 test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1371 jz .xcr0_before_skip
1372
1373 xor ecx, ecx
1374 xgetbv ; Save the host one on the stack.
1375 push xDX
1376 push xAX
1377
1378 mov eax, [xSI + CPUMCTX.aXcr] ; Load the guest one.
1379 mov edx, [xSI + CPUMCTX.aXcr + 4]
1380 xor ecx, ecx ; paranoia
1381 xsetbv
1382
1383 push 0 ; Indicate that we must restore XCR0 (popped into ecx, thus 0).
1384 jmp .xcr0_before_done
1385
1386.xcr0_before_skip:
1387 push 3fh ; indicate that we need not.
1388.xcr0_before_done:
1389
1390 ;
1391 ; Save segment registers.
1392 ; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
1393 ;
1394 MYPUSHSEGS xAX, ax
1395
1396%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1397 mov ecx, [xBX + VMCSCACHE.Write.cValidEntries]
1398 cmp ecx, 0
1399 je .no_cached_writes
1400 mov edx, ecx
1401 mov ecx, 0
1402 jmp .cached_write
1403
1404ALIGN(16)
1405.cached_write:
1406 mov eax, [xBX + VMCSCACHE.Write.aField + xCX * 4]
1407 vmwrite xAX, [xBX + VMCSCACHE.Write.aFieldVal + xCX * 8]
1408 inc xCX
1409 cmp xCX, xDX
1410 jl .cached_write
1411
1412 mov dword [xBX + VMCSCACHE.Write.cValidEntries], 0
1413.no_cached_writes:
1414
1415 ; Save the pCache pointer.
1416 push xBX
1417%endif
1418
1419 ; Save the pCtx pointer.
1420 push xSI
1421
1422 ; Save host LDTR.
1423 xor eax, eax
1424 sldt ax
1425 push xAX
1426
1427%ifndef VMX_SKIP_TR
1428 ; The host TR limit is reset to 0x67; save & restore it manually.
1429 str eax
1430 push xAX
1431%endif
1432
1433%ifndef VMX_SKIP_GDTR
1434 ; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
1435 sub xSP, xCB * 2
1436 sgdt [xSP]
1437%endif
1438%ifndef VMX_SKIP_IDTR
1439 sub xSP, xCB * 2
1440 sidt [xSP]
1441%endif
1442
1443 ; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
1444 mov xBX, [xSI + CPUMCTX.cr2]
1445 mov xDX, cr2
1446 cmp xBX, xDX
1447 je .skip_cr2_write32
1448 mov cr2, xBX
1449
1450.skip_cr2_write32:
1451 mov eax, VMX_VMCS_HOST_RSP
1452 vmwrite xAX, xSP
1453 ; Note: assumes success!
1454 ; Don't mess with ESP anymore!!!
1455
1456 ; Fight spectre.
1457 INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
1458
1459 ; Load guest general purpose registers.
1460 mov eax, [xSI + CPUMCTX.eax]
1461 mov ebx, [xSI + CPUMCTX.ebx]
1462 mov ecx, [xSI + CPUMCTX.ecx]
1463 mov edx, [xSI + CPUMCTX.edx]
1464 mov ebp, [xSI + CPUMCTX.ebp]
1465
1466 ; Resume or start VM?
1467 cmp xDI, 0 ; fResume
1468
1469 ; Load guest edi & esi.
1470 mov edi, [xSI + CPUMCTX.edi]
1471 mov esi, [xSI + CPUMCTX.esi]
1472
1473 je .vmlaunch_launch
1474
1475 vmresume
1476 jc near .vmxstart_invalid_vmcs_ptr
1477 jz near .vmxstart_start_failed
1478 jmp .vmlaunch_done; ; Here if vmresume detected a failure.
1479
1480.vmlaunch_launch:
1481 vmlaunch
1482 jc near .vmxstart_invalid_vmcs_ptr
1483 jz near .vmxstart_start_failed
1484 jmp .vmlaunch_done; ; Here if vmlaunch detected a failure.
1485
1486ALIGNCODE(16) ;; @todo YASM BUG - this alignment is wrong on darwin, it's 1 byte off.
1487.vmlaunch_done:
1488 RESTORE_STATE_VM32
1489 mov eax, VINF_SUCCESS
1490
1491.vmstart_end:
1492 popf
1493 pop xBP
1494 ret
1495
1496.vmxstart_invalid_vmcs_ptr:
1497 RESTORE_STATE_VM32
1498 mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1499 jmp .vmstart_end
1500
1501.vmxstart_start_failed:
1502 RESTORE_STATE_VM32
1503 mov eax, VERR_VMX_UNABLE_TO_START_VM
1504 jmp .vmstart_end
1505
1506ENDPROC VMXR0StartVM32
1507
1508
1509%ifdef RT_ARCH_AMD64
1510;; @def RESTORE_STATE_VM64
1511; Macro restoring essential host state and updating guest state
1512; for 64-bit host, 64-bit guest for VT-x.
1513;
1514%macro RESTORE_STATE_VM64 0
1515 ; Restore base and limit of the IDTR & GDTR
1516 %ifndef VMX_SKIP_IDTR
1517 lidt [xSP]
1518 add xSP, xCB * 2
1519 %endif
1520 %ifndef VMX_SKIP_GDTR
1521 lgdt [xSP]
1522 add xSP, xCB * 2
1523 %endif
1524
1525 push xDI
1526 %ifndef VMX_SKIP_TR
1527 mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR)
1528 %else
1529 mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR)
1530 %endif
1531
1532 mov qword [xDI + CPUMCTX.eax], rax
1533 mov rax, SPECTRE_FILLER64
1534 mov qword [xDI + CPUMCTX.ebx], rbx
1535 mov rbx, rax
1536 mov qword [xDI + CPUMCTX.ecx], rcx
1537 mov rcx, rax
1538 mov qword [xDI + CPUMCTX.edx], rdx
1539 mov rdx, rax
1540 mov qword [xDI + CPUMCTX.esi], rsi
1541 mov rsi, rax
1542 mov qword [xDI + CPUMCTX.ebp], rbp
1543 mov rbp, rax
1544 mov qword [xDI + CPUMCTX.r8], r8
1545 mov r8, rax
1546 mov qword [xDI + CPUMCTX.r9], r9
1547 mov r9, rax
1548 mov qword [xDI + CPUMCTX.r10], r10
1549 mov r10, rax
1550 mov qword [xDI + CPUMCTX.r11], r11
1551 mov r11, rax
1552 mov qword [xDI + CPUMCTX.r12], r12
1553 mov r12, rax
1554 mov qword [xDI + CPUMCTX.r13], r13
1555 mov r13, rax
1556 mov qword [xDI + CPUMCTX.r14], r14
1557 mov r14, rax
1558 mov qword [xDI + CPUMCTX.r15], r15
1559 mov r15, rax
1560 mov rax, cr2
1561 mov qword [xDI + CPUMCTX.cr2], rax
1562
1563 pop xAX ; The guest rdi we pushed above
1564 mov qword [xDI + CPUMCTX.edi], rax
1565
1566 ; Fight spectre.
1567 INDIRECT_BRANCH_PREDICTION_BARRIER xDI, CPUMCTX_WSF_IBPB_EXIT
1568
1569 %ifndef VMX_SKIP_TR
1570 ; Restore TSS selector; must mark it as not busy before using ltr (!)
1571 ; ASSUME that this is supposed to be 'BUSY'. (saves 20-30 ticks on the T42p).
1572 ; @todo get rid of sgdt
1573 pop xBX ; Saved TR
1574 sub xSP, xCB * 2
1575 sgdt [xSP]
1576 mov xAX, xBX
1577 and eax, X86_SEL_MASK_OFF_RPL ; Mask away TI and RPL bits leaving only the descriptor offset.
1578 add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset.
1579 and dword [xAX + 4], ~RT_BIT(9) ; Clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit).
1580 ltr bx
1581 add xSP, xCB * 2
1582 %endif
1583
1584 pop xAX ; Saved LDTR
1585 cmp eax, 0
1586 je %%skip_ldt_write64
1587 lldt ax
1588
1589%%skip_ldt_write64:
1590 pop xSI ; pCtx (needed in rsi by the macros below)
1591
1592 %ifdef VMX_USE_CACHED_VMCS_ACCESSES
1593 pop xDX ; Saved pCache
1594
1595 ; Note! If we get here as a result of invalid VMCS pointer, all the following
1596 ; vmread's will fail (only eflags.cf=1 will be set) but that shouldn't cause any
1597 ; trouble only just less efficient.
1598 mov ecx, [xDX + VMCSCACHE.Read.cValidEntries]
1599 cmp ecx, 0 ; Can't happen
1600 je %%no_cached_read64
1601 jmp %%cached_read64
1602
1603ALIGN(16)
1604%%cached_read64:
1605 dec xCX
1606 mov eax, [xDX + VMCSCACHE.Read.aField + xCX * 4]
1607 vmread [xDX + VMCSCACHE.Read.aFieldVal + xCX * 8], xAX
1608 cmp xCX, 0
1609 jnz %%cached_read64
1610%%no_cached_read64:
1611 %endif
1612
1613 ; Restore segment registers.
1614 MYPOPSEGS xAX, ax
1615
1616 ; Restore the host XCR0 if necessary.
1617 pop xCX
1618 test ecx, ecx
1619 jnz %%xcr0_after_skip
1620 pop xAX
1621 pop xDX
1622 xsetbv ; ecx is already zero.
1623%%xcr0_after_skip:
1624
1625 ; Restore general purpose registers.
1626 MYPOPAD
1627%endmacro
1628
1629
1630;;
1631; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
1632;
1633; @returns VBox status code
1634; @param fResume msc:rcx, gcc:rdi Whether to use vmlauch/vmresume.
1635; @param pCtx msc:rdx, gcc:rsi Pointer to the guest-CPU context.
1636; @param pCache msc:r8, gcc:rdx Pointer to the VMCS cache.
1637; @param pVM msc:r9, gcc:rcx The cross context VM structure.
1638; @param pVCpu msc:[ebp+30], gcc:r8 The cross context virtual CPU structure of the calling EMT.
1639;
1640ALIGNCODE(16)
1641BEGINPROC VMXR0StartVM64
1642 push xBP
1643 mov xBP, xSP
1644
1645 pushf
1646 cli
1647
1648 ; Save all general purpose host registers.
1649 MYPUSHAD
1650
1651 ; First we have to save some final CPU context registers.
1652 lea r10, [.vmlaunch64_done wrt rip]
1653 mov rax, VMX_VMCS_HOST_RIP ; Return address (too difficult to continue after VMLAUNCH?).
1654 vmwrite rax, r10
1655 ; Note: assumes success!
1656
1657 ;
1658 ; Unify the input parameter registers.
1659 ;
1660%ifdef ASM_CALL64_GCC
1661 ; fResume already in rdi
1662 ; pCtx already in rsi
1663 mov rbx, rdx ; pCache
1664%else
1665 mov rdi, rcx ; fResume
1666 mov rsi, rdx ; pCtx
1667 mov rbx, r8 ; pCache
1668%endif
1669
1670 ;
1671 ; Save the host XCR0 and load the guest one if necessary.
1672 ; Note! Trashes rdx and rcx.
1673 ;
1674%ifdef ASM_CALL64_MSC
1675 mov rax, [xBP + 30h] ; pVCpu
1676%else
1677 mov rax, r8 ; pVCpu
1678%endif
1679 test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1680 jz .xcr0_before_skip
1681
1682 xor ecx, ecx
1683 xgetbv ; Save the host one on the stack.
1684 push xDX
1685 push xAX
1686
1687 mov eax, [xSI + CPUMCTX.aXcr] ; Load the guest one.
1688 mov edx, [xSI + CPUMCTX.aXcr + 4]
1689 xor ecx, ecx ; paranoia
1690 xsetbv
1691
1692 push 0 ; Indicate that we must restore XCR0 (popped into ecx, thus 0).
1693 jmp .xcr0_before_done
1694
1695.xcr0_before_skip:
1696 push 3fh ; indicate that we need not.
1697.xcr0_before_done:
1698
1699 ;
1700 ; Save segment registers.
1701 ; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
1702 ;
1703 MYPUSHSEGS xAX, ax
1704
1705%ifdef VMX_USE_CACHED_VMCS_ACCESSES
1706 mov ecx, [xBX + VMCSCACHE.Write.cValidEntries]
1707 cmp ecx, 0
1708 je .no_cached_writes
1709 mov edx, ecx
1710 mov ecx, 0
1711 jmp .cached_write
1712
1713ALIGN(16)
1714.cached_write:
1715 mov eax, [xBX + VMCSCACHE.Write.aField + xCX * 4]
1716 vmwrite xAX, [xBX + VMCSCACHE.Write.aFieldVal + xCX * 8]
1717 inc xCX
1718 cmp xCX, xDX
1719 jl .cached_write
1720
1721 mov dword [xBX + VMCSCACHE.Write.cValidEntries], 0
1722.no_cached_writes:
1723
1724 ; Save the pCache pointer.
1725 push xBX
1726%endif
1727
1728 ; Save the pCtx pointer.
1729 push xSI
1730
1731 ; Save host LDTR.
1732 xor eax, eax
1733 sldt ax
1734 push xAX
1735
1736%ifndef VMX_SKIP_TR
1737 ; The host TR limit is reset to 0x67; save & restore it manually.
1738 str eax
1739 push xAX
1740%endif
1741
1742%ifndef VMX_SKIP_GDTR
1743 ; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
1744 sub xSP, xCB * 2
1745 sgdt [xSP]
1746%endif
1747%ifndef VMX_SKIP_IDTR
1748 sub xSP, xCB * 2
1749 sidt [xSP]
1750%endif
1751
1752 ; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
1753 mov rbx, qword [xSI + CPUMCTX.cr2]
1754 mov rdx, cr2
1755 cmp rbx, rdx
1756 je .skip_cr2_write
1757 mov cr2, rbx
1758
1759.skip_cr2_write:
1760 mov eax, VMX_VMCS_HOST_RSP
1761 vmwrite xAX, xSP
1762 ; Note: assumes success!
1763 ; Don't mess with ESP anymore!!!
1764
1765 ; Fight spectre.
1766 INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
1767
1768 ; Load guest general purpose registers.
1769 mov rax, qword [xSI + CPUMCTX.eax]
1770 mov rbx, qword [xSI + CPUMCTX.ebx]
1771 mov rcx, qword [xSI + CPUMCTX.ecx]
1772 mov rdx, qword [xSI + CPUMCTX.edx]
1773 mov rbp, qword [xSI + CPUMCTX.ebp]
1774 mov r8, qword [xSI + CPUMCTX.r8]
1775 mov r9, qword [xSI + CPUMCTX.r9]
1776 mov r10, qword [xSI + CPUMCTX.r10]
1777 mov r11, qword [xSI + CPUMCTX.r11]
1778 mov r12, qword [xSI + CPUMCTX.r12]
1779 mov r13, qword [xSI + CPUMCTX.r13]
1780 mov r14, qword [xSI + CPUMCTX.r14]
1781 mov r15, qword [xSI + CPUMCTX.r15]
1782
1783 ; Resume or start VM?
1784 cmp xDI, 0 ; fResume
1785
1786 ; Load guest rdi & rsi.
1787 mov rdi, qword [xSI + CPUMCTX.edi]
1788 mov rsi, qword [xSI + CPUMCTX.esi]
1789
1790 je .vmlaunch64_launch
1791
1792 vmresume
1793 jc near .vmxstart64_invalid_vmcs_ptr
1794 jz near .vmxstart64_start_failed
1795 jmp .vmlaunch64_done; ; Here if vmresume detected a failure.
1796
1797.vmlaunch64_launch:
1798 vmlaunch
1799 jc near .vmxstart64_invalid_vmcs_ptr
1800 jz near .vmxstart64_start_failed
1801 jmp .vmlaunch64_done; ; Here if vmlaunch detected a failure.
1802
1803ALIGNCODE(16)
1804.vmlaunch64_done:
1805 RESTORE_STATE_VM64
1806 mov eax, VINF_SUCCESS
1807
1808.vmstart64_end:
1809 popf
1810 pop xBP
1811 ret
1812
1813.vmxstart64_invalid_vmcs_ptr:
1814 RESTORE_STATE_VM64
1815 mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1816 jmp .vmstart64_end
1817
1818.vmxstart64_start_failed:
1819 RESTORE_STATE_VM64
1820 mov eax, VERR_VMX_UNABLE_TO_START_VM
1821 jmp .vmstart64_end
1822ENDPROC VMXR0StartVM64
1823%endif ; RT_ARCH_AMD64
1824
1825
1826;;
1827; Prepares for and executes VMRUN (32 bits guests)
1828;
1829; @returns VBox status code
1830; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB.
1831; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB.
1832; @param pCtx msc:r8,gcc:rdx Pointer to the guest CPU-context.
1833; @param pVM msc:r9,gcc:rcx The cross context VM structure.
1834; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1835;
1836ALIGNCODE(16)
1837BEGINPROC SVMR0VMRun
1838%ifdef RT_ARCH_AMD64 ; fake a cdecl stack frame
1839 %ifdef ASM_CALL64_GCC
1840 push r8 ; pVCpu
1841 push rcx ; pVM
1842 push rdx ; pCtx
1843 push rsi ; HCPhysVmcb
1844 push rdi ; HCPhysVmcbHost
1845 %else
1846 mov rax, [rsp + 28h]
1847 push rax ; pVCpu
1848 push r9 ; pVM
1849 push r8 ; pCtx
1850 push rdx ; HCPhysVmcb
1851 push rcx ; HCPhysVmcbHost
1852 %endif
1853 push 0
1854%endif
1855 push xBP
1856 mov xBP, xSP
1857 pushf
1858
1859 ; Save all general purpose host registers.
1860 MYPUSHAD
1861
1862 ; Load pCtx into xSI.
1863 mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1864
1865 ; Save the host XCR0 and load the guest one if necessary.
1866 mov xAX, [xBP + xCB * 2 + RTHCPHYS_CB * 2 + xCB * 2] ; pVCpu
1867 test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1868 jz .xcr0_before_skip
1869
1870 xor ecx, ecx
1871 xgetbv ; Save the host XCR0 on the stack
1872 push xDX
1873 push xAX
1874
1875 mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1876 mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0
1877 mov edx, [xSI + CPUMCTX.aXcr + 4]
1878 xor ecx, ecx ; paranoia
1879 xsetbv
1880
1881 push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
1882 jmp .xcr0_before_done
1883
1884.xcr0_before_skip:
1885 push 3fh ; indicate that we need not restore XCR0
1886.xcr0_before_done:
1887
1888 ; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
1889 push xSI
1890
1891 ; Save host fs, gs, sysenter msr etc.
1892 mov xAX, [xBP + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
1893 push xAX ; save for the vmload after vmrun
1894 vmsave
1895
1896 ; Fight spectre.
1897 INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
1898
1899 ; Setup xAX for VMLOAD.
1900 mov xAX, [xBP + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; x86: take low dword only)
1901
1902 ; Load guest general purpose registers.
1903 ; eax is loaded from the VMCB by VMRUN.
1904 mov ebx, [xSI + CPUMCTX.ebx]
1905 mov ecx, [xSI + CPUMCTX.ecx]
1906 mov edx, [xSI + CPUMCTX.edx]
1907 mov edi, [xSI + CPUMCTX.edi]
1908 mov ebp, [xSI + CPUMCTX.ebp]
1909 mov esi, [xSI + CPUMCTX.esi]
1910
1911 ; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1912 clgi
1913 sti
1914
1915 ; Load guest fs, gs, sysenter msr etc.
1916 vmload
1917
1918 ; Run the VM.
1919 vmrun
1920
1921 ; Save guest fs, gs, sysenter msr etc.
1922 vmsave
1923
1924 ; Load host fs, gs, sysenter msr etc.
1925 pop xAX ; load HCPhysVmcbHost (pushed above)
1926 vmload
1927
1928 ; Set the global interrupt flag again, but execute cli to make sure IF=0.
1929 cli
1930 stgi
1931
1932 ; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
1933 pop xAX
1934
1935 mov [ss:xAX + CPUMCTX.ebx], ebx
1936 mov xBX, SPECTRE_FILLER
1937 mov [ss:xAX + CPUMCTX.ecx], ecx
1938 mov xCX, xBX
1939 mov [ss:xAX + CPUMCTX.edx], edx
1940 mov xDX, xBX
1941 mov [ss:xAX + CPUMCTX.esi], esi
1942 mov xSI, xBX
1943 mov [ss:xAX + CPUMCTX.edi], edi
1944 mov xDI, xBX
1945 mov [ss:xAX + CPUMCTX.ebp], ebp
1946 mov xBP, xBX
1947
1948 ; Fight spectre. Note! Trashes xAX!
1949 INDIRECT_BRANCH_PREDICTION_BARRIER ss:xAX, CPUMCTX_WSF_IBPB_EXIT
1950
1951 ; Restore the host xcr0 if necessary.
1952 pop xCX
1953 test ecx, ecx
1954 jnz .xcr0_after_skip
1955 pop xAX
1956 pop xDX
1957 xsetbv ; ecx is already zero
1958.xcr0_after_skip:
1959
1960 ; Restore host general purpose registers.
1961 MYPOPAD
1962
1963 mov eax, VINF_SUCCESS
1964
1965 popf
1966 pop xBP
1967%ifdef RT_ARCH_AMD64
1968 add xSP, 6*xCB
1969%endif
1970 ret
1971ENDPROC SVMR0VMRun
1972
1973
1974%ifdef RT_ARCH_AMD64
1975;;
1976; Prepares for and executes VMRUN (64 bits guests)
1977;
1978; @returns VBox status code
1979; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB.
1980; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB.
1981; @param pCtx msc:r8,gcc:rdx Pointer to the guest-CPU context.
1982; @param pVM msc:r9,gcc:rcx The cross context VM structure.
1983; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1984;
1985ALIGNCODE(16)
1986BEGINPROC SVMR0VMRun64
1987 ; Fake a cdecl stack frame
1988 %ifdef ASM_CALL64_GCC
1989 push r8 ;pVCpu
1990 push rcx ;pVM
1991 push rdx ;pCtx
1992 push rsi ;HCPhysVmcb
1993 push rdi ;HCPhysVmcbHost
1994 %else
1995 mov rax, [rsp + 28h]
1996 push rax ; rbp + 30h pVCpu
1997 push r9 ; rbp + 28h pVM
1998 push r8 ; rbp + 20h pCtx
1999 push rdx ; rbp + 18h HCPhysVmcb
2000 push rcx ; rbp + 10h HCPhysVmcbHost
2001 %endif
2002 push 0 ; rbp + 08h "fake ret addr"
2003 push rbp ; rbp + 00h
2004 mov rbp, rsp
2005 pushf
2006
2007 ; Manual save and restore:
2008 ; - General purpose registers except RIP, RSP, RAX
2009 ;
2010 ; Trashed:
2011 ; - CR2 (we don't care)
2012 ; - LDTR (reset to 0)
2013 ; - DRx (presumably not changed at all)
2014 ; - DR7 (reset to 0x400)
2015
2016 ; Save all general purpose host registers.
2017 MYPUSHAD
2018
2019 ; Load pCtx into xSI.
2020 mov xSI, [rbp + xCB * 2 + RTHCPHYS_CB * 2]
2021
2022 ; Save the host XCR0 and load the guest one if necessary.
2023 mov rax, [xBP + 30h] ; pVCpu
2024 test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
2025 jz .xcr0_before_skip
2026
2027 xor ecx, ecx
2028 xgetbv ; save the host XCR0 on the stack.
2029 push xDX
2030 push xAX
2031
2032 mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
2033 mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0
2034 mov edx, [xSI + CPUMCTX.aXcr + 4]
2035 xor ecx, ecx ; paranoia
2036 xsetbv
2037
2038 push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
2039 jmp .xcr0_before_done
2040
2041.xcr0_before_skip:
2042 push 3fh ; indicate that we need not restore XCR0
2043.xcr0_before_done:
2044
2045 ; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
2046 push rsi
2047
2048 ; Save host fs, gs, sysenter msr etc.
2049 mov rax, [rbp + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
2050 push rax ; save for the vmload after vmrun
2051 vmsave
2052
2053 ; Fight spectre.
2054 INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
2055
2056 ; Setup rax for VMLOAD.
2057 mov rax, [rbp + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; take low dword only)
2058
2059 ; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
2060 mov rbx, qword [xSI + CPUMCTX.ebx]
2061 mov rcx, qword [xSI + CPUMCTX.ecx]
2062 mov rdx, qword [xSI + CPUMCTX.edx]
2063 mov rdi, qword [xSI + CPUMCTX.edi]
2064 mov rbp, qword [xSI + CPUMCTX.ebp]
2065 mov r8, qword [xSI + CPUMCTX.r8]
2066 mov r9, qword [xSI + CPUMCTX.r9]
2067 mov r10, qword [xSI + CPUMCTX.r10]
2068 mov r11, qword [xSI + CPUMCTX.r11]
2069 mov r12, qword [xSI + CPUMCTX.r12]
2070 mov r13, qword [xSI + CPUMCTX.r13]
2071 mov r14, qword [xSI + CPUMCTX.r14]
2072 mov r15, qword [xSI + CPUMCTX.r15]
2073 mov rsi, qword [xSI + CPUMCTX.esi]
2074
2075 ; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
2076 clgi
2077 sti
2078
2079 ; Load guest FS, GS, Sysenter MSRs etc.
2080 vmload
2081
2082 ; Run the VM.
2083 vmrun
2084
2085 ; Save guest fs, gs, sysenter msr etc.
2086 vmsave
2087
2088 ; Load host fs, gs, sysenter msr etc.
2089 pop rax ; load HCPhysVmcbHost (pushed above)
2090 vmload
2091
2092 ; Set the global interrupt flag again, but execute cli to make sure IF=0.
2093 cli
2094 stgi
2095
2096 ; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
2097 pop rax
2098
2099 mov qword [rax + CPUMCTX.ebx], rbx
2100 mov rbx, SPECTRE_FILLER64
2101 mov qword [rax + CPUMCTX.ecx], rcx
2102 mov rcx, rbx
2103 mov qword [rax + CPUMCTX.edx], rdx
2104 mov rdx, rbx
2105 mov qword [rax + CPUMCTX.esi], rsi
2106 mov rsi, rbx
2107 mov qword [rax + CPUMCTX.edi], rdi
2108 mov rdi, rbx
2109 mov qword [rax + CPUMCTX.ebp], rbp
2110 mov rbp, rbx
2111 mov qword [rax + CPUMCTX.r8], r8
2112 mov r8, rbx
2113 mov qword [rax + CPUMCTX.r9], r9
2114 mov r9, rbx
2115 mov qword [rax + CPUMCTX.r10], r10
2116 mov r10, rbx
2117 mov qword [rax + CPUMCTX.r11], r11
2118 mov r11, rbx
2119 mov qword [rax + CPUMCTX.r12], r12
2120 mov r12, rbx
2121 mov qword [rax + CPUMCTX.r13], r13
2122 mov r13, rbx
2123 mov qword [rax + CPUMCTX.r14], r14
2124 mov r14, rbx
2125 mov qword [rax + CPUMCTX.r15], r15
2126 mov r15, rbx
2127
2128 ; Fight spectre. Note! Trashes rax!
2129 INDIRECT_BRANCH_PREDICTION_BARRIER rax, CPUMCTX_WSF_IBPB_EXIT
2130
2131 ; Restore the host xcr0 if necessary.
2132 pop xCX
2133 test ecx, ecx
2134 jnz .xcr0_after_skip
2135 pop xAX
2136 pop xDX
2137 xsetbv ; ecx is already zero
2138.xcr0_after_skip:
2139
2140 ; Restore host general purpose registers.
2141 MYPOPAD
2142
2143 mov eax, VINF_SUCCESS
2144
2145 popf
2146 pop rbp
2147 add rsp, 6 * xCB
2148 ret
2149ENDPROC SVMR0VMRun64
2150%endif ; RT_ARCH_AMD64
2151
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette