Changeset 61806 in vbox

Timestamp:

Jun 21, 2016 5:39:13 PM (8 years ago)

Author:

vboxsync

Message:

VMM: bugref:8412, ticketref:15439: workaround for changed stack layout of Linux >= 4.6

Location:

trunk/src/VBox/VMM

Files:

• VMMR0/VMMR0JmpA-amd64.asm (modified) (7 diffs)
• testcase/tstVMMR0CallHost-1.cpp (modified) (1 diff)

trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm

@@ -31 +31 @@
 %define STACK_PADDING   0eeeeeeeeeeeeeeeeh
 
+;; Workaround for linux 4.6 fast/slow syscall stack depth difference.
+%ifdef VMM_R0_SWITCH_STACK
+ %define STACK_FUZZ_SIZE 0
+%else
+ %define STACK_FUZZ_SIZE 128
+%endif
 
 
@@ -57 +63 @@
     mov     rbp, rsp
  %ifdef ASM_CALL64_MSC
-    sub     rsp, 30h
+    sub     rsp, 30h + STACK_FUZZ_SIZE  ; (10h is used by resume (??), 20h for callee spill area)
     mov     r11, rdx                    ; pfn
     mov     rdx, rcx                    ; pJmpBuf;
  %else
-    sub     rsp, 10h
+    sub     rsp, 10h + STACK_FUZZ_SIZE  ; (10h is used by resume (??))
     mov     r8, rdx                     ; pvUser1 (save it like MSC)
     mov     r9, rcx                     ; pvUser2 (save it like MSC)
@@ -72 +78 @@
     mov     [xDX + VMMR0JMPBUF.rdi], rdi
  %endif
-    mov     r10, [rbp]
-    mov     [xDX + VMMR0JMPBUF.rbp], r10
+    mov     [xDX + VMMR0JMPBUF.rbp], rbp
     mov     [xDX + VMMR0JMPBUF.r12], r12
     mov     [xDX + VMMR0JMPBUF.r13], r13
     mov     [xDX + VMMR0JMPBUF.r14], r14
     mov     [xDX + VMMR0JMPBUF.r15], r15
-    mov     xAX, [rbp + 8]
+    mov     xAX, [rbp + 8]              ; (not really necessary, except for validity check)
     mov     [xDX + VMMR0JMPBUF.rip], xAX
-    lea     r10, [rbp + 10h]            ; (used in resume)
+ %ifdef ASM_CALL64_MSC
+    lea     r10, [rsp + 20h]            ; must save the spill area
+ %else
+    lea     r10, [rsp]
+ %endif
     mov     [xDX + VMMR0JMPBUF.rsp], r10
  %ifdef RT_OS_WINDOWS
@@ -140 +149 @@
 
     ;
-    ; Return like in the long jump but clear eip, no short cuts here.
+    ; Return like in the long jump but clear eip, no shortcuts here.
     ;
 .proper_return:
@@ -165 +174 @@
     mov     r15, [xDX + VMMR0JMPBUF.r15]
     mov     rbp, [xDX + VMMR0JMPBUF.rbp]
-    mov     xCX, [xDX + VMMR0JMPBUF.rip]
     and     qword [xDX + VMMR0JMPBUF.rip], byte 0 ; used for valid check.
     mov     rsp, [xDX + VMMR0JMPBUF.rsp]
     push    qword [xDX + VMMR0JMPBUF.rflags]
     popf
-    jmp     xCX
+    leave
+    ret
 
 .entry_error:
@@ -203 +212 @@
     ;
 .resume:
-    ; Sanity checks.
- %ifdef VMM_R0_SWITCH_STACK
-    ;; @todo amd64/switch/resume sanity.
- %else  ; !VMM_R0_SWITCH_STACK
-    cmp     r10, [xDX + VMMR0JMPBUF.SpCheck]
-    jne     .bad
-
+ %ifndef VMM_R0_SWITCH_STACK
+    ; Sanity checks incoming stack, applying fuzz if needed.
+    sub     r10, [xDX + VMMR0JMPBUF.SpCheck]
+    jz      .resume_stack_checked_out
+    add     r10, STACK_FUZZ_SIZE        ; plus/minus STACK_FUZZ_SIZE is fine.
+    cmp     r10, STACK_FUZZ_SIZE * 2
+    ja      .bad
+
+    mov     r10, [xDX + VMMR0JMPBUF.SpCheck]
+    mov     [xDX + VMMR0JMPBUF.rsp], r10 ; Must be update in case of another long jump (used for save calc).
+
+.resume_stack_checked_out:
     mov     ecx, [xDX + VMMR0JMPBUF.cbSavedStack]
     cmp     rcx, VMM_STACK_SIZE
     ja      .bad
-    test    rcx, 3
+    test    rcx, 7
     jnz     .bad
-    mov     rdi, [xDX + VMMR0JMPBUF.rsp]
+    mov     rdi, [xDX + VMMR0JMPBUF.SpCheck]
     sub     rdi, [xDX + VMMR0JMPBUF.SpResume]
     cmp     rcx, rdi
@@ -395 +409 @@
     mov     r15, [xDX + VMMR0JMPBUF.r15]
     mov     rbp, [xDX + VMMR0JMPBUF.rbp]
-    mov     rcx, [xDX + VMMR0JMPBUF.rip]
     mov     rsp, [xDX + VMMR0JMPBUF.rsp]
     push    qword [xDX + VMMR0JMPBUF.rflags]
     popf
-    jmp     rcx
+    leave
+    ret
 
     ;

trunk/src/VBox/VMM/testcase/tstVMMR0CallHost-1.cpp

@@ -103 +103 @@
 DECLCALLBACK(DECL_NO_INLINE(RT_NOTHING, int)) stackRandom(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP pfn, PVM pVM, PVMCPU pVCpu)
 {
-#if 0
-    uint32_t            cbRand  = RTRandU32Ex(1, 64);
+#ifdef RT_ARCH_AMD64
+    uint32_t            cbRand  = RTRandU32Ex(1, 96);
 #else
     uint32_t            cbRand  = 1;