Changeset 75061 in vbox for trunk/src/libs/xpcom18a4

Timestamp:

Oct 25, 2018 10:44:02 AM (6 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

126112

Message:

xpcom: make xptcinvoke_x86_64_linux.cpp work with gcc sanitisers.
bugref:8019: GCC sanitisers
XPTC_InvokeByIndex for x86_64 uses a rather original way of setting up a call
stack frame with alloca, which breaks with gcc 8.2 and asan. Conditional on
asan being in use, replace that with a slightly less orininal one.

File:

• trunk/src/libs/xpcom18a4/xpcom/reflect/xptcall/src/md/unix/xptcinvoke_x86_64_linux.cpp (modified) (2 diffs)

trunk/src/libs/xpcom18a4/xpcom/reflect/xptcall/src/md/unix/xptcinvoke_x86_64_linux.cpp

@@ -144 +144 @@
         nr_stack = (nr_stack + 1) & ~1;
 
+#ifndef VBOX_WITH_GCC_SANITIZER
     // Load parameters to stack, if necessary
     PRUint64 *stack = (PRUint64 *) __builtin_alloca(nr_stack * 8);
+#else
+    typedef struct { PRUint64 stack[20]; } methodStack;
+    if (nr_stack > 20)
+        return NS_ERROR_CALL_FAILED;
+    methodStack stack;
+#endif
     PRUint64 gpregs[GPR_COUNT];
     double fpregs[FPR_COUNT];
+#ifndef VBOX_WITH_GCC_SANITIZER
     invoke_copy_to_stack(stack, paramCount, params, gpregs, fpregs);
+#else
+    invoke_copy_to_stack(stack.stack, paramCount, params, gpregs, fpregs);
+#endif
 
     // Load FPR registers from fpregs[]
@@ -206 +217 @@
     methodAddress = *((PRUint64 *)methodAddress);
     
+#ifndef VBOX_WITH_GCC_SANITIZER
     typedef PRUint32 (*Method)(PRUint64, PRUint64, PRUint64, PRUint64, PRUint64, PRUint64);
     PRUint32 result = ((Method)methodAddress)(a0, a1, a2, a3, a4, a5);
+#else
+    typedef PRUint32 (*Method)(PRUint64, PRUint64, PRUint64, PRUint64, PRUint64, PRUint64, methodStack);
+    PRUint32 result = ((Method)methodAddress)(a0, a1, a2, a3, a4, a5, stack);
+#endif
     return result;
 }