Changeset 53172 in vbox for trunk/src/VBox/Disassembler/DisasmCore.cpp

Timestamp:

Oct 31, 2014 5:44:19 PM (10 years ago)

Author:

vboxsync

Message:

DIS: #6249: Complex instructions support (like gather) and some fixes. VSIB fix.

File:

• trunk/src/VBox/Disassembler/DisasmCore.cpp (modified) (7 diffs)

trunk/src/VBox/Disassembler/DisasmCore.cpp

@@ -709 +709 @@
     unsigned index = pDis->SIB.Bits.Index;
 
-    unsigned regtype;
-    if (pDis->uAddrMode == DISCPUMODE_32BIT)
-        regtype    = DISUSE_REG_GEN32;
-    else
-        regtype    = DISUSE_REG_GEN64;
+    unsigned regtype, vregtype;
+    /* There's no way to distinguish between SIB and VSIB
+     * and having special parameter to parse explicitly VSIB
+     * is not an options since only one instruction (gather)
+     * supports it currently. May be changed in the future. */
+        if (pDis->uAddrMode == DISCPUMODE_32BIT)
+            regtype    = DISUSE_REG_GEN32;
+        else
+            regtype    = DISUSE_REG_GEN64;
+    if (pDis->pCurInstr->uOpcode == OP_GATHER)
+        vregtype = (VEXREG_IS256B(pDis->bVexDestReg) ? DISUSE_REG_YMM : DISUSE_REG_XMM);
+    else
+        vregtype = regtype;
 
     if (index != 4)
     {
-         pParam->fUse |= DISUSE_INDEX | regtype;
-         pParam->Index.idxGenReg = index;
-
-         if (scale != 0)
-         {
-             pParam->fUse  |= DISUSE_SCALE;
-             pParam->uScale = (1<<scale);
-         }
+        pParam->fUse |= DISUSE_INDEX | vregtype;
+        pParam->Index.idxGenReg = index;
+
+        if (scale != 0)
+        {
+            pParam->fUse  |= DISUSE_SCALE;
+            pParam->uScale = (1<<scale);
+        }
     }
 
@@ -2488 +2496 @@
             pOpCode = g_aVexOpcodesMap[0] + pDis->bOpCode;
         break;
-        // OPSIZE 0x66 prefix
+        // 0x66 prefix
         case 1:
             pOpCode = g_aVexOpcodesMap_66H[0] + pDis->bOpCode;
 
-            /* TODO: Check if we need to set this prefix */
-            pDis->fPrefix |= DISPREFIX_OPSIZE;
-            if (pDis->uCpuMode == DISCPUMODE_16BIT)
-                pDis->uOpMode = DISCPUMODE_32BIT;
-            else
-                pDis->uOpMode = DISCPUMODE_16BIT;  /* for 32 and 64 bits mode (there is no 32 bits operand size override prefix) */
         break;
 
-        // REPE 0xF3 prefix
+        // 0xF3 prefix
         case 2:
             pOpCode = g_aVexOpcodesMap_F3H[0] + pDis->bOpCode;
         break;
 
-        // REPNE 0xF2 prefix
+        // 0xF2 prefix
         case 3:
             pOpCode = g_aVexOpcodesMap_F2H[0] + pDis->bOpCode;
@@ -2530 +2532 @@
     uint8_t implOpcode = (byte1 & 0x1f);
 
-    if (pDis->uCpuMode == DISCPUMODE_64BIT)
-    {
-        // REX.RXB
-        if (~(byte1 & 0xe0))
-        {
-            pDis->fRexPrefix = (byte1 >> 5) ^ 7;
-            if (pDis->fRexPrefix)
-                pDis->fPrefix |= DISPREFIX_REX;
-        }
-
-        // REX.W
-        if (!(byte2 & 0x80))
-        {
-            pDis->fRexPrefix |= DISPREFIX_REX_FLAGS_W;
-            if (pDis->fRexPrefix)
-                 pDis->fPrefix |= DISPREFIX_REX;
-        }
-    }
+    // REX.RXB
+    if (pDis->uCpuMode == DISCPUMODE_64BIT && ~(byte1 & 0xe0))
+        pDis->fRexPrefix |= (byte1 >> 5) ^ 7;
+
+    // VEX.W
+    pDis->bVexWFlag = !(byte2 & 0x80);
+
+    if (pDis->fRexPrefix)
+        pDis->fPrefix |= DISPREFIX_REX;
 
     switch(byte2 & 3)
@@ -2570 +2563 @@
             }
         break;
-        // OPSIZE 0x66 prefix
+        // 0x66 prefix
         case 1:
             if (implOpcode >= 1 && implOpcode <= 3) // Other values are #UD.
@@ -2577 +2570 @@
                 if (pOpCode != NULL)
                     pOpCode = &pOpCode[pDis->bOpCode];
-                /* TODO: check if we need to set this prefix */
-                pDis->fPrefix |= DISPREFIX_OPSIZE;
-                if (pDis->uCpuMode == DISCPUMODE_16BIT)
-                    pDis->uOpMode = DISCPUMODE_32BIT;
-                else
-                    pDis->uOpMode = DISCPUMODE_16BIT;  /* for 32 and 64 bits mode (there is no 32 bits operand size override prefix) */
             }
         break;
 
-        // REPE 0xF3 prefix
+        // 0xF3 prefix
         case 2:
             if (implOpcode >= 1 && implOpcode <= 3) // Other values are #UD.
@@ -2608 +2595 @@
         break;
 
-        // REPNE 0xF2 prefix
+        // 0xF2 prefix
         case 3:
             if (implOpcode >= 1 && implOpcode <= 3) // Other values are #UD.
@@ -2824 +2811 @@
 
         /* Check if this is a VEX prefix. Not for 32-bit mode. */
-        if ((opcode == OP_LES || opcode == OP_LDS)
-            && (disReadByte(pDis, offInstr) & 0xc0) == 0xc0
-            && (opcode != OP_LES || pDis->uCpuMode == DISCPUMODE_64BIT || !(disReadByte(pDis, offInstr + 1) & 0x80)))
+        if (pDis->uCpuMode != DISCPUMODE_64BIT
+            && (opcode == OP_LES || opcode == OP_LDS)
+            && (disReadByte(pDis, offInstr) & 0xc0) == 0xc0)
         {
             paOneByteMap = g_aOneByteMapX64;