Changeset 103513 in vbox for trunk/src/VBox/VMM/VMMAll

Timestamp:

Feb 22, 2024 3:50:36 AM (12 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

161856

Message:

VMM/IEM: The vex3 & vex2 byte prefixes must set the IEM_OP_PRF_REX_R/X/B flags so IEM_GET_MODRM_EX will work correctly for vex instructions. bugref:10370

File:

• trunk/src/VBox/VMM/VMMAll/IEMAllInstOneByte.cpp.h (modified) (2 diffs)

trunk/src/VBox/VMM/VMMAll/IEMAllInstOneByte.cpp.h

@@ -8563 +8563 @@
             uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
             pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX;
-            if ((bVex2 & 0x80 /* VEX.W */) && IEM_IS_64BIT_CODE(pVCpu))
-                pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W;
+            if (IEM_IS_64BIT_CODE(pVCpu))
+            {
+#if 1
+                AssertCompile(IEM_OP_PRF_SIZE_REX_W == RT_BIT_32(9));
+                pVCpu->iem.s.fPrefixes |= (uint32_t)(bVex2 & 0x80) << (9 - 7);
+                AssertCompile(IEM_OP_PRF_REX_B == RT_BIT_32(25) && IEM_OP_PRF_REX_X == RT_BIT_32(26) && IEM_OP_PRF_REX_R == RT_BIT_32(27));
+                pVCpu->iem.s.fPrefixes |= (uint32_t)(~bRm & 0xe0) << (25 - 5);
+#else
+                if (bVex2 & 0x80 /* VEX.W */)
+                    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W;
+                if (~bRm & 0x20 /* VEX.~B */)
+                    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_B;
+                if (~bRm & 0x40 /* VEX.~X */)
+                    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_X;
+                if (~bRm & 0x80 /* VEX.~R */)
+                    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_R;
+#endif
+            }
             pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
             pVCpu->iem.s.uRexIndex  = (~bRm >> (6 - 3)) & 0x8;
@@ -8634 +8650 @@
             uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
             pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX;
+            AssertCompile(IEM_OP_PRF_REX_R == RT_BIT_32(27));
+            pVCpu->iem.s.fPrefixes |= (uint32_t)(~bRm & 0x80) << (27 - 7);
             pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
             pVCpu->iem.s.uVex3rdReg = (~bRm >> 3) & 0xf;