Changeset 12404 in vbox for trunk/src

Timestamp:

Sep 11, 2008 2:58:38 PM (16 years ago)

Author:

vboxsync

Message:

Cut off calculated addresses when not in 64 bits mode. (fixes OS/2 guests #3115)

File:

• trunk/src/VBox/VMM/VMMAll/SELMAll.cpp (modified) (3 diffs)

trunk/src/VBox/VMM/VMMAll/SELMAll.cpp

@@ -127 +127 @@
         }
     }
-    return (RTGCPTR)(pHiddenSel->u64Base + (RTGCUINTPTR)Addr);
+    /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */
+    Assert(pHiddenSel->u64Base <= 0xffffffff);
+    return ((pHiddenSel->u64Base + (RTGCUINTPTR)Addr) & 0xffffffff);
 }
 
@@ -208 +210 @@
         }
         else
-            pvFlat = (RTGCPTR)(pHiddenSel->u64Base + (RTGCUINTPTR)Addr);
+        {
+            /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */
+            Assert(pHiddenSel->u64Base <= 0xffffffff);
+            pvFlat = (RTGCPTR)((pHiddenSel->u64Base + (RTGCUINTPTR)Addr) & 0xffffffff);
+        }
 
         /*
@@ -489 +495 @@
         u32Limit      = pHiddenSel->u32Limit;
         pvFlat        = (RTGCPTR)(pHiddenSel->u64Base + (RTGCUINTPTR)Addr);
+
+        if (   !CPUMIsGuestInLongMode(pVM)
+            || !pHiddenSel->Attr.n.u1Long)
+        {
+            /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */
+            pvFlat &= 0xffffffff;
+        }
     }
     else