Index: trunk/src/VBox/VMM/VMMR3/DBGF.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/DBGF.cpp	(revision 99891)
+++ trunk/src/VBox/VMM/VMMR3/DBGF.cpp	(revision 99897)
@@ -432,5 +432,5 @@
 
         case EMSTATE_IEM:
-        case EMSTATE_RAW:
+        case EMSTATE_RAW_OBSOLETE:
         case EMSTATE_IEM_THEN_REM:
         case EMSTATE_DEBUG_GUEST_IEM:
@@ -439,6 +439,6 @@
 
 
-        case EMSTATE_REM:
-        case EMSTATE_DEBUG_GUEST_REM:
+        case EMSTATE_RECOMPILER:
+        case EMSTATE_DEBUG_GUEST_RECOMPILER:
             return DBGFEVENTCTX_REM;
 
Index: trunk/src/VBox/VMM/VMMR3/EM.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/EM.cpp	(revision 99891)
+++ trunk/src/VBox/VMM/VMMR3/EM.cpp	(revision 99897)
@@ -33,5 +33,5 @@
  * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
  * modes has different inner loops (emR3RawExecute, emR3HmExecute, and
- * emR3RemExecute).
+ * emR3RmExecute).
  *
  * The interpreted execution is only used to avoid switching between
@@ -78,4 +78,6 @@
 #include <iprt/thread.h>
 
+#include "EMInline.h"
+
 
 /*********************************************************************************************************************************
@@ -88,8 +90,4 @@
 #endif
 static VBOXSTRICTRC emR3Debug(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc);
-#if defined(VBOX_WITH_REM) || defined(DEBUG)
-static int emR3RemStep(PVM pVM, PVMCPU pVCpu);
-#endif
-static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
 
 
@@ -262,17 +260,11 @@
         EM_REG_COUNTER(&pVCpu->em.s.StatNEMExecuteCalled,   "/PROF/CPU%u/EM/NEMExecuteCalled",  "Number of times enmR3NEMExecute is called.");
 #ifdef VBOX_WITH_STATISTICS
-        EM_REG_PROFILE(&pVCpu->em.s.StatREMEmu,             "/PROF/CPU%u/EM/REMEmuSingle",      "Profiling single instruction REM execution.");
         EM_REG_PROFILE(&pVCpu->em.s.StatREMExec,            "/PROF/CPU%u/EM/REMExec",           "Profiling REM execution.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatREMSync,            "/PROF/CPU%u/EM/REMSync",           "Profiling REM context syncing.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatRAWEntry,           "/PROF/CPU%u/EM/RAWEnter",          "Profiling Raw Mode entry overhead.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatRAWExec,            "/PROF/CPU%u/EM/RAWExec",           "Profiling Raw Mode execution.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatRAWTail,            "/PROF/CPU%u/EM/RAWTail",           "Profiling Raw Mode tail overhead.");
-#endif /* VBOX_WITH_STATISTICS */
+#endif
 
         EM_REG_COUNTER(&pVCpu->em.s.StatForcedActions,      "/PROF/CPU%u/EM/ForcedActions",     "Profiling forced action execution.");
         EM_REG_COUNTER(&pVCpu->em.s.StatHalted,             "/PROF/CPU%u/EM/Halted",            "Profiling halted state (VMR3WaitHalted).");
         EM_REG_PROFILE_ADV(&pVCpu->em.s.StatCapped,         "/PROF/CPU%u/EM/Capped",            "Profiling capped state (sleep).");
-        EM_REG_COUNTER(&pVCpu->em.s.StatREMTotal,           "/PROF/CPU%u/EM/REMTotal",          "Profiling emR3RemExecute (excluding FFs).");
-        EM_REG_COUNTER(&pVCpu->em.s.StatRAWTotal,           "/PROF/CPU%u/EM/RAWTotal",          "Profiling emR3RawExecute (excluding FFs).");
+        EM_REG_COUNTER(&pVCpu->em.s.StatREMTotal,           "/PROF/CPU%u/EM/REMTotal",          "Profiling emR3RecompilerExecute (excluding FFs).");
 
         EM_REG_PROFILE_ADV(&pVCpu->em.s.StatTotal,          "/PROF/CPU%u/EM/Total",             "Profiling EMR3ExecuteVM.");
@@ -418,5 +410,9 @@
         Assert(pVCpu->em.s.enmState     == EMSTATE_SUSPENDED);
         Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
-        SSMR3PutU32(pSSM, pVCpu->em.s.enmPrevState);
+        SSMR3PutU32(pSSM,
+                       pVCpu->em.s.enmPrevState == EMSTATE_NONE
+                    || pVCpu->em.s.enmPrevState == EMSTATE_HALTED
+                    || pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
+                    ? pVCpu->em.s.enmPrevState : EMSTATE_NONE);
 
         /* Save mwait state. */
@@ -447,6 +443,6 @@
      * Validate version.
      */
-    if (    uVersion > EM_SAVED_STATE_VERSION
-        ||  uVersion < EM_SAVED_STATE_VERSION_PRE_SMP)
+    if (   uVersion > EM_SAVED_STATE_VERSION
+        || uVersion < EM_SAVED_STATE_VERSION_PRE_SMP)
     {
         AssertMsgFailed(("emR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, EM_SAVED_STATE_VERSION));
@@ -468,6 +464,12 @@
         if (uVersion > EM_SAVED_STATE_VERSION_PRE_SMP)
         {
+            /* We are only intereseted in two enmPrevState values for use when
+               EMR3ExecuteVM is called.
+               Since ~r157540. only these two and EMSTATE_NONE are saved. */
             SSM_GET_ENUM32_RET(pSSM, pVCpu->em.s.enmPrevState, EMSTATE);
             Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
+            if (   pVCpu->em.s.enmPrevState != EMSTATE_WAIT_SIPI
+                && pVCpu->em.s.enmPrevState != EMSTATE_HALTED)
+                pVCpu->em.s.enmPrevState = EMSTATE_NONE;
 
             pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
@@ -517,7 +519,4 @@
         switch (pArgs->enmPolicy)
         {
-            case EMEXECPOLICY_RECOMPILE_RING0:
-            case EMEXECPOLICY_RECOMPILE_RING3:
-                break;
             case EMEXECPOLICY_IEM_ALL:
                 pVM->em.s.fIemExecutesAll = pArgs->fEnforce;
@@ -530,8 +529,11 @@
                     switch (pVCpuX->em.s.enmState)
                     {
+                        case EMSTATE_DEBUG_GUEST_RECOMPILER:
+                            if (pVM->em.s.fIemRecompiled)
+                                break;
+                            RT_FALL_THROUGH();
                         case EMSTATE_DEBUG_GUEST_RAW:
                         case EMSTATE_DEBUG_GUEST_HM:
                         case EMSTATE_DEBUG_GUEST_NEM:
-                        case EMSTATE_DEBUG_GUEST_REM:
                             Log(("EM: idCpu=%u: %s -> EMSTATE_DEBUG_GUEST_IEM\n", i, emR3GetStateName(pVCpuX->em.s.enmState) ));
                             pVCpuX->em.s.enmState = EMSTATE_DEBUG_GUEST_IEM;
@@ -543,18 +545,25 @@
                 }
                 break;
+
+            case EMEXECPOLICY_IEM_RECOMPILED:
+                pVM->em.s.fIemRecompiled = pArgs->fEnforce;
+                break;
+
             default:
                 AssertFailedReturn(VERR_INVALID_PARAMETER);
         }
-        Log(("EM: Set execution policy (fIemExecutesAll=%RTbool)\n", pVM->em.s.fIemExecutesAll));
+        Log(("EM: Set execution policy: fIemExecutesAll=%RTbool fIemRecompiled=%RTbool\n",
+             pVM->em.s.fIemExecutesAll, pVM->em.s.fIemRecompiled));
     }
 
     /*
-     * Force rescheduling if in RAW, HM, NEM, IEM, or REM.
+     * Force rescheduling if in HM, NEM, IEM/interpreter or IEM/recompiler.
      */
-    return    pVCpu->em.s.enmState == EMSTATE_RAW
-           || pVCpu->em.s.enmState == EMSTATE_HM
+    Assert(pVCpu->em.s.enmState != EMSTATE_RAW_OBSOLETE);
+    return    pVCpu->em.s.enmState == EMSTATE_HM
            || pVCpu->em.s.enmState == EMSTATE_NEM
            || pVCpu->em.s.enmState == EMSTATE_IEM
-           || pVCpu->em.s.enmState == EMSTATE_REM
+           || pVCpu->em.s.enmState == EMSTATE_RECOMPILER
+           /* obsolete stuff: */
            || pVCpu->em.s.enmState == EMSTATE_IEM_THEN_REM
          ? VINF_EM_RESCHEDULE
@@ -607,11 +616,10 @@
     switch (enmPolicy)
     {
-        case EMEXECPOLICY_RECOMPILE_RING0:
-        case EMEXECPOLICY_RECOMPILE_RING3:
-            *pfEnforced = false;
-            break;
         case EMEXECPOLICY_IEM_ALL:
             *pfEnforced = pVM->em.s.fIemExecutesAll;
             break;
+        case EMEXECPOLICY_IEM_RECOMPILED:
+            *pfEnforced = pVM->em.s.fIemRecompiled;
+            break;
         default:
             AssertFailedReturn(VERR_INTERNAL_ERROR_2);
@@ -670,23 +678,23 @@
     switch (enmState)
     {
-        case EMSTATE_NONE:              return "EMSTATE_NONE";
-        case EMSTATE_RAW:               return "EMSTATE_RAW";
-        case EMSTATE_HM:                return "EMSTATE_HM";
-        case EMSTATE_IEM:               return "EMSTATE_IEM";
-        case EMSTATE_REM:               return "EMSTATE_REM";
-        case EMSTATE_HALTED:            return "EMSTATE_HALTED";
-        case EMSTATE_WAIT_SIPI:         return "EMSTATE_WAIT_SIPI";
-        case EMSTATE_SUSPENDED:         return "EMSTATE_SUSPENDED";
-        case EMSTATE_TERMINATING:       return "EMSTATE_TERMINATING";
-        case EMSTATE_DEBUG_GUEST_RAW:   return "EMSTATE_DEBUG_GUEST_RAW";
-        case EMSTATE_DEBUG_GUEST_HM:    return "EMSTATE_DEBUG_GUEST_HM";
-        case EMSTATE_DEBUG_GUEST_IEM:   return "EMSTATE_DEBUG_GUEST_IEM";
-        case EMSTATE_DEBUG_GUEST_REM:   return "EMSTATE_DEBUG_GUEST_REM";
-        case EMSTATE_DEBUG_HYPER:       return "EMSTATE_DEBUG_HYPER";
-        case EMSTATE_GURU_MEDITATION:   return "EMSTATE_GURU_MEDITATION";
-        case EMSTATE_IEM_THEN_REM:      return "EMSTATE_IEM_THEN_REM";
-        case EMSTATE_NEM:               return "EMSTATE_NEM";
-        case EMSTATE_DEBUG_GUEST_NEM:   return "EMSTATE_DEBUG_GUEST_NEM";
-        default:                        return "Unknown!";
+        case EMSTATE_NONE:                      return "EMSTATE_NONE";
+        case EMSTATE_RAW_OBSOLETE:              return "EMSTATE_RAW_OBSOLETE";
+        case EMSTATE_HM:                        return "EMSTATE_HM";
+        case EMSTATE_IEM:                       return "EMSTATE_IEM";
+        case EMSTATE_RECOMPILER:                return "EMSTATE_RECOMPILER";
+        case EMSTATE_HALTED:                    return "EMSTATE_HALTED";
+        case EMSTATE_WAIT_SIPI:                 return "EMSTATE_WAIT_SIPI";
+        case EMSTATE_SUSPENDED:                 return "EMSTATE_SUSPENDED";
+        case EMSTATE_TERMINATING:               return "EMSTATE_TERMINATING";
+        case EMSTATE_DEBUG_GUEST_RAW:           return "EMSTATE_DEBUG_GUEST_RAW";
+        case EMSTATE_DEBUG_GUEST_HM:            return "EMSTATE_DEBUG_GUEST_HM";
+        case EMSTATE_DEBUG_GUEST_IEM:           return "EMSTATE_DEBUG_GUEST_IEM";
+        case EMSTATE_DEBUG_GUEST_RECOMPILER:    return "EMSTATE_DEBUG_GUEST_RECOMPILER";
+        case EMSTATE_DEBUG_HYPER:               return "EMSTATE_DEBUG_HYPER";
+        case EMSTATE_GURU_MEDITATION:           return "EMSTATE_GURU_MEDITATION";
+        case EMSTATE_IEM_THEN_REM:              return "EMSTATE_IEM_THEN_REM";
+        case EMSTATE_NEM:                       return "EMSTATE_NEM";
+        case EMSTATE_DEBUG_GUEST_NEM:           return "EMSTATE_DEBUG_GUEST_NEM";
+        default:                                return "Unknown!";
     }
 }
@@ -874,8 +882,4 @@
                 else if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_NEM)
                     rc = VBOXSTRICTRC_TODO(emR3NemSingleInstruction(pVM, pVCpu, 0 /*fFlags*/));
-#ifdef VBOX_WITH_REM /** @todo fix me? */
-                else if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM)
-                    rc = emR3RemStep(pVM, pVCpu);
-#endif
                 else
                 {
@@ -924,7 +928,4 @@
             case VERR_VMM_RING0_ASSERTION: /** @todo Make a guru meditation event! */
                 rc = DBGFR3EventSrc(pVM, DBGFEVENT_FATAL_ERROR, "VERR_VMM_RING0_ASSERTION", 0, NULL, NULL);
-                break;
-            case VERR_REM_TOO_MANY_TRAPS: /** @todo Make a guru meditation event! */
-                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VERR_REM_TOO_MANY_TRAPS", 0, NULL, NULL);
                 break;
             case VINF_EM_TRIPLE_FAULT:    /** @todo Make a guru meditation event! */
@@ -1031,34 +1032,4 @@
 
 
-#if defined(VBOX_WITH_REM) || defined(DEBUG)
-/**
- * Steps recompiled code.
- *
- * @returns VBox status code. The most important ones are: VINF_EM_STEP_EVENT,
- *          VINF_EM_RESCHEDULE, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
- *
- * @param   pVM         The cross context VM structure.
- * @param   pVCpu       The cross context virtual CPU structure.
- */
-static int emR3RemStep(PVM pVM, PVMCPU pVCpu)
-{
-#if defined(VBOX_VMM_TARGET_ARMV8)
-    Log3(("emR3RemStep: pc=%08x\n", CPUMGetGuestFlatPC(pVCpu)));
-#else
-    Log3(("emR3RemStep: cs:eip=%04x:%08x\n", CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
-#endif
-
-    int rc = VBOXSTRICTRC_TODO(IEMExecOne(pVCpu)); NOREF(pVM);
-
-#if defined(VBOX_VMM_TARGET_ARMV8)
-    Log3(("emR3RemStep: pc=%08x\n", CPUMGetGuestFlatPC(pVCpu)));
-#else
-    Log3(("emR3RemStep: returns %Rrc cs:eip=%04x:%08x\n", rc, CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
-#endif
-    return rc;
-}
-#endif /* VBOX_WITH_REM || DEBUG */
-
-
 /**
  * Executes recompiled code.
@@ -1076,36 +1047,45 @@
  *
  */
-static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
-{
+static VBOXSTRICTRC emR3RecompilerExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+    STAM_REL_PROFILE_START(&pVCpu->em.s.StatREMTotal, a);
+#ifdef VBOX_VMM_TARGET_ARMV8
+    LogFlow(("emR3RecompilerExecute/%u: (pc=%RGv)\n", pVCpu->idCpu, (RTGCPTR)pVCpu->cpum.GstCtx.Pc.u64));
+#else
+    LogFlow(("emR3RecompilerExecute/%u: (cs:eip=%04x:%RGv)\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, (RTGCPTR)pVCpu->cpum.GstCtx.rip));
+#endif
+
+    /*
+     * Loop till we get a forced action which returns anything but VINF_SUCCESS.
+     */
+    *pfFFDone = false;
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    for (;;)
+    {
 #ifdef LOG_ENABLED
 # if defined(VBOX_VMM_TARGET_ARMV8)
-    Log3(("EM: pc=%08x\n", CPUMGetGuestFlatPC(pVCpu)));
+        Log3(("EM: pc=%08x\n", CPUMGetGuestFlatPC(pVCpu)));
 # else
-    uint32_t cpl = CPUMGetGuestCPL(pVCpu);
-
-    if (pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
-        Log(("EMV86: %04X:%08X IF=%d\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.Bits.u1IF));
-    else
-        Log(("EMR%d: %04X:%08X ESP=%08X IF=%d CR0=%x eflags=%x\n", cpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF, (uint32_t)pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.eflags.u));
+        if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+            Log(("EMR%d: %04X:%08RX64 RSP=%08RX64 IF=%d CR0=%x eflags=%x\n", CPUMGetGuestCPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel,
+                 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.Bits.u1IF,
+                 (uint32_t)pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.eflags.u));
+        else
+            Log(("EMV86: %04X:%08X IF=%d\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.Bits.u1IF));
 # endif
 #endif
-    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatREMTotal, a);
-
-    /*
-     * Spin till we get a forced action which returns anything but VINF_SUCCESS
-     * or the REM suggests raw-mode execution.
-     */
-    *pfFFDone = false;
-    uint32_t cLoops     = 0;
-    int     rc          = VINF_SUCCESS;
-    for (;;)
-    {
-        /*
-         * Execute REM.
+
+        /*
+         * Execute.
          */
         if (RT_LIKELY(emR3IsExecutionAllowed(pVM, pVCpu)))
         {
             STAM_PROFILE_START(&pVCpu->em.s.StatREMExec, c);
-            rc = VBOXSTRICTRC_TODO(IEMExecLots(pVCpu, 8192 /*cMaxInstructions*/, 4095 /*cPollRate*/, NULL /*pcInstructions*/));
+#ifdef VBOX_WITH_IEM_RECOMPILER
+            if (pVM->em.s.fIemRecompiled)
+                rcStrict = IEMExecLots(pVCpu, 8192 /*cMaxInstructions*/, 4095 /*cPollRate*/, NULL /*pcInstructions*/);
+            else
+#endif
+                rcStrict = IEMExecLots(pVCpu, 8192 /*cMaxInstructions*/, 4095 /*cPollRate*/, NULL /*pcInstructions*/);
             STAM_PROFILE_STOP(&pVCpu->em.s.StatREMExec, c);
         }
@@ -1116,5 +1096,5 @@
             RTThreadSleep(5);
             STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
-            rc = VINF_SUCCESS;
+            rcStrict = VINF_SUCCESS;
         }
 
@@ -1125,34 +1105,17 @@
         if (    VM_FF_IS_ANY_SET(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
             ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
-            rc = VBOXSTRICTRC_TODO(emR3HighPriorityPostForcedActions(pVM, pVCpu, rc));
+            rcStrict = emR3HighPriorityPostForcedActions(pVM, pVCpu, rcStrict);
 
         /*
          * Process the returned status code.
          */
-        if (rc != VINF_SUCCESS)
-        {
-            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+        if (rcStrict != VINF_SUCCESS)
+        {
+#if 0
+            if (RT_LIKELY(rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST))
                 break;
-            if (rc != VINF_REM_INTERRUPED_FF)
-            {
-                /* Try dodge unimplemented IEM trouble by reschduling. */
-                if (   rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED
-                    || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED)
-                {
-                    EMSTATE enmNewState = emR3Reschedule(pVM, pVCpu);
-                    if (enmNewState != EMSTATE_REM && enmNewState != EMSTATE_IEM_THEN_REM)
-                    {
-                        rc = VINF_EM_RESCHEDULE;
-                        break;
-                    }
-                }
-
-                /*
-                 * Anything which is not known to us means an internal error
-                 * and the termination of the VM!
-                 */
-                AssertMsg(rc == VERR_REM_TOO_MANY_TRAPS, ("Unknown GC return code: %Rra\n", rc));
-                break;
-            }
+            /* Fatal error: */
+#endif
+            break;
         }
 
@@ -1168,13 +1131,11 @@
 #endif
         AssertCompile(VMCPU_FF_ALL_REM_MASK & VMCPU_FF_TIMER);
-        if (    VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_REM_MASK)
-            ||  VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_ALL_REM_MASK) )
-        {
-            STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatREMTotal, a);
-            rc = emR3ForcedActions(pVM, pVCpu, rc);
-            VBOXVMM_EM_FF_ALL_RET(pVCpu, rc);
-            STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatREMTotal, a);
-            if (    rc != VINF_SUCCESS
-                &&  rc != VINF_EM_RESCHEDULE_REM)
+        if (   VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_REM_MASK)
+            || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_ALL_REM_MASK) )
+        {
+            rcStrict = emR3ForcedActions(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict));
+            VBOXVMM_EM_FF_ALL_RET(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+            if (   rcStrict != VINF_SUCCESS
+                && rcStrict != VINF_EM_RESCHEDULE_REM)
             {
                 *pfFFDone = true;
@@ -1183,55 +1144,9 @@
         }
 
-        /*
-         * Have to check if we can get back to fast execution mode every so often.
-         */
-        if (!(++cLoops & 7))
-        {
-            EMSTATE enmCheck = emR3Reschedule(pVM, pVCpu);
-            if (   enmCheck != EMSTATE_REM
-                && enmCheck != EMSTATE_IEM_THEN_REM)
-            {
-                LogFlow(("emR3RemExecute: emR3Reschedule -> %d -> VINF_EM_RESCHEDULE\n", enmCheck));
-                STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatREMTotal, a);
-                return VINF_EM_RESCHEDULE;
-            }
-            Log2(("emR3RemExecute: emR3Reschedule -> %d\n", enmCheck));
-        }
-
     } /* The Inner Loop, recompiled execution mode version. */
 
-    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatREMTotal, a);
-    return rc;
-}
-
-
-#ifdef DEBUG
-
-int emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
-{
-    EMSTATE  enmOldState = pVCpu->em.s.enmState;
-
-    pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
-
-    Log(("Single step BEGIN:\n"));
-    for (uint32_t i = 0; i < cIterations; i++)
-    {
-        DBGFR3PrgStep(pVCpu);
-        DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, "RSS");
-        emR3RemStep(pVM, pVCpu);
-        if (emR3Reschedule(pVM, pVCpu) != EMSTATE_REM)
-            break;
-    }
-    Log(("Single step END:\n"));
-#if defined(VBOX_VMM_TARGET_ARMV8)
-    AssertReleaseFailed();
-#else
-    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
-#endif
-    pVCpu->em.s.enmState = enmOldState;
-    return VINF_EM_RESCHEDULE;
-}
-
-#endif /* DEBUG */
+    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatREMTotal, a);
+    return rcStrict;
+}
 
 
@@ -1277,5 +1192,5 @@
 
         EMSTATE enmNewState = emR3Reschedule(pVM, pVCpu);
-        if (enmNewState != EMSTATE_REM && enmNewState != EMSTATE_IEM_THEN_REM)
+        if (enmNewState != EMSTATE_RECOMPILER && enmNewState != EMSTATE_IEM_THEN_REM)
         {
             LogFlow(("emR3ExecuteIemThenRem: -> %d (%s) after %u instructions\n",
@@ -1297,6 +1212,6 @@
      * Switch to REM.
      */
-    Log(("emR3ExecuteIemThenRem: -> EMSTATE_REM (after %u instructions)\n", pVCpu->em.s.cIemThenRemInstructions));
-    pVCpu->em.s.enmState = EMSTATE_REM;
+    Log(("emR3ExecuteIemThenRem: -> EMSTATE_RECOMPILER (after %u instructions)\n", pVCpu->em.s.cIemThenRemInstructions));
+    pVCpu->em.s.enmState = EMSTATE_RECOMPILER;
     return VINF_SUCCESS;
 }
@@ -1323,5 +1238,9 @@
     if (   pVM->em.s.fIemExecutesAll
         || VM_IS_EXEC_ENGINE_IEM(pVM))
+#ifdef VBOX_WITH_IEM_RECOMPILER
+        return pVM->em.s.fIemRecompiled ? EMSTATE_RECOMPILER : EMSTATE_IEM;
+#else
         return EMSTATE_IEM;
+#endif
 
 #if !defined(VBOX_VMM_TARGET_ARMV8)
@@ -2171,24 +2090,25 @@
  * @param   pVCpu       The cross context virtual CPU structure.
  */
-bool emR3IsExecutionAllowed(PVM pVM, PVMCPU pVCpu)
-{
-    uint64_t u64UserTime, u64KernelTime;
-
-    if (    pVM->uCpuExecutionCap != 100
-        &&  RT_SUCCESS(RTThreadGetExecutionTimeMilli(&u64KernelTime, &u64UserTime)))
-    {
-        uint64_t u64TimeNow = RTTimeMilliTS();
-        if (pVCpu->em.s.u64TimeSliceStart + EM_TIME_SLICE < u64TimeNow)
+bool emR3IsExecutionAllowedSlow(PVM pVM, PVMCPU pVCpu)
+{
+    Assert(pVM->uCpuExecutionCap != 100);
+    uint64_t cMsUserTime;
+    uint64_t cMsKernelTime;
+    if (RT_SUCCESS(RTThreadGetExecutionTimeMilli(&cMsKernelTime, &cMsUserTime)))
+    {
+        uint64_t const msTimeNow = RTTimeMilliTS();
+        if (pVCpu->em.s.u64TimeSliceStart + EM_TIME_SLICE < msTimeNow)
         {
             /* New time slice. */
-            pVCpu->em.s.u64TimeSliceStart     = u64TimeNow;
-            pVCpu->em.s.u64TimeSliceStartExec = u64KernelTime + u64UserTime;
+            pVCpu->em.s.u64TimeSliceStart     = msTimeNow;
+            pVCpu->em.s.u64TimeSliceStartExec = cMsKernelTime + cMsUserTime;
             pVCpu->em.s.u64TimeSliceExec      = 0;
         }
-        pVCpu->em.s.u64TimeSliceExec = u64KernelTime + u64UserTime - pVCpu->em.s.u64TimeSliceStartExec;
-
-        Log2(("emR3IsExecutionAllowed: start=%RX64 startexec=%RX64 exec=%RX64 (cap=%x)\n", pVCpu->em.s.u64TimeSliceStart, pVCpu->em.s.u64TimeSliceStartExec, pVCpu->em.s.u64TimeSliceExec, (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100));
-        if (pVCpu->em.s.u64TimeSliceExec >= (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100)
-            return false;
+        pVCpu->em.s.u64TimeSliceExec = cMsKernelTime + cMsUserTime - pVCpu->em.s.u64TimeSliceStartExec;
+
+        bool const fRet = pVCpu->em.s.u64TimeSliceExec < (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100;
+        Log2(("emR3IsExecutionAllowed: start=%RX64 startexec=%RX64 exec=%RX64 (cap=%x)\n", pVCpu->em.s.u64TimeSliceStart,
+              pVCpu->em.s.u64TimeSliceStartExec, pVCpu->em.s.u64TimeSliceExec, (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100));
+        return fRet;
     }
     return true;
@@ -2483,10 +2403,5 @@
                 case VINF_EM_DBG_BREAKPOINT:
                 case VINF_EM_DBG_STEP:
-                    if (enmOldState == EMSTATE_RAW)
-                    {
-                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_RAW));
-                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RAW;
-                    }
-                    else if (enmOldState == EMSTATE_HM)
+                    if (enmOldState == EMSTATE_HM)
                     {
                         Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_HM));
@@ -2498,8 +2413,8 @@
                         pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_NEM;
                     }
-                    else if (enmOldState == EMSTATE_REM)
-                    {
-                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_REM));
-                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
+                    else if (enmOldState == EMSTATE_RECOMPILER)
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_RECOMPILER));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RECOMPILER;
                     }
                     else
@@ -2569,14 +2484,13 @@
                     && (   (pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_ACTIVE)
                         || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_UNHALT))
-                    && (   enmNewState == EMSTATE_RAW
-                        || enmNewState == EMSTATE_HM
+                    && (   enmNewState == EMSTATE_HM
                         || enmNewState == EMSTATE_NEM
-                        || enmNewState == EMSTATE_REM
-                        || enmNewState == EMSTATE_IEM_THEN_REM
-                        || enmNewState == EMSTATE_DEBUG_GUEST_RAW
+                        || enmNewState == EMSTATE_RECOMPILER
                         || enmNewState == EMSTATE_DEBUG_GUEST_HM
                         || enmNewState == EMSTATE_DEBUG_GUEST_NEM
                         || enmNewState == EMSTATE_DEBUG_GUEST_IEM
-                        || enmNewState == EMSTATE_DEBUG_GUEST_REM) )
+                        || enmNewState == EMSTATE_DEBUG_GUEST_RECOMPILER
+                        /* Obsolete stuff: */
+                        || enmNewState == EMSTATE_IEM_THEN_REM ) )
                 {
                     if (pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_ACTIVE)
@@ -2604,12 +2518,4 @@
             {
                 /*
-                 * Execute raw.
-                 */
-                case EMSTATE_RAW:
-                    AssertLogRelMsgFailed(("%Rrc\n", rc));
-                    rc = VERR_EM_INTERNAL_ERROR;
-                    break;
-
-                /*
                  * Execute hardware accelerated raw.
                  */
@@ -2632,7 +2538,7 @@
                  * Execute recompiled.
                  */
-                case EMSTATE_REM:
-                    rc = emR3RemExecute(pVM, pVCpu, &fFFDone);
-                    Log2(("EMR3ExecuteVM: emR3RemExecute -> %Rrc\n", rc));
+                case EMSTATE_RECOMPILER:
+                    rc = VBOXSTRICTRC_TODO(emR3RecompilerExecute(pVM, pVCpu, &fFFDone));
+                    Log2(("EMR3ExecuteVM: emR3RecompilerExecute -> %Rrc\n", rc));
                     break;
 
@@ -2763,5 +2669,5 @@
                 case EMSTATE_DEBUG_GUEST_NEM:
                 case EMSTATE_DEBUG_GUEST_IEM:
-                case EMSTATE_DEBUG_GUEST_REM:
+                case EMSTATE_DEBUG_GUEST_RECOMPILER:
                     TMR3NotifySuspend(pVM, pVCpu);
                     rc = VBOXSTRICTRC_TODO(emR3Debug(pVM, pVCpu, rc));
@@ -2818,4 +2724,5 @@
                  */
                 case EMSTATE_NONE:
+                case EMSTATE_RAW_OBSOLETE:
                 case EMSTATE_TERMINATING:
                 default:
Index: trunk/src/VBox/VMM/VMMR3/EMHM.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/EMHM.cpp	(revision 99891)
+++ trunk/src/VBox/VMM/VMMR3/EMHM.cpp	(revision 99897)
@@ -58,4 +58,6 @@
 #include <iprt/asm.h>
 
+#include "EMInline.h"
+
 
 /*********************************************************************************************************************************
Index: trunk/src/VBox/VMM/VMMR3/EMR3Dbg.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/EMR3Dbg.cpp	(revision 99891)
+++ trunk/src/VBox/VMM/VMMR3/EMR3Dbg.cpp	(revision 99897)
@@ -44,5 +44,5 @@
  * Implements the '.alliem' command. }
  */
-static DECLCALLBACK(int) enmR3DbgCmdAllIem(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
+static DECLCALLBACK(int) emR3DbgCmdAllIem(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PUVM pUVM, PCDBGCVAR paArgs, unsigned cArgs)
 {
     int  rc;
@@ -76,5 +76,5 @@
 {
     {
-        "alliem", 0, 1, &g_BoolArg, 1, 0, enmR3DbgCmdAllIem, "[boolean]",
+        "alliem", 0, 1, &g_BoolArg, 1, 0, emR3DbgCmdAllIem, "[boolean]",
         "Enables or disabled executing ALL code in IEM, if no arguments are given it displays the current status."
     },
Index: trunk/src/VBox/VMM/VMMR3/EMR3Nem.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/EMR3Nem.cpp	(revision 99891)
+++ trunk/src/VBox/VMM/VMMR3/EMR3Nem.cpp	(revision 99897)
@@ -58,4 +58,6 @@
 #include <iprt/asm.h>
 
+#include "EMInline.h"
+
 
 /*********************************************************************************************************************************
Index: trunk/src/VBox/VMM/include/EMInternal.h
===================================================================
--- trunk/src/VBox/VMM/include/EMInternal.h	(revision 99891)
+++ trunk/src/VBox/VMM/include/EMInternal.h	(revision 99897)
@@ -130,11 +130,10 @@
     /** Whether IEM executes everything. */
     bool                    fIemExecutesAll;
+    /** Whether IEM execution (pure) is recompiled (true) or interpreted (false). */
+    bool                    fIemRecompiled;
     /** Whether a triple fault triggers a guru. */
     bool                    fGuruOnTripleFault;
     /** Alignment padding. */
-    bool                    afPadding[2];
-
-    /** Id of the VCPU that last executed code in the recompiler. */
-    VMCPUID                 idLastRemCpu;
+    bool                    afPadding[5];
 } EM;
 /** Pointer to EM VM instance data. */
@@ -224,12 +223,6 @@
     STAMPROFILEADV          StatNEMEntry;
     STAMPROFILE             StatNEMExec;
-    STAMPROFILE             StatREMEmu;
     STAMPROFILE             StatREMExec;
-    STAMPROFILE             StatREMSync;
-    STAMPROFILEADV          StatREMTotal;
-    STAMPROFILE             StatRAWExec;
-    STAMPROFILEADV          StatRAWEntry;
-    STAMPROFILEADV          StatRAWTail;
-    STAMPROFILEADV          StatRAWTotal;
+    STAMPROFILE             StatREMTotal;
     STAMPROFILEADV          StatTotal;
     /** @} */
@@ -326,7 +319,5 @@
 VBOXSTRICTRC    emR3NemSingleInstruction(PVM pVM, PVMCPU pVCpu, uint32_t fFlags);
 
-int             emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations);
-
-bool            emR3IsExecutionAllowed(PVM pVM, PVMCPU pVCpu);
+bool            emR3IsExecutionAllowedSlow(PVM pVM, PVMCPU pVCpu);
 
 VBOXSTRICTRC    emR3ExecutePendingIoPortWrite(PVM pVM, PVMCPU pVCpu);
Index: trunk/src/VBox/VMM/testcase/tstAnimate.cpp
===================================================================
--- trunk/src/VBox/VMM/testcase/tstAnimate.cpp	(revision 99891)
+++ trunk/src/VBox/VMM/testcase/tstAnimate.cpp	(revision 99897)
@@ -881,6 +881,5 @@
                     if (RT_SUCCESS(rc))
                     {
-                        rc = EMR3SetExecutionPolicy(pUVM, EMEXECPOLICY_RECOMPILE_RING0, true); AssertReleaseRC(rc);
-                        rc = EMR3SetExecutionPolicy(pUVM, EMEXECPOLICY_RECOMPILE_RING3, true); AssertReleaseRC(rc);
+                        rc = EMR3SetExecutionPolicy(pUVM, EMEXECPOLICY_IEM_ALL, true); AssertReleaseRC(rc);
                         DBGFR3Info(pUVM, "cpumguest", "verbose", NULL);
                         if (fPowerOn)
