Index: trunk/src/VBox/VMM/VBoxVMM.d
===================================================================
--- trunk/src/VBox/VMM/VBoxVMM.d	(revision 71221)
+++ trunk/src/VBox/VMM/VBoxVMM.d	(revision 71222)
@@ -58,4 +58,5 @@
     probe r0__vmm__return__to__ring3__rc(struct VMCPU *a_pVCpu, struct CPUMCTX *p_Ctx, int a_rc);
     probe r0__vmm__return__to__ring3__hm(struct VMCPU *a_pVCpu, struct CPUMCTX *p_Ctx, int a_rc);
+    probe r0__vmm__return__to__ring3__nem(struct VMCPU *a_pVCpu, struct CPUMCTX *p_Ctx, int a_rc);
 
 
Index: trunk/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h
===================================================================
--- trunk/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h	(revision 71221)
+++ trunk/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h	(revision 71222)
@@ -40,4 +40,7 @@
 NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
 
+/** HV_INTERCEPT_ACCESS_TYPE names. */
+static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" };
+
 
 /*********************************************************************************************************************************
@@ -67,5 +70,8 @@
     PGVM pGVM = GVMMR0FastGetGVMByVM(pVM);
     AssertReturn(pGVM, VERR_INVALID_VM_HANDLE);
-    return nemR0WinMapPages(pGVM, pVM, &pGVM->aCpus[pVCpu->idCpu], GCPhysSrc, GCPhysDst, 1, fFlags);
+    return nemR0WinMapPages(pGVM, pVM, &pGVM->aCpus[pVCpu->idCpu],
+                            GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                            GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                            1, fFlags);
 #else
     pVCpu->nem.s.Hypercall.MapPages.GCPhysSrc   = GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
@@ -91,5 +97,5 @@
     PGVM pGVM = GVMMR0FastGetGVMByVM(pVM);
     AssertReturn(pGVM, VERR_INVALID_VM_HANDLE);
-    return nemR0WinUnmapPages(pGVM, &pGVM->aCpus[pVCpu->idCpu], GCPhys, 1);
+    return nemR0WinUnmapPages(pGVM, &pGVM->aCpus[pVCpu->idCpu], GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, 1);
 # else
     pVCpu->nem.s.Hypercall.UnmapPages.GCPhys    = GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
@@ -100,11 +106,9 @@
 
 #endif /* NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
-
-
 #ifndef IN_RING0
 
 NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
 {
-#ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
+# ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
     NOREF(pCtx);
     int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPORT_STATE, 0, NULL);
@@ -112,5 +116,5 @@
     return rc;
 
-#else
+# else
     WHV_REGISTER_NAME  aenmNames[128];
     WHV_REGISTER_VALUE aValues[128];
@@ -157,5 +161,5 @@
 
     /* Segments */
-#define COPY_OUT_SEG(a_idx, a_enmName, a_SReg) \
+#  define COPY_OUT_SEG(a_idx, a_enmName, a_SReg) \
         do { \
             aenmNames[a_idx]                  = a_enmName; \
@@ -391,8 +395,8 @@
     Assert(iReg < RT_ELEMENTS(aValues));
     Assert(iReg < RT_ELEMENTS(aenmNames));
-#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+#  ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
     Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
            pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
-#endif
+#  endif
     HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
     if (SUCCEEDED(hrc))
@@ -402,5 +406,5 @@
                            hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
     return VERR_INTERNAL_ERROR;
-#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
 }
 
@@ -408,5 +412,5 @@
 NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint64_t fWhat)
 {
-#ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
+# ifdef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
     /* See NEMR0ImportState */
     NOREF(pCtx);
@@ -421,5 +425,5 @@
     return rc;
 
-#else
+# else
     WHV_REGISTER_NAME  aenmNames[128];
 
@@ -534,8 +538,8 @@
     Assert(RT_ELEMENTS(aValues) >= cRegs);
     Assert(RT_ELEMENTS(aenmNames) >= cRegs);
-#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+#  ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
     Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
           pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
-#endif
+#  endif
     HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues);
     if (SUCCEEDED(hrc))
@@ -567,5 +571,5 @@
 
         /* Segments */
-#define COPY_BACK_SEG(a_idx, a_enmName, a_SReg) \
+#  define COPY_BACK_SEG(a_idx, a_enmName, a_SReg) \
             do { \
                 Assert(aenmNames[a_idx] == a_enmName); \
@@ -774,6 +778,8 @@
                            hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
     return VERR_INTERNAL_ERROR;
-#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
-}
+# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
+}
+
+#endif /* !IN_RING0 */
 
 
@@ -1172,5 +1178,33 @@
 }
 
-#ifdef IN_RING3
+#ifdef IN_RING0
+/**
+ * Wrapper around nemR0WinImportState that converts VERR_NEM_CHANGE_PGM_MODE and
+ * VERR_NEM_FLUSH_TBL into informational status codes and logs+asserts statuses.
+ *
+ * @returns VBox strict status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   pGVCpu          The global (ring-0) per CPU structure.
+ * @param   pCtx            The CPU context to import into.
+ * @param   fWhat           What to import.
+ * @param   pszCaller       Whoe is doing the importing.
+ */
+DECLINLINE(VBOXSTRICTRC) nemR0WinImportStateStrict(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx, uint64_t fWhat, const char *pszCaller)
+{
+    int rc = nemR0WinImportState(pGVM, pGVCpu, pCtx, fWhat);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(rc == VINF_SUCCESS);
+        return VINF_SUCCESS;
+    }
+
+    if (rc == VERR_NEM_CHANGE_PGM_MODE || rc == VERR_NEM_FLUSH_TLB)
+    {
+        Log4(("%s/%u: nemR0WinImportState -> %Rrc\n", pszCaller, pGVCpu->idCpu, -rc));
+        return -rc;
+    }
+    AssertMsgFailedReturn(("%s/%u: nemR0WinImportState failed: %Rrc\n", pszCaller, pGVCpu->idCpu, rc), rc);
+}
+#endif /* IN_RING0 */
 
 /**
@@ -1201,8 +1235,13 @@
  * @param   pMsg            The message.
  * @param   pCtx            The register context.
+ * @param   pGVCpu          The global (ring-0) per CPU structure (NULL in r3).
  */
 NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageMemory(PVM pVM, PVMCPU pVCpu, HV_X64_MEMORY_INTERCEPT_MESSAGE const *pMsg,
-                                                         PCPUMCTX pCtx)
-{
+                                                         PCPUMCTX pCtx, PGVMCPU pGVCpu)
+{
+    Assert(   pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
+
     /*
      * Whatever we do, we must clear pending event ejection upon resume.
@@ -1226,19 +1265,22 @@
             if (State.fCanResume)
             {
-                Log4(("MemExit: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
+                Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
+                      pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
                       pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
                       Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
-                      State.fDidSomething ? "" : " no-change", g_apszWHvMemAccesstypes[pMsg->Header.InterceptAccessType]));
+                      State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
                 return VINF_SUCCESS;
             }
         }
-        Log4(("MemExit: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
+        Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
               pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
               Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
-              State.fDidSomething ? "" : " no-change", g_apszWHvMemAccesstypes[pMsg->Header.InterceptAccessType]));
+              State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
     }
     else
-        Log4(("MemExit: %RGp rc=%Rrc%s; emulating (%s)\n", pMsg->GuestPhysicalAddress, rc,
-              State.fDidSomething ? " modified-backing" : "", g_apszWHvMemAccesstypes[pMsg->Header.InterceptAccessType]));
+        Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating (%s)\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, pMsg->GuestPhysicalAddress, rc,
+              State.fDidSomething ? " modified-backing" : "", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
 
     /*
@@ -1246,8 +1288,15 @@
      */
     nemHCWinCopyStateFromX64Header(pCtx, &pMsg->Header);
+    VBOXSTRICTRC rcStrict;
+#ifdef IN_RING0
+    rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "MemExit");
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+#else
     rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
     AssertRCReturn(rc, rc);
-
-    VBOXSTRICTRC rcStrict;
+    NOREF(pGVCpu);
+#endif
+
     if (pMsg->InstructionByteCount > 0)
         rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(pCtx), pMsg->Header.Rip,
@@ -1268,11 +1317,14 @@
  * @param   pVCpu           The cross context per CPU structure.
  * @param   pMsg            The message.
+ * @param   pGVCpu          The global (ring-0) per CPU structure (NULL in r3).
  */
 NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageIoPort(PVM pVM, PVMCPU pVCpu, HV_X64_IO_PORT_INTERCEPT_MESSAGE const *pMsg,
-                                                         PCPUMCTX pCtx)
+                                                         PCPUMCTX pCtx, PGVMCPU pGVCpu)
 {
     Assert(   pMsg->AccessInfo.AccessSize == 1
            || pMsg->AccessInfo.AccessSize == 2
            || pMsg->AccessInfo.AccessSize == 4);
+    Assert(   pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
+           || pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
 
     /*
@@ -1294,6 +1346,7 @@
         {
             rcStrict = IOMIOPortWrite(pVM, pVCpu, pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize);
-            Log4(("IOExit: %04x:%08RX64: OUT %#x, %#x LB %u rcStrict=%Rrc\n", pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
-                  pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
+            Log4(("IOExit/%u: %04x:%08RX64: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
+                  pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, pMsg->PortNumber,
+                  (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
             if (IOM_SUCCESS(rcStrict))
             {
@@ -1306,6 +1359,6 @@
             uint32_t uValue = 0;
             rcStrict = IOMIOPortRead(pVM, pVCpu, pMsg->PortNumber, &uValue, pMsg->AccessInfo.AccessSize);
-            Log4(("IOExit: %04x:%08RX64: IN %#x LB %u -> %#x, rcStrict=%Rrc\n", pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
-                  pMsg->PortNumber, pMsg->AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+            Log4(("IOExit/%u: %04x:%08RX64: IN %#x LB %u -> %#x, rcStrict=%Rrc\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
+                  pMsg->Header.Rip, pMsg->PortNumber, pMsg->AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
             if (IOM_SUCCESS(rcStrict))
             {
@@ -1315,5 +1368,5 @@
                     pCtx->rax = uValue;
                 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RAX;
-                Log4(("IOExit: RAX %#RX64 -> %#RX64\n", pMsg->Rax, pCtx->rax));
+                Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pMsg->Rax, pCtx->rax));
                 nemHCWinCopyStateFromX64Header(pCtx, &pMsg->Header);
                 nemHCWinAdvanceGuestRipAndClearRF(pVCpu, pCtx, &pMsg->Header);
@@ -1346,8 +1399,16 @@
         pCtx->rdi = pMsg->Rdi;
         pCtx->rsi = pMsg->Rsi;
+#ifdef IN_RING0
+        rcStrict = nemR0WinImportStateStrict(pGVCpu->pGVM, pGVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+#else
         int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
         AssertRCReturn(rc, rc);
-
-        Log4(("IOExit: %04x:%08RX64: %s%s %#x LB %u (emulating)\n", pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
+        RT_NOREF(pGVCpu);
+#endif
+
+        Log4(("IOExit/%u: %04x:%08RX64: %s%s %#x LB %u (emulating)\n",
+              pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
               pMsg->AccessInfo.RepPrefix ? "REP " : "",
               pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUTS" : "INS",
@@ -1379,7 +1440,8 @@
  * @param   pMappingHeader  The message slot mapping.
  * @param   pCtx            The register context.
+ * @param   pGVCpu          The global (ring-0) per CPU structure (NULL in r3).
  */
 NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessage(PVM pVM, PVMCPU pVCpu, VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader,
-                                                   PCPUMCTX pCtx)
+                                                   PCPUMCTX pCtx, PGVMCPU pGVCpu)
 {
     if (pMappingHeader->enmVidMsgType == VidMessageHypervisorMessage)
@@ -1391,13 +1453,13 @@
             case HvMessageTypeUnmappedGpa:
                 Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
-                return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pCtx);
+                return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pCtx, pGVCpu);
 
             case HvMessageTypeGpaIntercept:
                 Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
-                return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pCtx);
+                return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept, pCtx, pGVCpu);
 
             case HvMessageTypeX64IoPortIntercept:
                 Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64IoPortIntercept));
-                return nemHCWinHandleMessageIoPort(pVM, pVCpu, &pMsg->X64IoPortIntercept, pCtx);
+                return nemHCWinHandleMessageIoPort(pVM, pVCpu, &pMsg->X64IoPortIntercept, pCtx, pGVCpu);
 
             case HvMessageTypeX64Halt:
@@ -1454,7 +1516,10 @@
  *                          exit.
  * @param   pMappingHeader  The message slot mapping.
+ * @param   pGVM            The global (ring-0) VM structure (NULL in r3).
+ * @param   pGVCpu          The global (ring-0) per CPU structure (NULL in r3).
  */
 NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinStopCpu(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict,
-                                             VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader)
+                                             VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader,
+                                             PGVM pGVM, PGVMCPU pGVCpu)
 {
     /*
@@ -1462,4 +1527,15 @@
      * does another VM exit.
      */
+#ifdef IN_RING0
+    pVCpu->nem.s.uIoCtlBuf.idCpu = pGVCpu->idCpu;
+    NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlStopVirtualProcessor.uFunction,
+                                            &pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
+                                            NULL, 0);
+    if (NT_SUCCESS(rcNt))
+    {
+        Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
+        return rcStrict;
+    }
+#else
     BOOL fRet = VidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu);
     if (fRet)
@@ -1468,11 +1544,18 @@
         return rcStrict;
     }
+    RT_NOREF(pGVM, pGVCpu);
+#endif
 
     /*
      * Dang. The CPU stopped by itself and we got a couple of message to deal with.
      */
+#ifdef IN_RING0
+    AssertLogRelMsgReturn(rcNt == ERROR_VID_STOP_PENDING, ("rcNt=%#x\n", rcNt),
+                          RT_SUCCESS(rcStrict) ?  VERR_INTERNAL_ERROR_3 : rcStrict);
+#else
     DWORD dwErr = RTNtLastErrorValue();
-    AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u\n", dwErr),
+    AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u (%#x)\n", dwErr, dwErr),
                           RT_SUCCESS(rcStrict) ?  VERR_INTERNAL_ERROR_3 : rcStrict);
+#endif
     Log8(("nemHCWinStopCpu: Stopping CPU pending...\n"));
 
@@ -1481,9 +1564,20 @@
      * Note! We can safely ASSUME that rcStrict isn't an important information one.
      */
+#ifdef IN_RING0
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu     = pGVCpu->idCpu;
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags   = VID_MSHAGN_F_GET_NEXT_MESSAGE;
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = 30000; /*ms*/
+    rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
+                                   &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
+                                   sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
+                                   NULL, 0);
+    AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
+                          RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
+#else
     BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
                                                      VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /*ms*/);
-    AssertLogRelMsgReturn(fWait,
-                          ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
+    AssertLogRelMsgReturn(fWait, ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
                           RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
+#endif
 
     /* It should be a hypervisor message and definitely not a stop request completed message. */
@@ -1494,5 +1588,5 @@
                           RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
 
-    VBOXSTRICTRC rcStrict2 = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, CPUMQueryGuestCtxPtr(pVCpu));
+    VBOXSTRICTRC rcStrict2 = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, CPUMQueryGuestCtxPtr(pVCpu), pGVCpu);
     if (rcStrict2 != VINF_SUCCESS && RT_SUCCESS(rcStrict))
         rcStrict = rcStrict2;
@@ -1502,9 +1596,20 @@
      * that as handled too.  CPU is back into fully stopped stated then.
      */
+#ifdef IN_RING0
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu     = pGVCpu->idCpu;
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags   = VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE;
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = 30000; /*ms*/
+    rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
+                                   &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
+                                   sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
+                                   NULL, 0);
+    AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("2st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
+                          RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
+#else
     fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
                                                 VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /*ms*/);
-    AssertLogRelMsgReturn(fWait,
-                          ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
+    AssertLogRelMsgReturn(fWait, ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
                           RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
+#endif
 
     /* It should be a stop request completed message. */
@@ -1516,9 +1621,19 @@
 
     /* Mark this as handled. */
-    fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
-                                                VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /*ms*/);
-    AssertLogRelMsgReturn(fWait,
-                          ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
+#ifdef IN_RING0
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu     = pGVCpu->idCpu;
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags   = VID_MSHAGN_F_HANDLE_MESSAGE;
+    pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = 30000; /*ms*/
+    rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
+                                   &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
+                                   sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
+                                   NULL, 0);
+    AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
                           RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
+#else
+    fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /*ms*/);
+    AssertLogRelMsgReturn(fWait, ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
+                          RT_SUCCESS(rcStrict) ? VERR_INTERNAL_ERROR_3 : rcStrict);
+#endif
     Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict) ));
     return rcStrict;
@@ -1526,8 +1641,8 @@
 
 
-NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVM pVM, PVMCPU pVCpu)
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVM pVM, PVMCPU pVCpu, PGVM pGVM, PGVMCPU pGVCpu)
 {
     PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
-    LogFlow(("nemHCWinRunGC: Entering #%u cs:rip=%04x:%08RX64 efl=%#08RX64\n", pVCpu->idCpu, pCtx->cs.Sel, pCtx->rip, pCtx->rflags));
+    LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pCtx->cs.Sel, pCtx->rip, pCtx->rflags));
 #ifdef LOG_ENABLED
     if (LogIs3Enabled())
@@ -1552,5 +1667,10 @@
         if ((pCtx->fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK)) != (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK))
         {
+#ifdef IN_RING0
+            int rc2 = nemR0WinExportState(pGVM, pGVCpu, pCtx);
+#else
             int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu, pCtx);
+            RT_NOREF(pGVM, pGVCpu);
+#endif
             AssertRCReturn(rc2, rc2);
         }
@@ -1566,24 +1686,52 @@
             else
             {
-                if (g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu))
-                    pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
-                else
-                    AssertLogRelMsgFailedReturn(("VidStartVirtualProcessor failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
-                                                 pVCpu->idCpu, RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()),
-                                                VERR_INTERNAL_ERROR_3);
+#ifdef IN_RING0
+                pVCpu->nem.s.uIoCtlBuf.idCpu = pGVCpu->idCpu;
+                NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlStartVirtualProcessor.uFunction,
+                                                        &pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
+                                                        NULL, 0);
+                LogFlow(("NEM/%u: IoCtlStartVirtualProcessor -> %#x\n", pVCpu->idCpu, rcNt));
+                AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("VidStartVirtualProcessor failed for CPU #%u: %#x\n", pGVCpu->idCpu, rcNt),
+                                      VERR_INTERNAL_ERROR_3);
+#else
+                AssertLogRelMsgReturn(g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
+                                      ("VidStartVirtualProcessor failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
+                                       pVCpu->idCpu, RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()),
+                                      VERR_INTERNAL_ERROR_3);
+#endif
+                pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
             }
 
-            if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
+            if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED))
             {
+#ifdef IN_RING0
+                pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu     = pGVCpu->idCpu;
+                pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags   = pVCpu->nem.s.fHandleAndGetFlags;
+                pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
+                NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVM->nem.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
+                                                        &pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
+                                                        sizeof(pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext),
+                                                        NULL, 0);
+                VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
+                if (rcNt == STATUS_SUCCESS)
+#else
                 BOOL fRet = VidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
                                                            pVCpu->nem.s.fHandleAndGetFlags, cMillies);
-                VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM);
+                VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
                 if (fRet)
+#endif
                 {
                     /*
                      * Deal with the message.
                      */
-                    rcStrict = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, pCtx);
+                    rcStrict = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader, pCtx, pGVCpu);
                     pVCpu->nem.s.fHandleAndGetFlags |= VID_MSHAGN_F_HANDLE_MESSAGE;
+                    if (rcStrict == VINF_SUCCESS)
+                    { /* hopefully likely */ }
+                    else
+                    {
+                        LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                        break;
+                    }
                 }
                 else
@@ -1592,13 +1740,14 @@
                        so after NtAlertThread we end up here with a STATUS_TIMEOUT.  And yeah,
                        the error code conversion is into WAIT_XXX, i.e. NT status codes. */
-                    DWORD dwErr = GetLastError();
-                    if (   dwErr == STATUS_TIMEOUT
-                        || dwErr == STATUS_ALERTED  /* just in case */
-                        || dwErr == STATUS_USER_APC /* ditto */ )
-                        pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE; /* exits are likely */
-                    else
-                        AssertLogRelMsgFailedReturn(("VidMessageSlotHandleAndGetNext failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
-                                                     pVCpu->idCpu, dwErr, dwErr, RTNtLastStatusValue()),
-                                                    VERR_INTERNAL_ERROR_3);
+#ifndef IN_RING0
+                    DWORD rcNt = GetLastError();
+#endif
+                    LogFlow(("NEM/%u: VidMessageSlotHandleAndGetNext -> %#x\n", pVCpu->idCpu, rcNt));
+                    AssertLogRelMsgReturn(   rcNt == STATUS_TIMEOUT
+                                          || rcNt == STATUS_ALERTED  /* just in case */
+                                          || rcNt == STATUS_USER_APC /* ditto */
+                                          , ("VidMessageSlotHandleAndGetNext failed for CPU #%u: %#x (%u)\n", pVCpu->idCpu, rcNt, rcNt),
+                                          VERR_INTERNAL_ERROR_3);
+                    pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE; /* exits are likely */
                 }
 
@@ -1612,11 +1761,12 @@
                 /** @todo Try handle pending flags, not just return to EM loops.  Take care
                  *        not to set important RCs here unless we've handled a message. */
-                LogFlow(("nemHCWinRunGC: returning: pending FF (%#x / %#x)\n", pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
+                LogFlow(("NEM/%u: breaking: pending FF (%#x / %#x)\n",
+                         pVCpu->idCpu, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
             }
             else
-                LogFlow(("nemHCWinRunGC: returning: canceled %d (pre exec)\n", VMCPU_GET_STATE(pVCpu) ));
+                LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
         }
         else
-            LogFlow(("nemHCWinRunGC: returning: pending FF (pre exec)\n"));
+            LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
         break;
     } /* the run loop */
@@ -1630,5 +1780,5 @@
     {
         pVCpu->nem.s.fHandleAndGetFlags = 0;
-        rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader);
+        rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader, pGVM, pGVCpu);
     }
 
@@ -1637,7 +1787,24 @@
     if (pCtx->fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
     {
+#ifdef IN_RING0
+        int rc2 = nemR0WinImportState(pGVM, pGVCpu, pCtx, CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK);
+        if (RT_SUCCESS(rc2))
+            pCtx->fExtrn = 0;
+        else if (rc2 == VERR_NEM_CHANGE_PGM_MODE || rc2 == VERR_NEM_FLUSH_TLB)
+        {
+            pCtx->fExtrn = 0;
+            if (rcStrict == VINF_SUCCESS || rcStrict == -rc2)
+                rcStrict = -rc2;
+            else
+            {
+                pVCpu->nem.s.rcPgmPending = -rc2;
+                LogFlow(("NEM/%u: rcPgmPending=%Rrc (rcStrict=%Rrc)\n", pVCpu->idCpu, rc2, VBOXSTRICTRC_VAL(rcStrict) ));
+            }
+        }
+#else
         int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, pCtx, CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK);
         if (RT_SUCCESS(rc2))
             pCtx->fExtrn = 0;
+#endif
         else if (RT_SUCCESS(rcStrict))
             rcStrict = rc2;
@@ -1646,10 +1813,9 @@
         pCtx->fExtrn = 0;
 
-    LogFlow(("nemHCWinRunGC: Leaving #%u cs:rip=%04x:%08RX64 efl=%#08RX64\n", pVCpu->idCpu, pCtx->cs.Sel, pCtx->rip, pCtx->rflags));
+    LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
+             pVCpu->idCpu, pCtx->cs.Sel, pCtx->rip, pCtx->rflags, VBOXSTRICTRC_VAL(rcStrict) ));
     return rcStrict;
 }
-#endif
-
-#endif /* IN_RING0 */
+
 
 
Index: trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp	(revision 71222)
@@ -943,6 +943,7 @@
                                 if (RT_SUCCESS(rc))
                                 {
-                                    pVM->pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
-                                    AssertPtr((void *)pVM->pVMR3);
+                                    PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
+                                    pVM->pVMR3 = pVMR3;
+                                    AssertPtr((void *)pVMR3);
 
                                     /* Initialize all the VM pointers. */
@@ -950,5 +951,5 @@
                                     {
                                         pVM->aCpus[i].pVMR0           = pVM;
-                                        pVM->aCpus[i].pVMR3           = pVM->pVMR3;
+                                        pVM->aCpus[i].pVMR3           = pVMR3;
                                         pVM->aCpus[i].idHostCpu       = NIL_RTCPUID;
                                         pVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
@@ -972,7 +973,14 @@
                                         pHandle->ProcId               = ProcId;
                                         pGVM->pVM                     = pVM;
+                                        pGVM->pVMR3                   = pVMR3;
                                         pGVM->aCpus[0].hEMT           = hEMT0;
                                         pVM->aCpus[0].hNativeThreadR0 = hEMT0;
                                         pGVMM->cEMTs += cCpus;
+
+                                        for (uint32_t i = 0; i < cCpus; i++)
+                                        {
+                                            pGVM->aCpus[i].pVCpu          = &pVM->aCpus[i];
+                                            pGVM->aCpus[i].pVM            = pVM;
+                                        }
 
                                         /* Associate it with the session and create the context hook for EMT0. */
@@ -994,5 +1002,5 @@
 
                                                 *ppVM = pVM;
-                                                Log(("GVMMR0CreateVM: pVM=%p pVMR3=%p pGVM=%p hGVM=%d\n", pVM, pVM->pVMR3, pGVM, iHandle));
+                                                Log(("GVMMR0CreateVM: pVM=%p pVMR3=%p pGVM=%p hGVM=%d\n", pVM, pVMR3, pGVM, iHandle));
                                                 return VINF_SUCCESS;
                                             }
@@ -1063,4 +1071,7 @@
         pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
         pGVM->aCpus[i].hEMT                  = NIL_RTNATIVETHREAD;
+        pGVM->aCpus[i].pGVM                  = pGVM;
+        pGVM->aCpus[i].pVCpu                 = NULL;
+        pGVM->aCpus[i].pVM                   = NULL;
     }
 }
@@ -1186,12 +1197,12 @@
 
     /* Be careful here because we might theoretically be racing someone else cleaning up. */
-    if (    pHandle->pVM == pVM
-        &&  (   (   pHandle->hEMT0  == hSelf
-                 && pHandle->ProcId == ProcId)
-             || pHandle->hEMT0 == NIL_RTNATIVETHREAD)
-        &&  VALID_PTR(pHandle->pvObj)
-        &&  VALID_PTR(pHandle->pSession)
-        &&  VALID_PTR(pHandle->pGVM)
-        &&  pHandle->pGVM->u32Magic == GVM_MAGIC)
+    if (   pHandle->pVM == pVM
+        && (   (   pHandle->hEMT0  == hSelf
+                && pHandle->ProcId == ProcId)
+            || pHandle->hEMT0 == NIL_RTNATIVETHREAD)
+        && VALID_PTR(pHandle->pvObj)
+        && VALID_PTR(pHandle->pSession)
+        && VALID_PTR(pHandle->pGVM)
+        && pHandle->pGVM->u32Magic == GVM_MAGIC)
     {
         /* Check that other EMTs have deregistered. */
@@ -1350,6 +1361,6 @@
      */
     PGVM pGVM = pHandle->pGVM;
-    if (    VALID_PTR(pGVM)
-        &&  pGVM->u32Magic == GVM_MAGIC)
+    if (   VALID_PTR(pGVM)
+        && pGVM->u32Magic == GVM_MAGIC)
     {
         pGVMM->cEMTs -= pGVM->cCpus;
Index: trunk/src/VBox/VMM/VMMR0/NEMR0Native-win.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR0/NEMR0Native-win.cpp	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR0/NEMR0Native-win.cpp	(revision 71222)
@@ -24,4 +24,5 @@
 #include <iprt/nt/hyperv.h>
 #include <iprt/nt/vid.h>
+#include <winerror.h>
 
 #include <VBox/vmm/nem.h>
@@ -50,4 +51,10 @@
 
 /*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+typedef uint32_t DWORD; /* for winerror.h constants */
+
+
+/*********************************************************************************************************************************
 *   Global Variables                                                                                                             *
 *********************************************************************************************************************************/
@@ -58,7 +65,11 @@
 *   Internal Functions                                                                                                           *
 *********************************************************************************************************************************/
-NEM_TMPL_STATIC int nemR0WinMapPages(PGVM pGVM, PVM pVM, PGVMCPU pGVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
-                                     uint32_t cPages, uint32_t fFlags);
-NEM_TMPL_STATIC int nemR0WinUnmapPages(PGVM pGVM, PGVMCPU pGVCpu, RTGCPHYS GCPhys, uint32_t cPages);
+NEM_TMPL_STATIC int  nemR0WinMapPages(PGVM pGVM, PVM pVM, PGVMCPU pGVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
+                                      uint32_t cPages, uint32_t fFlags);
+NEM_TMPL_STATIC int  nemR0WinUnmapPages(PGVM pGVM, PGVMCPU pGVCpu, RTGCPHYS GCPhys, uint32_t cPages);
+NEM_TMPL_STATIC int  nemR0WinExportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx);
+NEM_TMPL_STATIC int  nemR0WinImportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx, uint64_t fWhat);
+DECLINLINE(NTSTATUS) nemR0NtPerformIoControl(PGVM pGVM, uint32_t uFunction, void *pvInput, uint32_t cbInput,
+                                                     void *pvOutput, uint32_t cbOutput);
 
 
@@ -497,5 +508,5 @@
  * @param   fWhat       What to export. To be defined, UINT64_MAX for now.
  */
-static int nemR0WinExportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx)
+NEM_TMPL_STATIC int nemR0WinExportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx)
 {
     PVMCPU                     pVCpu  = &pGVM->pVM->aCpus[pGVCpu->idCpu];
@@ -1075,5 +1086,5 @@
  * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX.
  */
-static int nemR0WinImportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx, uint64_t fWhat)
+NEM_TMPL_STATIC int nemR0WinImportState(PGVM pGVM, PGVMCPU pGVCpu, PCPUMCTX pCtx, uint64_t fWhat)
 {
     HV_INPUT_GET_VP_REGISTERS *pInput = (HV_INPUT_GET_VP_REGISTERS *)pGVCpu->nem.s.pbHypercallData;
@@ -1694,4 +1705,11 @@
     pCtx->fExtrn &= ~fWhat;
 
+    /* Typical. */
+    if (!fMaybeChangedMode && !fFlushTlb)
+        return VINF_SUCCESS;
+
+    /*
+     * Slow.
+     */
     int rc = VINF_SUCCESS;
     if (fMaybeChangedMode)
@@ -1699,10 +1717,16 @@
         rc = PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
         if (rc == VINF_PGM_CHANGE_MODE)
-            rc = VERR_NEM_CHANGE_PGM_MODE;
-        else
-            AssertRC(rc);
-    }
-    if (fFlushTlb && rc == VINF_SUCCESS)
+        {
+            LogFlow(("nemR0WinImportState: -> VERR_NEM_CHANGE_PGM_MODE!\n"));
+            return VERR_NEM_CHANGE_PGM_MODE;
+        }
+        AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
+    }
+
+    if (fFlushTlb)
+    {
+        LogFlow(("nemR0WinImportState: -> VERR_NEM_FLUSH_TLB!\n"));
         rc = VERR_NEM_FLUSH_TLB; /* Calling PGMFlushTLB w/o long jump setup doesn't work, ring-3 does it. */
+    }
 
     return rc;
@@ -1741,2 +1765,9 @@
 }
 
+
+VMMR0_INT_DECL(VBOXSTRICTRC) NEMR0RunGuestCode(PGVM pGVM, VMCPUID idCpu)
+{
+    PVM pVM = pGVM->pVM;
+    return nemHCWinRunGC(pVM, &pVM->aCpus[idCpu], pGVM, &pGVM->aCpus[idCpu]);
+}
+
Index: trunk/src/VBox/VMM/VMMR0/VMMR0.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR0/VMMR0.cpp	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR0/VMMR0.cpp	(revision 71222)
@@ -506,4 +506,38 @@
 
 /**
+ * Does EMT specific VM initialization.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The ring-0 VM structure.
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       The EMT that's calling.
+ */
+static int vmmR0InitVMEmt(PGVM pGVM, PVM pVM, VMCPUID idCpu)
+{
+    /* Paranoia (caller checked these already). */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
+
+#ifdef LOG_ENABLED
+    /*
+     * Registration of ring 0 loggers.
+     */
+    PVMCPU       pVCpu     = &pVM->aCpus[idCpu];
+    PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
+    if (   pR0Logger
+        && !pR0Logger->fRegistered)
+    {
+        RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
+        pR0Logger->fRegistered = true;
+    }
+#endif
+    RT_NOREF(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
  * Terminates the R0 bits for a particular VM instance.
  *
@@ -1167,20 +1201,4 @@
                 VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
 
-#ifdef LOG_ENABLED
-                /*
-                 * Ugly: Lazy registration of ring 0 loggers.
-                 */
-                if (pVCpu->idCpu > 0)
-                {
-                    PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
-                    if (   pR0Logger
-                        && RT_UNLIKELY(!pR0Logger->fRegistered))
-                    {
-                        RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
-                        pR0Logger->fRegistered = true;
-                    }
-                }
-#endif
-
 #ifdef VMM_R0_TOUCH_FPU
                 /*
@@ -1321,4 +1339,27 @@
         }
 
+        case VMMR0_DO_NEM_RUN:
+        {
+            /*
+             * Setup the longjmp machinery and execute guest code (calls NEMR0RunGuestCode).
+             */
+            VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
+            int rc = vmmR0CallRing3SetJmp2(&pVCpu->vmm.s.CallRing3JmpBufR0, NEMR0RunGuestCode, pGVM, idCpu);
+            VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
+            STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
+
+            pVCpu->vmm.s.iLastGZRc = rc;
+
+            /*
+             * Fire dtrace probe and collect statistics.
+             */
+            VBOXVMM_R0_VMM_RETURN_TO_RING3_NEM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
+#ifdef VBOX_WITH_STATISTICS
+            vmmR0RecordRC(pVM, pVCpu, rc);
+#endif
+            break;
+        }
+
+
         /*
          * For profiling.
@@ -1539,4 +1580,12 @@
         case VMMR0_DO_VMMR0_INIT:
             rc = vmmR0InitVM(pGVM, pVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
+            VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
+            break;
+
+        /*
+         * Does EMT specific ring-0 init.
+         */
+        case VMMR0_DO_VMMR0_INIT_EMT:
+            rc = vmmR0InitVMEmt(pGVM, pVM, idCpu);
             VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
             break;
Index: trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm
===================================================================
--- trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm	(revision 71222)
@@ -56,4 +56,5 @@
 ;
 BEGINPROC vmmR0CallRing3SetJmp
+GLOBALNAME vmmR0CallRing3SetJmp2
 GLOBALNAME vmmR0CallRing3SetJmpEx
     ;
Index: trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm
===================================================================
--- trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-x86.asm	(revision 71222)
@@ -53,4 +53,5 @@
 ;
 BEGINPROC vmmR0CallRing3SetJmp
+GLOBALNAME vmmR0CallRing3SetJmp2
 GLOBALNAME vmmR0CallRing3SetJmpEx
     ;
Index: trunk/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp	(revision 71222)
@@ -1914,11 +1914,36 @@
 #ifndef NEM_WIN_USE_OUR_OWN_RUN_API
     return nemR3WinWHvRunGC(pVM, pVCpu);
-#elif 1
-    return nemHCWinRunGC(pVM, pVCpu);
+#elif 0
+    return nemHCWinRunGC(pVM, pVCpu, NULL /*pGVM*/, NULL /*pGVCpu*/);
 #else
-    int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_RUN_GC, 0, NULL);
-    if (RT_SUCCESS(rc))
-        return pVCpu->nem.s.rcRing0;
-    return rc;
+    VBOXSTRICTRC rcStrict = VMMR3CallR0EmtFast(pVM, pVCpu, VMMR0_DO_NEM_RUN);
+    if (RT_SUCCESS(rcStrict))
+    {
+        /* We deal wtih VINF_NEM_CHANGE_PGM_MODE and VINF_NEM_FLUSH_TLB here, since we're running
+           the risk of getting these while we already got another RC (I/O ports). */
+        VBOXSTRICTRC rcPgmPending = pVCpu->nem.s.rcPgmPending;
+        pVCpu->nem.s.rcPgmPending = VINF_SUCCESS;
+        if (   rcStrict == VINF_NEM_CHANGE_PGM_MODE
+            || rcStrict == VINF_PGM_CHANGE_MODE
+            || rcPgmPending == VINF_NEM_CHANGE_PGM_MODE )
+        {
+            LogFlow(("nemR3NativeRunGC: calling PGMChangeMode...\n"));
+            int rc = PGMChangeMode(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR4(pVCpu), CPUMGetGuestEFER(pVCpu));
+            AssertRCReturn(rc, rc);
+            if (rcStrict == VINF_NEM_CHANGE_PGM_MODE || rcStrict == VINF_NEM_FLUSH_TLB)
+                rcStrict = VINF_SUCCESS;
+        }
+        else if (rcStrict == VINF_NEM_FLUSH_TLB || rcPgmPending == VINF_NEM_FLUSH_TLB)
+        {
+            LogFlow(("nemR3NativeRunGC: calling PGMFlushTLB...\n"));
+            int rc = PGMFlushTLB(pVCpu, CPUMGetGuestCR3(pVCpu), true);
+            AssertRCReturn(rc, rc);
+            if (rcStrict == VINF_NEM_FLUSH_TLB || rcStrict == VINF_NEM_CHANGE_PGM_MODE)
+                rcStrict = VINF_SUCCESS;
+        }
+        else
+            AssertMsg(rcPgmPending == VINF_SUCCESS, ("rcPgmPending=%Rrc\n", VBOXSTRICTRC_VAL(rcPgmPending) ));
+    }
+    return rcStrict;
 #endif
 }
Index: trunk/src/VBox/VMM/VMMR3/PGM.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/PGM.cpp	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR3/PGM.cpp	(revision 71222)
@@ -3504,7 +3504,15 @@
     }
     /* Override the shadow mode is nested paging is active. */
-    pVM->pgm.s.fNestedPaging = HMIsNestedPagingActive(pVM);
-    if (pVM->pgm.s.fNestedPaging)
-        enmShadowMode = HMGetShwPagingMode(pVM);
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        pVM->pgm.s.fNestedPaging = true;
+        enmShadowMode = PGMMODE_NESTED;
+    }
+    else
+    {
+        pVM->pgm.s.fNestedPaging = HMIsNestedPagingActive(pVM);
+        if (pVM->pgm.s.fNestedPaging)
+            enmShadowMode = HMGetShwPagingMode(pVM);
+    }
 
     *penmSwitcher = enmSwitcher;
Index: trunk/src/VBox/VMM/VMMR3/PGMBth.h
===================================================================
--- trunk/src/VBox/VMM/VMMR3/PGMBth.h	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR3/PGMBth.h	(revision 71222)
@@ -135,5 +135,5 @@
     PVM pVM = pVCpu->pVMR3;
 
-    Assert(HMIsNestedPagingActive(pVM) == pVM->pgm.s.fNestedPaging);
+    Assert((HMIsNestedPagingActive(pVM) || VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
     Assert(!pVM->pgm.s.fNestedPaging);
 
Index: trunk/src/VBox/VMM/VMMR3/PGMShw.h
===================================================================
--- trunk/src/VBox/VMM/VMMR3/PGMShw.h	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR3/PGMShw.h	(revision 71222)
@@ -191,5 +191,5 @@
     PVM          pVM       = pVCpu->pVMR3;
 
-    Assert(HMIsNestedPagingActive(pVM) == pVM->pgm.s.fNestedPaging);
+    Assert((HMIsNestedPagingActive(pVM) || VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
     Assert(pVM->pgm.s.fNestedPaging);
     Assert(!pVCpu->pgm.s.pShwPageCR3R3);
Index: trunk/src/VBox/VMM/VMMR3/VMM.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMR3/VMM.cpp	(revision 71221)
+++ trunk/src/VBox/VMM/VMMR3/VMM.cpp	(revision 71222)
@@ -528,4 +528,18 @@
 
 /**
+ * Worker for VMMR3InitR0 that calls ring-0 to do EMT specific initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context per CPU structure.
+ * @thread  EMT(pVCpu)
+ */
+static DECLCALLBACK(int) vmmR3InitR0Emt(PVM pVM, PVMCPU pVCpu)
+{
+    return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_VMMR0_INIT_EMT, 0, NULL);
+}
+
+
+/**
  * Initializes the R0 VMM.
  *
@@ -561,6 +575,5 @@
         rc = VINF_SUCCESS;
 #else
-        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_VMMR0_INIT,
-                              RT_MAKE_U64(VMMGetSvnRev(), vmmGetBuildType()), NULL);
+        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_VMMR0_INIT, RT_MAKE_U64(VMMGetSvnRev(), vmmGetBuildType()), NULL);
 #endif
         /*
@@ -592,4 +605,10 @@
     else
         LogRel(("VMM: Thread-context hooks unavailable\n"));
+
+    /*
+     * Send all EMTs to ring-0 to get their logger initialized.
+     */
+    for (VMCPUID idCpu = 0; RT_SUCCESS(rc) && idCpu < pVM->cCpus; idCpu++)
+        rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)vmmR3InitR0Emt, 2, pVM, &pVM->aCpus[idCpu]);
 
     return rc;
@@ -1431,4 +1450,47 @@
         }
         rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+        /* Resume R0 */
+    }
+}
+
+
+/**
+ * Perform one of the fast I/O control VMMR0 operation.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmOperation    The operation to perform.
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) VMMR3CallR0EmtFast(PVM pVM, PVMCPU pVCpu, VMMR0OPERATION enmOperation)
+{
+    for (;;)
+    {
+        VBOXSTRICTRC rcStrict;
+        do
+        {
+#ifdef NO_SUPCALLR0VMM
+            rcStrict = VERR_GENERAL_FAILURE;
+#else
+            rcStrict = SUPR3CallVMMR0Fast(pVM->pVMR0, enmOperation, pVCpu->idCpu);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                rcStrict = pVCpu->vmm.s.iLastGZRc;
+#endif
+        } while (rcStrict == VINF_EM_RAW_INTERRUPT_HYPER);
+
+#ifdef LOG_ENABLED
+        /*
+         * Flush the log
+         */
+        PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
+        if (    pR0LoggerR3
+            &&  pR0LoggerR3->Logger.offScratch > 0)
+            RTLogFlushR0(NULL, &pR0LoggerR3->Logger);
+#endif /* !LOG_ENABLED */
+        if (rcStrict != VINF_VMM_CALL_HOST)
+            return rcStrict;
+        int rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
         if (RT_FAILURE(rc))
             return rc;
Index: trunk/src/VBox/VMM/include/NEMInternal.h
===================================================================
--- trunk/src/VBox/VMM/include/NEMInternal.h	(revision 71221)
+++ trunk/src/VBox/VMM/include/NEMInternal.h	(revision 71222)
@@ -169,4 +169,6 @@
 #ifdef RT_OS_WINDOWS
 # ifdef NEM_WIN_USE_OUR_OWN_RUN_API
+    /** Pending VERR_NEM_CHANGE_PGM_MODE or VERR_NEM_FLUSH_TLB. */
+    int32_t                     rcPgmPending;
     /** The VID_MSHAGN_F_XXX flags.
      * Either VID_MSHAGN_F_HANDLE_MESSAGE | VID_MSHAGN_F_GET_NEXT_MESSAGE or zero. */
@@ -201,4 +203,8 @@
         uint8_t                 ab[64];
         HV_PARTITION_ID         idPartition;
+        HV_VP_INDEX             idCpu;
+# ifdef VID_MSHAGN_F_GET_NEXT_MESSAGE
+        VID_IOCTL_INPUT_MESSAGE_SLOT_HANDLE_AND_GET_NEXT MsgSlotHandleAndGetNext;
+# endif
     } uIoCtlBuf;
 #endif
Index: trunk/src/VBox/VMM/include/VMMInternal.h
===================================================================
--- trunk/src/VBox/VMM/include/VMMInternal.h	(revision 71221)
+++ trunk/src/VBox/VMM/include/VMMInternal.h	(revision 71222)
@@ -610,12 +610,35 @@
 DECLASM(int)    vmmR0CallRing3SetJmp(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP pfn, PVM pVM, PVMCPU pVCpu);
 
-/**
- * Callback function for vmmR0CallRing3SetJmpEx.
+
+/**
+ * Callback function for vmmR0CallRing3SetJmp2.
  *
  * @returns VBox status code.
  * @param   pvUser      The user argument.
  */
+typedef DECLCALLBACK(int) FNVMMR0SETJMP2(PGVM pGVM, VMCPUID idCpu);
+/** Pointer to FNVMMR0SETJMP2(). */
+typedef FNVMMR0SETJMP2 *PFNVMMR0SETJMP2;
+
+/**
+ * Same as vmmR0CallRing3SetJmp except for the function signature.
+ *
+ * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+ * @param   pJmpBuf     The jmp_buf to set.
+ * @param   pfn         The function to be called when not resuming.
+ * @param   pGVM        The ring-0 VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ */
+DECLASM(int)    vmmR0CallRing3SetJmp2(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP2 pfn, PGVM pGVM, VMCPUID idCpu);
+
+
+/**
+ * Callback function for vmmR0CallRing3SetJmpEx.
+ *
+ * @returns VBox status code.
+ * @param   pvUser      The user argument.
+ */
 typedef DECLCALLBACK(int) FNVMMR0SETJMPEX(void *pvUser);
-/** Pointer to FNVMMR0SETJMP(). */
+/** Pointer to FNVMMR0SETJMPEX(). */
 typedef FNVMMR0SETJMPEX *PFNVMMR0SETJMPEX;
 
