Index: /trunk/include/VBox/mm.h
===================================================================
--- /trunk/include/VBox/mm.h	(revision 6795)
+++ /trunk/include/VBox/mm.h	(revision 6796)
@@ -340,11 +340,13 @@
  */
 
+MMR3DECL(int)       MMR3InitUVM(PUVM pUVM);
 MMR3DECL(int)       MMR3Init(PVM pVM);
 MMR3DECL(int)       MMR3InitPaging(PVM pVM);
 MMR3DECL(int)       MMR3HyperInitFinalize(PVM pVM);
 MMR3DECL(int)       MMR3Term(PVM pVM);
+MMR3DECL(void)      MMR3TermUVM(PUVM pUVM);
 MMR3DECL(void)      MMR3Reset(PVM pVM);
 MMR3DECL(int)       MMR3IncreaseBaseReservation(PVM pVM, uint64_t cAddBasePages);
-MMR3DECL(int)       MMR3IncreateFixedReservation(PVM pVM, uint32_t cAddFixedPages);
+MMR3DECL(int)       MMR3IncreaseFixedReservation(PVM pVM, uint32_t cAddFixedPages);
 MMR3DECL(int)       MMR3UpdateShadowReservation(PVM pVM, uint32_t cShadowPages);
 
@@ -404,9 +406,14 @@
  * @{ */
 MMR3DECL(void *)    MMR3HeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize);
+MMR3DECL(void *)    MMR3HeapAllocU(PUVM pUVM, MMTAG enmTag, size_t cbSize);
 MMR3DECL(int)       MMR3HeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv);
+MMR3DECL(int)       MMR3HeapAllocExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv);
 MMR3DECL(void *)    MMR3HeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize);
+MMR3DECL(void *)    MMR3HeapAllocZU(PUVM pUVM, MMTAG enmTag, size_t cbSize);
 MMR3DECL(int)       MMR3HeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv);
+MMR3DECL(int)       MMR3HeapAllocZExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv);
 MMR3DECL(void *)    MMR3HeapRealloc(void *pv, size_t cbNewSize);
 MMR3DECL(char *)    MMR3HeapStrDup(PVM pVM, MMTAG enmTag, const char *psz);
+MMR3DECL(char *)    MMR3HeapStrDupU(PUVM pUVM, MMTAG enmTag, const char *psz);
 MMR3DECL(void)      MMR3HeapFree(void *pv);
 /** @} */
Index: /trunk/include/VBox/pdmapi.h
===================================================================
--- /trunk/include/VBox/pdmapi.h	(revision 6795)
+++ /trunk/include/VBox/pdmapi.h	(revision 6796)
@@ -111,6 +111,6 @@
  */
 
-PDMR3DECL(int)  PDMR3LdrLoadVMMR0(void **ppvOpaque);
-PDMR3DECL(void) PDMR3LdrLoadVMMR0Part2(PVM pVM, void *pvOpaque);
+PDMR3DECL(int) PDMR3InitUVM(PUVM pUVM);
+PDMR3DECL(int) PDMR3LdrLoadVMMR0U(PUVM pUVM);
 
 /**
@@ -169,8 +169,8 @@
  * process so that components can resolve GC symbols during relocation.
  *
- * @param   pVM         VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   offDelta    Relocation delta relative to old location.
  */
-PDMR3DECL(void) PDMR3LdrRelocate(PVM pVM, RTGCINTPTR offDelta);
+PDMR3DECL(void) PDMR3LdrRelocateU(PUVM pUVM, RTGCINTPTR offDelta);
 
 /**
@@ -194,4 +194,5 @@
  */
 PDMR3DECL(int) PDMR3Term(PVM pVM);
+PDMR3DECL(void) PDMR3TermUVM(PUVM pUVM);
 
 
Index: /trunk/include/VBox/stam.h
===================================================================
--- /trunk/include/VBox/stam.h	(revision 6795)
+++ /trunk/include/VBox/stam.h	(revision 6796)
@@ -955,55 +955,10 @@
  */
 
-/**
- * Initializes the STAM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-STAMR3DECL(int) STAMR3Init(PVM pVM);
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM     The VM.
- */
-STAMR3DECL(void) STAMR3Relocate(PVM pVM);
-
-/**
- * Terminates the STAM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-STAMR3DECL(int) STAMR3Term(PVM pVM);
-
-/**
- * Registers a sample with the statistics mamanger.
- *
- * Statistics are maintained on a per VM basis and should therefore
- * be registered during the VM init stage. However, there is not problem
- * registering temporary samples or samples for hotpluggable devices. Samples
- * can be deregisterd using the STAMR3Deregister() function, but note that
- * this is only necessary for temporary samples or hotpluggable devices.
- *
- * It is not possible to register the same sample twice.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
- *                      Further nesting is possible.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- */
-STAMR3DECL(int)  STAMR3Register(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+STAMR3DECL(int) STAMR3InitUVM(PUVM pUVM);
+STAMR3DECL(void) STAMR3TermUVM(PUVM pUVM);
+STAMR3DECL(int) STAMR3RegisterU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
                                 const char *pszName, STAMUNIT enmUnit, const char *pszDesc);
+STAMR3DECL(int) STAMR3Register(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                               const char *pszName, STAMUNIT enmUnit, const char *pszDesc);
 
 /** @def STAM_REL_REG
@@ -1063,35 +1018,10 @@
     STAM_STATS({ STAM_REL_REG_USED(pVM, pvSample, enmType, pszName, enmUnit, pszDesc); })
 
-/**
- * Same as STAMR3Register except that the name is specified in a
- * RTStrPrintf like fashion.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   ...         Arguments to the format string.
- */
+STAMR3DECL(int)  STAMR3RegisterFU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                  const char *pszDesc, const char *pszName, ...);
 STAMR3DECL(int)  STAMR3RegisterF(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
                                  const char *pszDesc, const char *pszName, ...);
-
-/**
- * Same as STAMR3Register except that the name is specified in a
- * RTStrPrintfV like fashion.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   args        Arguments to the format string.
- */
+STAMR3DECL(int)  STAMR3RegisterVU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                  const char *pszDesc, const char *pszName, va_list args);
 STAMR3DECL(int)  STAMR3RegisterV(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
                                  const char *pszDesc, const char *pszName, va_list args);
@@ -1118,54 +1048,11 @@
 typedef FNSTAMR3CALLBACKPRINT *PFNSTAMR3CALLBACKPRINT;
 
-/**
- * Similar to STAMR3Register except for the two callbacks, the implied type (STAMTYPE_CALLBACK),
- * and name given in an RTStrPrintf like fashion.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
- * @param   pfnPrint    Print the sample.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   ...         Arguments to the format string.
- * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
- */
 STAMR3DECL(int)  STAMR3RegisterCallback(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
                                         PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
                                         const char *pszDesc, const char *pszName, ...);
-
-/**
- * Same as STAMR3RegisterCallback() except for the ellipsis which is a va_list here.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
- * @param   pfnPrint    Print the sample.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   args        Arguments to the format string.
- * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
- */
 STAMR3DECL(int)  STAMR3RegisterCallbackV(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
                                          PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
                                          const char *pszDesc, const char *pszName, va_list args);
-
-/**
- * Deregisters all samples previously registered by STAMR3Register(),
- * STAMR3RegisterF(), STAMR3RegisterV() or STAMR3RegisterCallback().
- *
- * This is intended used for devices which can be unplugged and for
- * temporary samples.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the register sample.
- */
+STAMR3DECL(int)  STAMR3DeregisterU(PUVM pUVM, void *pvSample);
 STAMR3DECL(int)  STAMR3Deregister(PVM pVM, void *pvSample);
 
@@ -1187,71 +1074,16 @@
     STAM_STATS({ STAM_REL_DEREG(pVM, pvSample); })
 
-/**
- * Resets statistics for the specified VM.
- * It's possible to select a subset of the samples.
- *
- * @returns VBox status. (Basically, it cannot fail.)
- * @param   pVM         The VM handle.
- * @param   pszPat      The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                      If NULL all samples are reset.
- */
-STAMR3DECL(int)  STAMR3Reset(PVM pVM, const char *pszPat);
-
-/**
- * Get a snapshot of the statistics.
- * It's possible to select a subset of the samples.
- *
- * @returns VBox status. (Basically, it cannot fail.)
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are reset.
- * @param   fWithDesc       Whether to include the descriptions.
- * @param   ppszSnapshot    Where to store the pointer to the snapshot data.
- *                          The format of the snapshot should be XML, but that will have to be discussed
- *                          when this function is implemented.
- *                          The returned pointer must be freed by calling STAMR3SnapshotFree().
- * @param   pcchSnapshot    Where to store the size of the snapshot data. (Excluding the trailing '\0')
- */
+STAMR3DECL(int) STAMR3ResetU(PUVM pUVM, const char *pszPat);
+STAMR3DECL(int) STAMR3Reset(PVM pVM, const char *pszPat);
+STAMR3DECL(int) STAMR3SnapshotU(PUVM pUVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc);
 STAMR3DECL(int) STAMR3Snapshot(PVM pVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc);
-
-/**
- * Releases a statistics snapshot returned by STAMR3Snapshot().
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
- *                          NULL is allowed.
- */
-STAMR3DECL(int)  STAMR3SnapshotFree(PVM pVM, char *pszSnapshot);
-
-/**
- * Dumps the selected statistics to the log.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are reset.
- */
-STAMR3DECL(int)  STAMR3Dump(PVM pVM, const char *pszPat);
-
-/**
- * Dumps the selected statistics to the release log.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are written to the log.
- */
-STAMR3DECL(int)  STAMR3DumpToReleaseLog(PVM pVM, const char *pszPat);
-
-/**
- * Prints the selected statistics to standard out.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are reset.
- */
-STAMR3DECL(int)  STAMR3Print(PVM pVM, const char *pszPat);
+STAMR3DECL(int) STAMR3SnapshotFreeU(PUVM pUVM, char *pszSnapshot);
+STAMR3DECL(int) STAMR3SnapshotFree(PVM pVM, char *pszSnapshot);
+STAMR3DECL(int) STAMR3DumpU(PUVM pUVM, const char *pszPat);
+STAMR3DECL(int) STAMR3Dump(PVM pVM, const char *pszPat);
+STAMR3DECL(int) STAMR3DumpToReleaseLogU(PUVM pUVM, const char *pszPat);
+STAMR3DECL(int) STAMR3DumpToReleaseLog(PVM pVM, const char *pszPat);
+STAMR3DECL(int) STAMR3PrintU(PUVM pUVM, const char *pszPat);
+STAMR3DECL(int) STAMR3Print(PVM pVM, const char *pszPat);
 
 /**
@@ -1273,22 +1105,6 @@
 typedef FNSTAMR3ENUM *PFNSTAMR3ENUM;
 
-/**
- * Enumerate the statistics by the means of a callback function.
- *
- * @returns Whatever the callback returns.
- *
- * @param   pVM         The VM handle.
- * @param   pszPat      The pattern to match samples.
- * @param   pfnEnum     The callback function.
- * @param   pvUser      The pvUser argument of the callback function.
- */
+STAMR3DECL(int) STAMR3EnumU(PUVM pUVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser);
 STAMR3DECL(int) STAMR3Enum(PVM pVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser);
-
-/**
- * Get the unit string.
- *
- * @returns Pointer to read only unit string.
- * @param   enmUnit     The unit.
- */
 STAMR3DECL(const char *) STAMR3GetUnit(STAMUNIT enmUnit);
 
Index: /trunk/include/VBox/types.h
===================================================================
--- /trunk/include/VBox/types.h	(revision 6795)
+++ /trunk/include/VBox/types.h	(revision 6796)
@@ -90,6 +90,9 @@
 typedef GCPTRTYPE(struct VM *)  PVMGC;
 
-/** Pointer to a the ring-0 (global) VM structure. */
+/** Pointer to a ring-0 (global) VM structure. */
 typedef R0PTRTYPE(struct GVM *) PGVM;
+
+/** Pointer to a ring-3 (user mode) VM structure. */
+typedef R3PTRTYPE(struct UVM *) PUVM;
 
 
Index: /trunk/include/VBox/uvm.h
===================================================================
--- /trunk/include/VBox/uvm.h	(revision 6796)
+++ /trunk/include/VBox/uvm.h	(revision 6796)
@@ -0,0 +1,98 @@
+/* $Id$ */
+/** @file
+ * GVM - The Global VM Data.
+ */
+
+/*
+ * Copyright (C) 2007 innotek GmbH
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
+ * VirtualBox OSE distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ */
+
+
+#ifndef ___VBox_uvm_h
+#define ___VBox_uvm_h
+
+
+#include <VBox/types.h>
+
+/**
+ * The ring-3 (user mode) VM structure.
+ *
+ * This structure is similar to VM and GVM except that it resides in swappable
+ * user memory. The main purpose is to assist bootstrapping, where it allows us
+ * to start EMT much earlier and gives PDMLdr somewhere to put it's VMMR0 data.
+ * It is also a nice place to put big things that are user mode only.
+ */
+typedef struct UVM
+{
+    /** Magic / eye-catcher (UVM_MAGIC). */
+    uint32_t        u32Magic;
+    uint32_t        uReserved;          /**< alignment */
+    /** The ring-3 mapping of the shared VM structure. */
+    PVM             pVM;
+    /** Pointer to the next VM.
+     * We keep a per process list of VM for the event that a process could
+     * contain more than one VM.
+     * @todo move this into vm.s!
+     */
+    struct UVM     *pNext;
+
+    /** The VM internal data. */
+    struct
+    {
+#ifdef ___VMInternal_h
+        struct VMINTUSERPERVM   s;
+#endif
+        uint8_t                 padding[768];
+    } vm;
+
+    /** The MM data. */
+    struct
+    {
+#ifdef ___MMInternal_h
+        struct MMUSERPERVM      s;
+#endif
+        uint8_t                 padding[32];
+    } mm;
+
+    /** The PDM data. */
+    struct
+    {
+#ifdef ___PDMInternal_h
+        struct PDMUSERPERVM     s;
+#endif
+        uint8_t                 padding[32];
+    } pdm;
+
+    /** The STAM data. */
+    struct
+    {
+#ifdef ___STAMInternal_h
+        struct STAMUSERPERVM    s;
+#endif
+        uint8_t                 padding[256];
+    } stam;
+
+} UVM;
+
+/** The UVM::u32Magic value (Brad Mehldau). */
+#define UVM_MAGIC       0x19700823
+
+#endif
+
Index: /trunk/include/VBox/vm.h
===================================================================
--- /trunk/include/VBox/vm.h	(revision 6795)
+++ /trunk/include/VBox/vm.h	(revision 6796)
@@ -239,9 +239,6 @@
     /** Session handle. For use when calling SUPR0 APIs. */
     PSUPDRVSESSION              pSession;
-    /** Pointer to the next VM.
-     * We keep a per process list of VM for the event that a process could
-     * contain more than one VM.
-     */
-    R3PTRTYPE(struct VM *)      pNext;
+    /** Pointer to the ring-3 VM structure. */
+    PUVM                        pUVM;
     /** Ring-3 Host Context VM Pointer. */
     R3PTRTYPE(struct VM *)      pVMR3;
@@ -498,13 +495,4 @@
     } dbgf;
 
-    /** STAM part. */
-    union
-    {
-#ifdef ___STAMInternal_h
-        struct STAM s;
-#endif
-        char        padding[256];       /* multiple of 32 */
-    } stam;
-
     /** SSM part. */
     union
Index: /trunk/include/VBox/vm.mac
===================================================================
--- /trunk/include/VBox/vm.mac	(revision 6795)
+++ /trunk/include/VBox/vm.mac	(revision 6796)
@@ -46,6 +46,6 @@
     .fForcedActions     resd 1
     .paVMPagesR3        RTR3PTR_RES 1
-    .pSession           RTR3PTR_RES 1
-    .pNext              RTHCPTR_RES 1
+    .pSession           RTR0PTR_RES 1
+    .pUVM               RTR3PTR_RES 1
     .pVMR3              RTR3PTR_RES 1
     .pVMR0              RTR0PTR_RES 1
Index: /trunk/include/VBox/vmapi.h
===================================================================
--- /trunk/include/VBox/vmapi.h	(revision 6795)
+++ /trunk/include/VBox/vmapi.h	(revision 6796)
@@ -55,5 +55,5 @@
 #else
 # define VM_GUEST_ADDR(pVM, pvInVM)     ( (RTGCPTR)(pvInVM) )
-#endif 
+#endif
 
 /** @def VM_R3_ADDR
@@ -71,5 +71,5 @@
 #else
 # define VM_R3_ADDR(pVM, pvInVM)       ( (RTR3PTR)(pvInVM) )
-#endif 
+#endif
 
 
@@ -88,5 +88,5 @@
 #else
 # define VM_R0_ADDR(pVM, pvInVM)       ( (RTR0PTR)(pvInVM) )
-#endif 
+#endif
 
 /** @def VM_HOST_ADDR
@@ -118,34 +118,5 @@
 typedef FNVMATERROR *PFNVMATERROR;
 
-/**
- * Sets the error message.
- *
- * @returns rc. Meaning you can do:
- *    @code
- *    return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message");
- *    @endcode
- * @param   pVM             VM handle. Must be non-NULL.
- * @param   rc              VBox status code.
- * @param   RT_SRC_POS_DECL Use RT_SRC_POS.
- * @param   pszFormat       Error message format string.
- * @param   ...             Error message arguments.
- * @thread  Any
- */
 VMDECL(int) VMSetError(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...);
-
-/**
- * Sets the error message.
- *
- * @returns rc. Meaning you can do:
- *    @code
- *    return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message");
- *    @endcode
- * @param   pVM             VM handle. Must be non-NULL.
- * @param   rc              VBox status code.
- * @param   RT_SRC_POS_DECL Use RT_SRC_POS.
- * @param   pszFormat       Error message format string.
- * @param   args            Error message arguments.
- * @thread  Any
- */
 VMDECL(int) VMSetErrorV(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args);
 
@@ -183,60 +154,6 @@
 typedef FNVMATRUNTIMEERROR *PFNVMATRUNTIMEERROR;
 
-/**
- * Sets the runtime error message.
- * As opposed VMSetError(), this method is intended to inform the VM user about
- * errors and error-like conditions that happen at an arbitrary point during VM
- * execution (like "host memory low" or "out of host disk space").
- *
- * The @a fFatal parameter defines whether the error is fatal or not. If it is
- * true, then it is expected that the caller has already paused the VM execution
- * before calling this method. The VM user is supposed to power off the VM
- * immediately after it has received the runtime error notification via the
- * FNVMATRUNTIMEERROR callback.
- *
- * If @a fFatal is false, then the paused state of the VM defines the kind of
- * the error. If the VM is paused before calling this method, it means that
- * the VM user may try to fix the error condition (i.e. free more host memory)
- * and then resume the VM execution. If the VM is not paused before calling
- * this method, it means that the given error is a warning about an error
- * condition that may happen soon but that doesn't directly affect the
- * VM execution by the time of the call.
- *
- * The @a pszErrorID parameter defines an unique error identificator.
- * It is used by the front-ends to show a proper message to the end user
- * containig possible actions (for example, Retry/Ignore). For this reason,
- * an error ID assigned once to some particular error condition should not
- * change in the future. The format of this parameter is "SomeErrorCondition". 
- *
- * @param   pVM             VM handle. Must be non-NULL.
- * @param   fFatal          Whether it is a fatal error or not.
- * @param   pszErrorID      Error ID string.
- * @param   pszFormat       Error message format string.
- * @param   ...             Error message arguments.
- *
- * @return  VBox status code (whether the error has been successfully set
- *          and delivered to callbacks or not).
- *
- * @thread  Any
- */
-VMDECL(int) VMSetRuntimeError(PVM pVM, bool fFatal, const char *pszErrorID,
-                              const char *pszFormat, ...);
-
-/**
- * va_list version of VMSetRuntimeError.
- *
- * @param   pVM             VM handle. Must be non-NULL.
- * @param   fFatal          Whether it is a fatal error or not.
- * @param   pszErrorID      Error ID string.
- * @param   pszFormat       Error message format string.
- * @param   args            Error message arguments.
- *
- * @return  VBox status code (whether the error has been successfully set
- *          and delivered to callbacks or not).
- *
- * @thread  Any
- */
-VMDECL(int) VMSetRuntimeErrorV(PVM pVM, bool fFatal, const char *pszErrorID,
-                               const char *pszFormat, va_list args);
+VMDECL(int) VMSetRuntimeError(PVM pVM, bool fFatal, const char *pszErrorID, const char *pszFormat, ...);
+VMDECL(int) VMSetRuntimeErrorV(PVM pVM, bool fFatal, const char *pszErrorID, const char *pszFormat, va_list args);
 
 
@@ -347,6 +264,6 @@
     /** Pointer to the next request in the chain. */
     struct VMREQ * volatile pNext;
-    /** Pointer to the VM this packet belongs to. */
-    PVM                     pVM;
+    /** Pointer to ring-3 VM structure which this request belongs to. */
+    PUVM                    pUVM;
     /** Request state. */
     volatile VMREQSTATE     enmState;
@@ -414,80 +331,9 @@
 
 
-/**
- * Creates a virtual machine by calling the supplied configuration constructor.
- *
- * On successful return the VM is powered off, i.e. VMR3PowerOn() should be
- * called to start the execution.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   pfnVMAtError        Pointer to callback function for setting VM errors.
- *                              This is called in the EM.
- * @param   pvUserVM            The user argument passed to pfnVMAtError.
- * @param   pfnCFGMConstructor  Pointer to callback function for constructing the VM configuration tree.
- *                              This is called in the EM.
- * @param   pvUserCFGM          The user argument passed to pfnCFGMConstructor.
- * @param   ppVM                Where to store the 'handle' of the created VM.
- * @thread      Any thread.
- * @vmstate     N/A
- * @vmstateto   Created
- */
 VMR3DECL(int)   VMR3Create(PFNVMATERROR pfnVMAtError, void *pvUserVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM, PVM *ppVM);
-
-/**
- * Power on a virtual machine.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   pVM         VM to power on.
- * @thread      Any thread.
- * @vmstate     Created
- * @vmstateto   Running
- */
 VMR3DECL(int)   VMR3PowerOn(PVM pVM);
-
-/**
- * Suspend a running VM.
- *
- * @returns VBox status.
- * @param   pVM     VM to suspend.
- * @thread      Any thread.
- * @vmstate     Running
- * @vmstateto   Suspended
- */
 VMR3DECL(int)   VMR3Suspend(PVM pVM);
-
-/**
- * Suspends a running VM and prevent state saving until the VM is resumed or stopped.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   pVM     VM to suspend.
- * @thread      Any thread.
- * @vmstate     Running
- * @vmstateto   Suspended
- */
-VMR3DECL(int) VMR3SuspendNoSave(PVM pVM);
-
-/**
- * Resume VM execution.
- *
- * @returns VBox status.
- * @param   pVM         The VM to resume.
- * @thread      Any thread.
- * @vmstate     Suspended
- * @vmstateto   Running
- */
+VMR3DECL(int)   VMR3SuspendNoSave(PVM pVM);
 VMR3DECL(int)   VMR3Resume(PVM pVM);
-
-/**
- * Reset the current VM.
- *
- * @returns VBox status code.
- * @param   pVM     VM to reset.
- * @thread      Any thread.
- * @vmstate     Suspended, Running
- * @vmstateto   Unchanged state.
- */
 VMR3DECL(int)   VMR3Reset(PVM pVM);
 
@@ -506,98 +352,11 @@
 typedef FNVMPROGRESS *PFNVMPROGRESS;
 
-/**
- * Save current VM state.
- *
- * To save and terminate the VM, the VM must be suspended before the call.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   pVM             VM which state should be saved.
- * @param   pszFilename     Name of the save state file.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvUser          User argument for the progress callback.
- * @thread      Any thread.
- * @vmstate     Suspended
- * @vmstateto   Unchanged state.
- */
 VMR3DECL(int)   VMR3Save(PVM pVM, const char *pszFilename, PFNVMPROGRESS pfnProgress, void *pvUser);
-
-/**
- * Load a new VM state.
- *
- * To restore a saved state on VM startup, call this function and then
- * resume the VM instead of powering it on.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   pVM             VM which state should be saved.
- * @param   pszFilename     Name of the save state file.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvUser          User argument for the progress callback.
- * @thread      Any thread.
- * @vmstate     Created, Suspended
- * @vmstateto   Suspended
- */
 VMR3DECL(int)   VMR3Load(PVM pVM, const char *pszFilename, PFNVMPROGRESS pfnProgress, void *pvUser);
-
-/**
- * Power off a VM.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   pVM     VM which should be destroyed.
- * @thread      Any thread.
- * @vmstate     Suspended, Running, Guru Mediation, Load Failure
- * @vmstateto   Off
- */
 VMR3DECL(int)   VMR3PowerOff(PVM pVM);
-
-/**
- * Destroy a VM.
- * The VM must be powered off (or never really powered on) to call this function.
- * The VM handle is destroyed and can no longer be used up successful return.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   pVM     VM which should be destroyed.
- * @thread      Any thread but the emulation thread.
- * @vmstate     Off, Created
- * @vmstateto   N/A
- */
 VMR3DECL(int)   VMR3Destroy(PVM pVM);
-
-/**
- * Calls the relocation functions for all VMM components so they can update
- * any GC pointers. When this function is called all the basic VM members
- * have been updated  and the actual memory relocation have been done
- * by the PGM/MM.
- *
- * This is used both on init and on runtime relocations.
- *
- * @param   pVM         VM handle.
- * @param   offDelta    Relocation delta relative to old location.
- * @thread  The emulation thread.
- */
-VMR3DECL(void)   VMR3Relocate(PVM pVM, RTGCINTPTR offDelta);
-
-/**
- * Enumerates the VMs in this process.
- *
- * @returns Pointer to the next VM.
- * @returns NULL when no more VMs.
- * @param   pVMPrev     The previous VM
- *                      Use NULL to start the enumeration.
- */
+VMR3DECL(void)  VMR3Relocate(PVM pVM, RTGCINTPTR offDelta);
 VMR3DECL(PVM)   VMR3EnumVMs(PVM pVMPrev);
-
-/**
- * Wait for VM to be resumed. Handle events like vmR3EmulationThread does.
- * In case the VM is stopped, clean up and long jump to the main EMT loop.
- *
- * @returns VINF_SUCCESS or doesn't return
- * @param   pVM             VM handle.
- */
-VMR3DECL(int) VMR3WaitForResume(PVM pVM);
-
+VMR3DECL(int)   VMR3WaitForResume(PVM pVM);
 
 /**
@@ -610,402 +369,39 @@
 typedef FNVMATDTOR *PFNVMATDTOR;
 
-/**
- * Registers an at VM destruction callback.
- *
- * @returns VBox status code.
- * @param   pfnAtDtor       Pointer to callback.
- * @param   pvUser          User argument.
- */
 VMR3DECL(int)   VMR3AtDtorRegister(PFNVMATDTOR pfnAtDtor, void *pvUser);
-
-/**
- * Deregisters an at VM destruction callback.
- *
- * @returns VBox status code.
- * @param   pfnAtDtor       Pointer to callback.
- */
 VMR3DECL(int)   VMR3AtDtorDeregister(PFNVMATDTOR pfnAtDtor);
-
-
-/**
- * Registers an at VM reset callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM.
- * @param   pDevInst        Device instance.
- * @param   pfnCallback     Callback function.
- * @param   pvUser          User argument.
- * @param   pszDesc         Description (optional).
- */
 VMR3DECL(int)   VMR3AtResetRegister(PVM pVM, PPDMDEVINS pDevInst, PFNVMATRESET pfnCallback, void *pvUser, const char *pszDesc);
-
-/**
- * Registers an at VM reset internal callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM.
- * @param   pfnCallback     Callback function.
- * @param   pvUser          User argument.
- * @param   pszDesc         Description (optional).
- */
 VMR3DECL(int)   VMR3AtResetRegisterInternal(PVM pVM, PFNVMATRESETINT pfnCallback, void *pvUser, const char *pszDesc);
-
-/**
- * Registers an at VM reset external callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM.
- * @param   pfnCallback     Callback function.
- * @param   pvUser          User argument.
- * @param   pszDesc         Description (optional).
- */
 VMR3DECL(int)   VMR3AtResetRegisterExternal(PVM pVM, PFNVMATRESETEXT pfnCallback, void *pvUser, const char *pszDesc);
-
-
-/**
- * Deregisters an at VM reset callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM.
- * @param   pDevInst        Device instance.
- * @param   pfnCallback     Callback function.
- */
 VMR3DECL(int)   VMR3AtResetDeregister(PVM pVM, PPDMDEVINS pDevInst, PFNVMATRESET pfnCallback);
-
-/**
- * Deregisters an at VM reset internal callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM.
- * @param   pfnCallback     Callback function.
- */
 VMR3DECL(int)   VMR3AtResetDeregisterInternal(PVM pVM, PFNVMATRESETINT pfnCallback);
-
-/**
- * Deregisters an at VM reset external callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM.
- * @param   pfnCallback     Callback function.
- */
 VMR3DECL(int)   VMR3AtResetDeregisterExternal(PVM pVM, PFNVMATRESETEXT pfnCallback);
-
-
-/**
- * Registers a VM state change callback.
- *
- * You are not allowed to call any function which changes the VM state from a
- * state callback, except VMR3Destroy().
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pfnAtState      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
 VMR3DECL(int)   VMR3AtStateRegister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser);
-
-/**
- * Deregisters a VM state change callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pfnAtState      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
 VMR3DECL(int)   VMR3AtStateDeregister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser);
-
-/**
- * Gets the current VM state.
- *
- * @returns The current VM state.
- * @param   pVM             The VM handle.
- * @thread  Any
- */
 VMR3DECL(VMSTATE) VMR3GetState(PVM pVM);
-
-/**
- * Gets the state name string for a VM state.
- *
- * @returns Pointer to the state name. (readonly)
- * @param   enmState        The state.
- */
 VMR3DECL(const char *) VMR3GetStateName(VMSTATE enmState);
-
-/**
- * Registers a VM error callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pfnAtError      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
 VMR3DECL(int)   VMR3AtErrorRegister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser);
-
-/**
- * Deregisters a VM error callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pfnAtError      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
+VMR3DECL(int)   VMR3AtErrorRegisterU(PUVM pVM, PFNVMATERROR pfnAtError, void *pvUser);
 VMR3DECL(int)   VMR3AtErrorDeregister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser);
-
-/**
- * This is a worker function for GC and Ring-0 calls to VMSetError and VMSetErrorV.
- * The message is found in VMINT.
- *
- * @param   pVM             The VM handle.
- * @thread  EMT.
- */
-VMR3DECL(void) VMR3SetErrorWorker(PVM pVM);
-
-/**
- * Registers a VM runtime error callback.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pfnAtRuntimeError   Pointer to callback.
- * @param   pvUser              User argument.
- * @thread  Any.
- */
+VMR3DECL(void)  VMR3SetErrorWorker(PVM pVM);
 VMR3DECL(int)   VMR3AtRuntimeErrorRegister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser);
-
-/**
- * Deregisters a VM runtime error callback.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pfnAtRuntimeError   Pointer to callback.
- * @param   pvUser              User argument.
- * @thread  Any.
- */
 VMR3DECL(int)   VMR3AtRuntimeErrorDeregister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser);
-
-/**
- * This is a worker function for GC and Ring-0 calls to VMSetRuntimeError and VMSetRuntimeErrorV.
- * The message is found in VMINT.
- *
- * @param   pVM             The VM handle.
- * @thread  EMT.
- */
-VMR3DECL(void) VMR3SetRuntimeErrorWorker(PVM pVM);
-
-
-/**
- * Allocate and queue a call request.
- *
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completation. In the other case
- * use RT_INDEFINITE_WAIT.
- * The returned request packet must be freed using VMR3ReqFree().
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pVM             The VM handle.
- * @param   ppReq           Where to store the pointer to the request.
- *                          This will be NULL or a valid request pointer not matter what happends.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- *                          Not possible to pass 64-bit arguments!
- * @param   ...             Function arguments.
- */
-VMR3DECL(int) VMR3ReqCall(PVM pVM, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...);
-
-/**
- * Allocate and queue a call request to a void function.
- *
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completation. In the other case
- * use RT_INDEFINITE_WAIT.
- * The returned request packet must be freed using VMR3ReqFree().
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pVM             The VM handle.
- * @param   ppReq           Where to store the pointer to the request.
- *                          This will be NULL or a valid request pointer not matter what happends.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- *                          Not possible to pass 64-bit arguments!
- * @param   ...             Function arguments.
- */
-VMR3DECL(int) VMR3ReqCallVoid(PVM pVM, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...);
-
-/**
- * Allocate and queue a call request to a void function.
- *
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completation. In the other case
- * use RT_INDEFINITE_WAIT.
- * The returned request packet must be freed using VMR3ReqFree().
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pVM             The VM handle.
- * @param   ppReq           Where to store the pointer to the request.
- *                          This will be NULL or a valid request pointer not matter what happends, unless fFlags
- *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- * @param   fFlags          A combination of the VMREQFLAGS values.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- *                          Not possible to pass 64-bit arguments!
- * @param   ...             Function arguments.
- */
-VMR3DECL(int) VMR3ReqCallEx(PVM pVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, ...);
-
-/**
- * Allocate and queue a call request.
- *
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completation. In the other case
- * use RT_INDEFINITE_WAIT.
- * The returned request packet must be freed using VMR3ReqFree().
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pVM             The VM handle.
- * @param   ppReq           Where to store the pointer to the request.
- *                          This will be NULL or a valid request pointer not matter what happends, unless fFlags
- *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- * @param   fFlags          A combination of the VMREQFLAGS values.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- *                          Not possible to pass 64-bit arguments!
- * @param   pvArgs          Pointer to function arguments.
- */
-VMR3DECL(int) VMR3ReqCallV(PVM pVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, va_list Args);
-
-/**
- * Allocates a request packet.
- *
- * The caller allocates a request packet, fills in the request data
- * union and queues the request.
- *
- * @returns VBox status code.
- *
- * @param   pVM             VM handle.
- * @param   ppReq           Where to store the pointer to the allocated packet.
- * @param   enmType         Package type.
- */
-VMR3DECL(int) VMR3ReqAlloc(PVM pVM, PVMREQ *ppReq, VMREQTYPE enmType);
-
-/**
- * Free a request packet.
- *
- * @returns VBox status code.
- *
- * @param   pReq            Package to free.
- * @remark  The request packet must be in allocated or completed state!
- */
-VMR3DECL(int) VMR3ReqFree(PVMREQ pReq);
-
-/**
- * Queue a request.
- *
- * The quest must be allocated using VMR3ReqAlloc() and contain
- * all the required data.
- * If it's disired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completation. In the other case
- * use RT_INDEFINITE_WAIT.
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pReq            The request to queue.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- */
-VMR3DECL(int) VMR3ReqQueue(PVMREQ pReq, unsigned cMillies);
-
-/**
- * Wait for a request to be completed.
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pReq            The request to wait for.
- * @param   cMillies        Number of milliseconds to wait.
- *                          Use RT_INDEFINITE_WAIT to only wait till it's completed.
- */
-VMR3DECL(int) VMR3ReqWait(PVMREQ pReq, unsigned cMillies);
-
-
-
-/**
- * Process pending request(s).
- *
- * This function is called from a forced action handler in
- * the EMT.
- *
- * @returns VBox status code.
- *
- * @param   pVM         VM handle.
- */
-VMR3DECL(int) VMR3ReqProcess(PVM pVM);
-
-
-/**
- * Notify the emulation thread (EMT) about pending Forced Action (FF).
- *
- * This function is called by thread other than EMT to make
- * sure EMT wakes up and promptly service a FF request.
- *
- * @param   pVM             VM handle.
- * @param   fNotifiedREM    Set if REM have already been notified. If clear the
- *                          generic REMR3NotifyFF() method is called.
- */
-VMR3DECL(void) VMR3NotifyFF(PVM pVM, bool fNotifiedREM);
-
-/**
- * Halted VM Wait.
- * Any external event will unblock the thread.
- *
- * @returns VINF_SUCCESS unless a fatal error occured. In the latter
- *          case an appropriate status code is returned.
- * @param   pVM                 VM handle.
- * @param   fIgnoreInterrupts   If set the VM_FF_INTERRUPT flags is ignored.
- * @thread  The emulation thread.
- */
-VMR3DECL(int) VMR3WaitHalted(PVM pVM, bool fIgnoreInterrupts);
-
-/**
- * Suspended VM Wait.
- * Only a handful of forced actions will cause the function to
- * return to the caller.
- *
- * @returns VINF_SUCCESS unless a fatal error occured. In the latter
- *          case an appropriate status code is returned.
- * @param   pVM         VM handle.
- * @thread  The emulation thread.
- */
-VMR3DECL(int) VMR3Wait(PVM pVM);
+VMR3DECL(void)  VMR3SetRuntimeErrorWorker(PVM pVM);
+VMR3DECL(int)   VMR3ReqCall(PVM pVM, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...);
+VMR3DECL(int)   VMR3ReqCallVoidU(PUVM pUVM, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...);
+VMR3DECL(int)   VMR3ReqCallVoid(PVM pVM, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...);
+VMR3DECL(int)   VMR3ReqCallEx(PVM pVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, ...);
+VMR3DECL(int)   VMR3ReqCallU(PUVM pUVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, ...);
+VMR3DECL(int)   VMR3ReqCallVU(PUVM pUVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, va_list Args);
+VMR3DECL(int)   VMR3ReqAlloc(PVM pVM, PVMREQ *ppReq, VMREQTYPE enmType);
+VMR3DECL(int)   VMR3ReqAllocU(PUVM pUVM, PVMREQ *ppReq, VMREQTYPE enmType);
+VMR3DECL(int)   VMR3ReqFree(PVMREQ pReq);
+VMR3DECL(int)   VMR3ReqQueue(PVMREQ pReq, unsigned cMillies);
+VMR3DECL(int)   VMR3ReqWait(PVMREQ pReq, unsigned cMillies);
+VMR3DECL(int)   VMR3ReqProcessU(PUVM pUVM);
+VMR3DECL(void)  VMR3NotifyFF(PVM pVM, bool fNotifiedREM);
+VMR3DECL(void)  VMR3NotifyFFU(PUVM pUVM, bool fNotifiedREM);
+VMR3DECL(int)   VMR3WaitHalted(PVM pVM, bool fIgnoreInterrupts);
+VMR3DECL(int)   VMR3WaitU(PUVM pUVM);
 
 /** @} */
Index: /trunk/src/VBox/VMM/DBGF.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGF.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/DBGF.cpp	(revision 6796)
@@ -604,5 +604,5 @@
             {
                 LogFlow(("dbgfr3VMMWait: Processes requests...\n"));
-                rc = VMR3ReqProcess(pVM);
+                rc = VMR3ReqProcessU(pVM->pUVM);
                 LogFlow(("dbgfr3VMMWait: VMR3ReqProcess -> %Vrc rcRet=%Vrc\n", rc, rcRet));
                 if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
Index: /trunk/src/VBox/VMM/EM.cpp
===================================================================
--- /trunk/src/VBox/VMM/EM.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/EM.cpp	(revision 6796)
@@ -2984,5 +2984,5 @@
         if (VM_FF_ISSET(pVM, VM_FF_REQUEST))
         {
-            rc2 = VMR3ReqProcess(pVM);
+            rc2 = VMR3ReqProcessU(pVM->pUVM);
             if (rc2 == VINF_EM_OFF || rc2 == VINF_EM_TERMINATE)
             {
Index: /trunk/src/VBox/VMM/MM.cpp
===================================================================
--- /trunk/src/VBox/VMM/MM.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/MM.cpp	(revision 6796)
@@ -88,4 +88,5 @@
 #include "MMInternal.h"
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 #include <VBox/err.h>
 #include <VBox/param.h>
@@ -107,8 +108,30 @@
 *   Internal Functions                                                         *
 *******************************************************************************/
-static int mmR3Term(PVM pVM, bool fKeepTheHeap);
 static DECLCALLBACK(int) mmR3Save(PVM pVM, PSSMHANDLE pSSM);
 static DECLCALLBACK(int) mmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version);
 
+
+/**
+ * Initializes the MM members of the UVM.
+ *
+ * This is currently only the ring-3 heap.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    Pointer to the user mode VM structure.
+ */
+MMR3DECL(int) MMR3InitUVM(PUVM pUVM)
+{
+    /*
+     * Assert sizes and order.
+     */
+    AssertCompile(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
+    AssertRelease(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
+    Assert(!pUVM->mm.s.pHeap);
+
+    /*
+     * Init the heap.
+     */
+    return mmR3HeapCreateU(pUVM, &pUVM->mm.s.pHeap);
+}
 
 
@@ -147,14 +170,4 @@
     pVM->mm.s.offVM = RT_OFFSETOF(VM, mm);
     pVM->mm.s.offLookupHyper = NIL_OFFSET;
-
-    /*
-     * Init the heap (may already be initialized already if someone used it).
-     */
-    if (!pVM->mm.s.pHeap)
-    {
-        int rc = mmR3HeapCreate(pVM, &pVM->mm.s.pHeap);
-        if (!VBOX_SUCCESS(rc))
-            return rc;
-    }
 
     /*
@@ -182,5 +195,5 @@
         }
     }
-    mmR3Term(pVM, true /* keep the heap */);
+    MMR3Term(pVM);
     return rc;
 }
@@ -346,21 +359,4 @@
  */
 MMR3DECL(int) MMR3Term(PVM pVM)
-{
-    return mmR3Term(pVM, false /* free the heap */);
-}
-
-
-/**
- * Worker for MMR3Term and MMR3Init.
- *
- * The tricky bit here is that we must not destroy the heap if we're
- * called from MMR3Init, otherwise we'll get into trouble when
- * CFGMR3Term is called later in the bailout process.
- *
- * @returns VBox status code.
- * @param   pVM             The VM to operate on.
- * @param   fKeepTheHeap    Whether or not to keep the heap.
- */
-static int mmR3Term(PVM pVM, bool fKeepTheHeap)
 {
     /*
@@ -395,13 +391,4 @@
 
     /*
-     * Destroy the heap if requested.
-     */
-    if (!fKeepTheHeap)
-    {
-        mmR3HeapDestroy(pVM->mm.s.pHeap);
-        pVM->mm.s.pHeap = NULL;
-    }
-
-    /*
      * Zero stuff to detect after termination use of the MM interface
      */
@@ -413,4 +400,23 @@
 
     return 0;
+}
+
+
+/**
+ * Terminates the UVM part of MM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+MMR3DECL(void) MMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * Destroy the heap.
+     */
+    mmR3HeapDestroy(pUVM->mm.s.pHeap);
+    pUVM->mm.s.pHeap = NULL;
 }
 
Index: /trunk/src/VBox/VMM/MMHeap.cpp
===================================================================
--- /trunk/src/VBox/VMM/MMHeap.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/MMHeap.cpp	(revision 6796)
@@ -22,7 +22,9 @@
 #define LOG_GROUP LOG_GROUP_MM_HEAP
 #include <VBox/mm.h>
+#include <VBox/stam.h>
 #include <VBox/pgm.h>
 #include "MMInternal.h"
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 #include <VBox/err.h>
 #include <VBox/param.h>
@@ -48,5 +50,5 @@
  * @param   ppHeap  Where to store the heap pointer.
  */
-int mmR3HeapCreate(PVM pVM, PMMHEAP *ppHeap)
+int mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap)
 {
     PMMHEAP pHeap = (PMMHEAP)RTMemAllocZ(sizeof(MMHEAP) + sizeof(MMHEAPSTAT));
@@ -59,17 +61,16 @@
              * Initialize the global stat record.
              */
-            pHeap->pVM   = pVM;
-
+            pHeap->pUVM = pUVM;
             pHeap->Stat.pHeap = pHeap;
 #ifdef MMR3HEAP_WITH_STATISTICS
             PMMHEAPSTAT pStat = &pHeap->Stat;
-            STAMR3Register(pVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
-            STAMR3Register(pVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
-            STAMR3Register(pVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFrees",           STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
-            STAMR3Register(pVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
-            STAMR3Register(pVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
-                                                                      STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
-            STAMR3Register(pVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
-            STAMR3Register(pVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
+            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFrees",           STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
+            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
+                                                                        STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
 #endif
             *ppHeap = pHeap;
@@ -121,5 +122,5 @@
  * The memory will be allocated from the default heap but a header
  * is added in which we keep track of which VM it belongs to and chain
- * all the allocations together so they can be freed in a one go.
+ * all the allocations together so they can be freed in one go.
  *
  * This interface is typically used for memory block which will not be
@@ -127,4 +128,28 @@
  *
  * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ */
+MMR3DECL(void *) MMR3HeapAllocU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
+{
+    Assert(pUVM->mm.s.pHeap);
+    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
+}
+
+
+/**
+ * Allocate memory associating it with the VM for collective cleanup.
+ *
+ * The memory will be allocated from the default heap but a header
+ * is added in which we keep track of which VM it belongs to and chain
+ * all the allocations together so they can be freed in one go.
+ *
+ * This interface is typically used for memory block which will not be
+ * freed during the life of the VM.
+ *
+ * @returns Pointer to allocated memory.
  * @param   pVM         VM handle.
  * @param   enmTag      Statistics tag. Statistics are collected on a per tag
@@ -135,20 +160,13 @@
 MMR3DECL(void *) MMR3HeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize)
 {
-    if (!pVM->mm.s.pHeap)
-    {
-        int rc = mmR3HeapCreate(pVM, &pVM->mm.s.pHeap);
-        if (VBOX_FAILURE(rc))
-            return NULL;
-    }
-    return mmR3HeapAlloc(pVM->mm.s.pHeap, enmTag, cbSize, false);
-}
-
-
-/**
- * Same as MMR3HeapAlloc().
- *
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
+    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
+}
+
+
+/**
+ * Same as MMR3HeapAllocU().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   enmTag      Statistics tag. Statistics are collected on a per tag
  *                      basis in addition to a global one. Thus we can easily
@@ -157,13 +175,8 @@
  * @param   ppv         Where to store the pointer to the allocated memory on success.
  */
-MMR3DECL(int) MMR3HeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    if (!pVM->mm.s.pHeap)
-    {
-        int rc = mmR3HeapCreate(pVM, &pVM->mm.s.pHeap);
-        if (VBOX_FAILURE(rc))
-            return rc;
-    }
-    void *pv = mmR3HeapAlloc(pVM->mm.s.pHeap, enmTag, cbSize, false);
+MMR3DECL(int) MMR3HeapAllocExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    Assert(pUVM->mm.s.pHeap);
+    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
     if (pv)
     {
@@ -176,6 +189,5 @@
 
 /**
- * Same as MMR3HeapAlloc() only the memory is zeroed.
- *
+ * Same as MMR3HeapAlloc().
  *
  * @returns Pointer to allocated memory.
@@ -185,38 +197,9 @@
  *                      identify how memory is used by the VM.
  * @param   cbSize      Size of the block.
- */
-MMR3DECL(void *) MMR3HeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize)
-{
-    if (!pVM->mm.s.pHeap)
-    {
-        int rc = mmR3HeapCreate(pVM, &pVM->mm.s.pHeap);
-        if (VBOX_FAILURE(rc))
-            return NULL;
-    }
-    return mmR3HeapAlloc(pVM->mm.s.pHeap, enmTag, cbSize, true);
-}
-
-
-/**
- * Same as MMR3HeapAllocZ().
- *
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
  * @param   ppv         Where to store the pointer to the allocated memory on success.
  */
-MMR3DECL(int) MMR3HeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    if (!pVM->mm.s.pHeap)
-    {
-        int rc = mmR3HeapCreate(pVM, &pVM->mm.s.pHeap);
-        if (VBOX_FAILURE(rc))
-            return rc;
-    }
-    void *pv = mmR3HeapAlloc(pVM->mm.s.pHeap, enmTag, cbSize, true);
+MMR3DECL(int) MMR3HeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
     if (pv)
     {
@@ -229,4 +212,83 @@
 
 /**
+ * Same as MMR3HeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ */
+MMR3DECL(void *) MMR3HeapAllocZU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
+}
+
+
+/**
+ * Same as MMR3HeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ */
+MMR3DECL(void *) MMR3HeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
+}
+
+
+/**
+ * Same as MMR3HeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+MMR3DECL(int) MMR3HeapAllocZExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    Assert(pUVM->mm.s.pHeap);
+    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3HeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+MMR3DECL(int) MMR3HeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
  * Allocate memory from the heap.
  *
@@ -239,5 +301,5 @@
  * @param   fZero       Whether or not to zero the memory block.
  */
-void * mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
+void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
 {
 #ifdef MMR3HEAP_WITH_STATISTICS
@@ -273,27 +335,27 @@
 
         /* register the statistics */
-        PVM pVM = pHeap->pVM;
+        PUVM pUVM = pHeap->pUVM;
         char szName[80];
         const char *pszTag = mmR3GetTagName(enmTag);
         RTStrPrintf(szName, sizeof(szName), "/MM/R3Heap/%s/cAllocations", pszTag);
-        STAMR3Register(pVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
+        STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
 
         RTStrPrintf(szName, sizeof(szName), "/MM/R3Heap/%s/cReallocations", pszTag);
-        STAMR3Register(pVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
+        STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
 
         RTStrPrintf(szName, sizeof(szName), "/MM/R3Heap/%s/cFrees", pszTag);
-        STAMR3Register(pVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
+        STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
 
         RTStrPrintf(szName, sizeof(szName), "/MM/R3Heap/%s/cFailures", pszTag);
-        STAMR3Register(pVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_COUNT, "Number of failures.");
+        STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_COUNT, "Number of failures.");
 
         RTStrPrintf(szName, sizeof(szName), "/MM/R3Heap/%s/cbCurAllocated", pszTag);
-        STAMR3Register(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_BYTES, "Number of bytes currently allocated.");
+        STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_BYTES, "Number of bytes currently allocated.");
 
         RTStrPrintf(szName, sizeof(szName), "/MM/R3Heap/%s/cbAllocated", pszTag);
-        STAMR3Register(pVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_BYTES, "Total number of bytes allocated.");
+        STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_BYTES, "Total number of bytes allocated.");
 
         RTStrPrintf(szName, sizeof(szName), "/MM/R3Heap/%s/cbFreed", pszTag);
-        STAMR3Register(pVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_BYTES, "Total number of bytes freed.");
+        STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName,  STAMUNIT_BYTES, "Total number of bytes freed.");
     }
 #endif
@@ -452,5 +514,5 @@
     RTCritSectEnter(&pHeap->Lock);
     pHdrNew->pStat->cbAllocated += cbNewSize - pHdrNew->cbSize;
-    pHeap->Stat.cbAllocated   += cbNewSize - pHdrNew->cbSize;
+    pHeap->Stat.cbAllocated += cbNewSize - pHdrNew->cbSize;
     RTCritSectLeave(&pHeap->Lock);
 #endif
@@ -459,4 +521,29 @@
 
     return pHdrNew + 1;
+}
+
+
+/**
+ * Duplicates the specified string.
+ *
+ * @returns Pointer to the duplicate.
+ * @returns NULL on failure or when input NULL.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   psz         The string to duplicate. NULL is allowed.
+ */
+MMR3DECL(char *) MMR3HeapStrDupU(PUVM pUVM, MMTAG enmTag, const char *psz)
+{
+    if (!psz)
+        return NULL;
+    AssertPtr(psz);
+
+    size_t cch = strlen(psz) + 1;
+    char *pszDup = (char *)MMR3HeapAllocU(pUVM, enmTag, cch);
+    if (pszDup)
+        memcpy(pszDup, psz, cch);
+    return pszDup;
 }
 
@@ -475,12 +562,5 @@
 MMR3DECL(char *) MMR3HeapStrDup(PVM pVM, MMTAG enmTag, const char *psz)
 {
-    if (!psz)
-        return NULL;
-    Assert(VALID_PTR(psz));
-    size_t cch = strlen(psz) + 1;
-    char *pszDup = (char *)MMR3HeapAlloc(pVM, enmTag, cch);
-    if (pszDup)
-        memcpy(pszDup, psz, cch);
-    return pszDup;
+    return MMR3HeapStrDupU(pVM->pUVM, enmTag, psz);
 }
 
Index: /trunk/src/VBox/VMM/MMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/MMInternal.h	(revision 6795)
+++ /trunk/src/VBox/VMM/MMInternal.h	(revision 6796)
@@ -117,5 +117,5 @@
     PAVLULNODECORE      pStatTree;
     /** The VM handle. */
-    PVM                 pVM;
+    PUVM                pUVM;
     /** Heap global statistics. */
     MMHEAPSTAT          Stat;
@@ -675,12 +675,21 @@
     /** The head of the ROM ranges. */
     R3PTRTYPE(PMMROMRANGE)      pRomHead;
-
-    /** Pointer to the MM R3 Heap. */
-    R3PTRTYPE(PMMHEAP)          pHeap;
-
 } MM;
 /** Pointer to MM Data (part of VM). */
 typedef MM *PMM;
 
+
+/**
+ * MM data kept in the UVM.
+ */
+typedef struct MMUSERPERVM
+{
+    /** Pointer to the MM R3 Heap. */
+    R3PTRTYPE(PMMHEAP)          pHeap;
+} MMUSERPERVM;
+/** Pointer to the MM data kept in the UVM. */
+typedef MMUSERPERVM *PMMUSERPERVM;
+
+
 __BEGIN_DECLS
 
@@ -691,5 +700,5 @@
 void mmR3PagePoolTerm(PVM pVM);
 
-int  mmR3HeapCreate(PVM pVM, PMMHEAP *ppHeap);
+int  mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap);
 void mmR3HeapDestroy(PMMHEAP pHeap);
 
Index: /trunk/src/VBox/VMM/PDM.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDM.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/PDM.cpp	(revision 6796)
@@ -109,4 +109,5 @@
 #include <VBox/ssm.h>
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 #include <VBox/vmm.h>
 #include <VBox/param.h>
@@ -141,4 +142,22 @@
 
 /**
+ * Initializes the PDM part of the UVM.
+ *
+ * This doesn't really do much right now but has to be here for the sake
+ * of completeness.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+PDMR3DECL(int) PDMR3InitUVM(PUVM pUVM)
+{
+    AssertCompile(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
+    AssertRelease(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
+    pUVM->pdm.s.pModules = NULL;
+    return VINF_SUCCESS;
+}
+
+
+/**
  * Initializes the PDM.
  *
@@ -174,5 +193,5 @@
         if (VBOX_SUCCESS(rc))
 #endif
-            rc = pdmR3LdrInit(pVM);
+            rc = pdmR3LdrInitU(pVM->pUVM);
         if (VBOX_SUCCESS(rc))
         {
@@ -426,5 +445,5 @@
      * Free modules.
      */
-    pdmR3LdrTerm(pVM);
+    pdmR3LdrTermU(pVM->pUVM);
 
 #ifdef VBOX_WITH_PDM_LOCK
@@ -438,4 +457,25 @@
     return VINF_SUCCESS;
 }
+
+
+/**
+ * Terminates the PDM part of the UVM.
+ *
+ * This will unload any modules left behind.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+PDMR3DECL(void) PDMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * In the normal cause of events we will now call pdmR3LdrTermU for
+     * the second time. In the case of init failure however, this might
+     * the first time, which is why we do it.
+     */
+    pdmR3LdrTermU(pUVM);
+}
+
+
+
 
 
Index: /trunk/src/VBox/VMM/PDMDevice.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDevice.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/PDMDevice.cpp	(revision 6796)
@@ -994,5 +994,5 @@
      * Load it.
      */
-    int rc = pdmR3LoadR3(pVM, pszFilename, pszName);
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
     if (VBOX_SUCCESS(rc))
     {
Index: /trunk/src/VBox/VMM/PDMDriver.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDriver.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/PDMDriver.cpp	(revision 6796)
@@ -95,5 +95,5 @@
 
 #ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
-static DECLCALLBACK(int) pdmR3DrvHlp_PDMAsyncCompletionTemplateCreate(PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, 
+static DECLCALLBACK(int) pdmR3DrvHlp_PDMAsyncCompletionTemplateCreate(PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
                                                                       PFNPDMASYNCCOMPLETEDRV pfnCompleted, const char *pszDesc);
 #endif
@@ -303,5 +303,5 @@
      * Load it.
      */
-    int rc = pdmR3LoadR3(pVM, pszFilename, pszName);
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
     if (VBOX_SUCCESS(rc))
     {
@@ -1075,5 +1075,5 @@
 
 #ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
-static DECLCALLBACK(int) pdmR3DrvHlp_PDMAsyncCompletionTemplateCreate(PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, 
+static DECLCALLBACK(int) pdmR3DrvHlp_PDMAsyncCompletionTemplateCreate(PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
                                                                       PFNPDMASYNCCOMPLETEDRV pfnCompleted, const char *pszDesc)
 {
@@ -1085,5 +1085,5 @@
     int rc = PDMR3AsyncCompletionTemplateCreateDriver(pDrvIns->Internal.s.pVM, pDrvIns, ppTemplate, pfnCompleted, pszDesc);
 
-    LogFlow(("pdmR3DrvHlp_PDMAsyncCompletionTemplateCreate: caller='%s'/%d: returns %Vrc *ppThread=%p\n", pDrvIns->pDrvReg->szDriverName, 
+    LogFlow(("pdmR3DrvHlp_PDMAsyncCompletionTemplateCreate: caller='%s'/%d: returns %Vrc *ppThread=%p\n", pDrvIns->pDrvReg->szDriverName,
              pDrvIns->iInstance, rc, *ppTemplate));
     return rc;
Index: /trunk/src/VBox/VMM/PDMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/PDMInternal.h	(revision 6795)
+++ /trunk/src/VBox/VMM/PDMInternal.h	(revision 6796)
@@ -776,7 +776,4 @@
     RTUINT                          uPadding0; /**< Alignment padding.*/
 
-    /** Pointer to list of loaded modules. This is HC only! */
-    R3PTRTYPE(PPDMMOD)              pModules;
-
     /** List of registered devices. (FIFO) */
     R3PTRTYPE(PPDMDEV)              pDevs;
@@ -867,4 +864,17 @@
 
 
+/**
+ * PDM data kept in the UVM.
+ */
+typedef struct PDMUSERPERVM
+{
+    /** Pointer to list of loaded modules. */
+    PPDMMOD                         pModules;
+    /** @todo move more stuff over here. */
+} PDMUSERPERVM;
+/** Pointer to the PDM data kept in the UVM. */
+typedef PDMUSERPERVM *PPDMUSERPERVM;
+
+
 
 /*******************************************************************************
@@ -902,8 +912,8 @@
 PPDMDRV     pdmR3DrvLookup(PVM pVM, const char *pszName);
 
-int         pdmR3LdrInit(PVM pVM);
-void        pdmR3LdrTerm(PVM pVM);
+int         pdmR3LdrInitU(PUVM pUVM);
+void        pdmR3LdrTermU(PUVM pUVM);
 char *      pdmR3FileR3(const char *pszFile, bool fShared = false);
-int         pdmR3LoadR3(PVM pVM, const char *pszFilename, const char *pszName);
+int         pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName);
 
 void        pdmR3QueueRelocate(PVM pVM, RTGCINTPTR offDelta);
Index: /trunk/src/VBox/VMM/PDMLdr.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMLdr.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/PDMLdr.cpp	(revision 6796)
@@ -27,4 +27,5 @@
 #include <VBox/vmm.h>
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 #include <VBox/sup.h>
 #include <VBox/param.h>
@@ -59,6 +60,6 @@
 *   Internal Functions                                                         *
 *******************************************************************************/
-static DECLCALLBACK(int) pdmr3GetImportGC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser);
-static int      pdmR3LoadR0(PVM pVM, const char *pszFilename, const char *pszName);
+static DECLCALLBACK(int) pdmR3GetImportGC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser);
+static int      pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName);
 static char *   pdmR3FileGC(const char *pszFile);
 static char *   pdmR3FileR0(const char *pszFile);
@@ -69,88 +70,12 @@
 
 /**
- * Loads the VMMR0.r0 module before the VM is created.
- *
- * The opqaue VMMR0 module pointer is passed on to PDMR3Init later in
- * the init process or PDMR3LdrUnloadVMMR0 in case of some init failure before PDMR3Init.
+ * Loads the VMMR0.r0 module early in the init process.
  *
  * @returns VBox status code.
- * @param   ppvOpaque       Where to return the opaque VMMR0.r0 module handle one success.
- *
- * @remarks Yes, this is a kind of hacky and should go away. See @todo in VMR3Create.
- */
-PDMR3DECL(int) PDMR3LdrLoadVMMR0(void **ppvOpaque)
-{
-    *ppvOpaque = NULL;
-
-    /*
-     * Resolve the filename and allocate the module list node.
-     */
-    char *pszFilename = pdmR3FileR0(VMMR0_MAIN_MODULE_NAME);
-    PPDMMOD pModule = (PPDMMOD)RTMemAllocZ(sizeof(*pModule) + strlen(pszFilename));
-    if (!pModule)
-    {
-        RTMemTmpFree(pszFilename);
-        return VERR_NO_MEMORY;
-    }
-    strcpy(pModule->szName, VMMR0_MAIN_MODULE_NAME);
-    pModule->eType = PDMMOD_TYPE_R0;
-    strcpy(pModule->szFilename, pszFilename);
-    RTMemTmpFree(pszFilename);
-
-    /*
-     * Ask the support library to load it.
-     */
-    void *pvImageBase;
-    int rc = SUPLoadModule(pModule->szFilename, pModule->szName, &pvImageBase);
-    if (RT_SUCCESS(rc))
-    {
-        pModule->hLdrMod = NIL_RTLDRMOD;
-        pModule->ImageBase = (uintptr_t)pvImageBase;
-        *ppvOpaque = pModule;
-
-        Log(("PDMR3LdrLoadVMMR0: Loaded %s at %VGvx (%s)\n", pModule->szName, (RTGCPTR)pModule->ImageBase, pModule->szFilename));
-        return VINF_SUCCESS;
-    }
-
-    LogRel(("PDMR3LdrLoadVMMR0: rc=%Vrc szName=%s szFilename=%s\n", rc, pModule->szName, pModule->szFilename));
-    RTMemFree(pModule);
-    return rc;
-}
-
-
-/**
- * Register the VMMR0.r0 module with the created VM or unload it if
- * we failed to create the VM (pVM == NULL).
- *
- * @param   pVM         The VM pointer. NULL if we failed to create the VM and
- *                      the module should be unloaded and freed.
- * @param   pvOpaque    The value returned by PDMR3LDrLoadVMMR0().
- *
- * @remarks Yes, this is a kind of hacky and should go away. See @todo in VMR3Create.
- */
-PDMR3DECL(void) PDMR3LdrLoadVMMR0Part2(PVM pVM, void *pvOpaque)
-{
-    PPDMMOD pModule = (PPDMMOD)pvOpaque;
-    AssertPtrReturnVoid(pModule);
-
-    if (pVM)
-    {
-        /*
-         * Register the R0 module loaded by PDMR3LdrLoadVMMR0
-         */
-        Assert(!pVM->pdm.s.pModules);
-        pModule->pNext = pVM->pdm.s.pModules;
-        pVM->pdm.s.pModules = pModule;
-    }
-    else
-    {
-        /*
-         * Failed, unload the module.
-         */
-        int rc2 = SUPFreeModule((void *)(uintptr_t)pModule->ImageBase);
-        AssertRC(rc2);
-        pModule->ImageBase = 0;
-        RTMemFree(pvOpaque);
-    }
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+PDMR3DECL(int) PDMR3LdrLoadVMMR0U(PUVM pUVM)
+{
+    return pdmR3LoadR0U(pUVM, NULL, VMMR0_MAIN_MODULE_NAME);
 }
 
@@ -163,8 +88,8 @@
  *
  * @returns VBox stutus code.
- * @param   pVM         VM handle.
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   pvVMMR0Mod  The opqaue returned by PDMR3LdrLoadVMMR0.
  */
-int pdmR3LdrInit(PVM pVM)
+int pdmR3LdrInitU(PUVM pUVM)
 {
 #ifdef PDMLDR_FAKE_MODE
@@ -176,5 +101,5 @@
      * Load the mandatory GC module, the VMMR0.r0 is loaded before VM creation.
      */
-    return PDMR3LoadGC(pVM, NULL, VMMGC_MAIN_MODULE_NAME);
+    return PDMR3LoadGC(pUVM->pVM, NULL, VMMGC_MAIN_MODULE_NAME);
 #endif
 }
@@ -187,11 +112,14 @@
  *
  * @param   pVM         The VM handle.
- */
-void pdmR3LdrTerm(PVM pVM)
+ *
+ * @remarks This is normally called twice during termination.
+ */
+void pdmR3LdrTermU(PUVM pUVM)
 {
     /*
      * Free the modules.
      */
-    PPDMMOD pModule = pVM->pdm.s.pModules;
+    PPDMMOD pModule = pUVM->pdm.s.pModules;
+    pUVM->pdm.s.pModules = NULL;
     while (pModule)
     {
@@ -240,8 +168,8 @@
  * process so that components can resolve GC symbols during relocation.
  *
- * @param   pVM         VM handle.
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   offDelta    Relocation delta relative to old location.
  */
-PDMR3DECL(void) PDMR3LdrRelocate(PVM pVM, RTGCINTPTR offDelta)
+PDMR3DECL(void) PDMR3LdrRelocateU(PUVM pUVM, RTGCINTPTR offDelta)
 {
     LogFlow(("PDMR3LdrRelocate: offDelta=%VGv\n", offDelta));
@@ -250,5 +178,5 @@
      * GC Modules.
      */
-    if (pVM->pdm.s.pModules)
+    if (pUVM->pdm.s.pModules)
     {
         /*
@@ -260,25 +188,25 @@
         /* pass 1 */
         PPDMMOD pCur;
-        for (pCur = pVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
         {
             if (pCur->eType == PDMMOD_TYPE_GC)
             {
                 pCur->OldImageBase = pCur->ImageBase;
-                pCur->ImageBase = MMHyperHC2GC(pVM, pCur->pvBits);
+                pCur->ImageBase = MMHyperHC2GC(pUVM->pVM, pCur->pvBits);
             }
         }
 
         /* pass 2 */
-        for (pCur = pVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
         {
             if (pCur->eType == PDMMOD_TYPE_GC)
             {
                 PDMGETIMPORTARGS Args;
-                Args.pVM = pVM;
+                Args.pVM = pUVM->pVM;
                 Args.pModule = pCur;
                 int rc = RTLdrRelocate(pCur->hLdrMod, pCur->pvBits, pCur->ImageBase, pCur->OldImageBase,
-                                       pdmr3GetImportGC, &Args);
+                                       pdmR3GetImportGC, &Args);
                 AssertFatalMsgRC(rc, ("RTLdrRelocate failed, rc=%d\n", rc));
-                DBGFR3ModuleRelocate(pVM, pCur->OldImageBase, pCur->ImageBase, RTLdrSize(pCur->hLdrMod),
+                DBGFR3ModuleRelocate(pUVM->pVM, pCur->OldImageBase, pCur->ImageBase, RTLdrSize(pCur->hLdrMod),
                                      pCur->szFilename, pCur->szName);
             }
@@ -301,14 +229,14 @@
  *
  * @returns VBox status code.
- * @param   pVM             The VM to load it into.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pszFilename     Filename of the module binary.
  * @param   pszName         Module name. Case sensitive and the length is limited!
  */
-int pdmR3LoadR3(PVM pVM, const char *pszFilename, const char *pszName)
+int pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName)
 {
     /*
      * Validate input.
      */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+    AssertMsg(pUVM->pVM->pdm.s.offVM, ("bad init order!\n"));
     Assert(pszFilename);
     size_t cchFilename = strlen(pszFilename);
@@ -325,5 +253,5 @@
      * Try lookup the name and see if the module exists.
      */
-    for (pCur = pVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
     {
         if (!strcmp(pCur->szName, pszName))
@@ -353,6 +281,6 @@
     if (VBOX_SUCCESS(rc))
     {
-        pModule->pNext = pVM->pdm.s.pModules;
-        pVM->pdm.s.pModules = pModule;
+        pModule->pNext = pUVM->pdm.s.pModules;
+        pUVM->pdm.s.pModules = pModule;
         return rc;
     }
@@ -360,5 +288,5 @@
     /* Something went wrong, most likely module not found. Don't consider other unlikely errors */
     RTMemFree(pModule);
-    return VMSetError(pVM, rc, RT_SRC_POS, N_("Unable to load R3 module %s"), pszFilename);
+    return VMSetError(pUVM->pVM, rc, RT_SRC_POS, N_("Unable to load R3 module %s"), pszFilename);
 }
 
@@ -375,5 +303,5 @@
  * @param   pvUser          User argument.
  */
-static DECLCALLBACK(int) pdmr3GetImportGC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser)
+static DECLCALLBACK(int) pdmR3GetImportGC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser)
 {
     PVM         pVM = ((PPDMGETIMPORTARGS)pvUser)->pVM;
@@ -427,5 +355,5 @@
      * Search for module.
      */
-    PPDMMOD  pCur = pVM->pdm.s.pModules;
+    PPDMMOD  pCur = pVM->pUVM->pdm.s.pModules;
     while (pCur)
     {
@@ -478,5 +406,5 @@
      */
     AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-    PPDMMOD  pCur = pVM->pdm.s.pModules;
+    PPDMMOD  pCur = pVM->pUVM->pdm.s.pModules;
     while (pCur)
     {
@@ -541,5 +469,5 @@
                 Args.pVM = pVM;
                 Args.pModule = pModule;
-                rc = RTLdrGetBits(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pdmr3GetImportGC, &Args);
+                rc = RTLdrGetBits(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pdmR3GetImportGC, &Args);
                 if (VBOX_SUCCESS(rc))
                 {
@@ -547,8 +475,9 @@
                      * Insert the module.
                      */
-                    if (pVM->pdm.s.pModules)
+                    PUVM pUVM = pVM->pUVM;
+                    if (pUVM->pdm.s.pModules)
                     {
                         /* we don't expect this list to be very long, so rather save the tail pointer. */
-                        PPDMMOD pCur = pVM->pdm.s.pModules;
+                        PPDMMOD pCur = pUVM->pdm.s.pModules;
                         while (pCur->pNext)
                             pCur = pCur->pNext;
@@ -556,5 +485,5 @@
                     }
                     else
-                        pVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
+                        pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
                     Log(("PDM: GC Module at %VGvx %s (%s)\n", (RTGCPTR)pModule->ImageBase, pszName, pszFilename));
                     RTMemTmpFree(pszFile);
@@ -587,15 +516,14 @@
  *
  * @returns VBox status code.
- * @param   pVM             The VM to load it into.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pszFilename     Filename of the module binary.
  * @param   pszName         Module name. Case sensitive and the length is limited!
  */
-static int pdmR3LoadR0(PVM pVM, const char *pszFilename, const char *pszName)
+static int pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName)
 {
     /*
      * Validate input.
      */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-    PPDMMOD  pCur = pVM->pdm.s.pModules;
+    PPDMMOD  pCur = pUVM->pdm.s.pModules;
     while (pCur)
     {
@@ -608,5 +536,4 @@
         pCur = pCur->pNext;
     }
-    AssertReturn(strcmp(pszName, VMMR0_MAIN_MODULE_NAME), VERR_INTERNAL_ERROR);
 
     /*
@@ -645,8 +572,8 @@
          * Insert the module.
          */
-        if (pVM->pdm.s.pModules)
+        if (pUVM->pdm.s.pModules)
         {
             /* we don't expect this list to be very long, so rather save the tail pointer. */
-            PPDMMOD pCur = pVM->pdm.s.pModules;
+            PPDMMOD pCur = pUVM->pdm.s.pModules;
             while (pCur->pNext)
                 pCur = pCur->pNext;
@@ -654,5 +581,5 @@
         }
         else
-            pVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
+            pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
         Log(("PDM: GC Module at %VGvx %s (%s)\n", (RTGCPTR)pModule->ImageBase, pszName, pszFilename));
         RTMemTmpFree(pszFile);
@@ -662,9 +589,9 @@
     RTMemFree(pModule);
     RTMemTmpFree(pszFile);
-    LogRel(("pdmR3LoadR0: pszName=\"%s\" rc=%Vrc\n", pszName, rc));
+    LogRel(("pdmR3LoadR0U: pszName=\"%s\" rc=%Vrc\n", pszName, rc));
 
     /* Don't consider VERR_PDM_MODULE_NAME_CLASH and VERR_NO_MEMORY above as these are very unlikely. */
-    if (VBOX_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS, N_("Cannot load R0 module %s"), pszFilename);
+    if (VBOX_FAILURE(rc) && pUVM->pVM) /** @todo VMR3SetErrorU. */
+        return VMSetError(pUVM->pVM, rc, RT_SRC_POS, N_("Cannot load R0 module %s"), pszFilename);
     return rc;
 }
@@ -692,5 +619,5 @@
      * Find the module.
      */
-    for (PPDMMOD pModule = pVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    for (PPDMMOD pModule = pVM->pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
     {
         if (    pModule->eType == PDMMOD_TYPE_R3
@@ -747,5 +674,5 @@
      * Find the module.
      */
-    for (PPDMMOD pModule = pVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    for (PPDMMOD pModule = pVM->pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
     {
         if (    pModule->eType == PDMMOD_TYPE_R0
@@ -794,5 +721,5 @@
         AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
         PPDMMOD pModule;
-        for (pModule = pVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+        for (pModule = pVM->pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
             if (    pModule->eType == PDMMOD_TYPE_R0
                 &&  !strcmp(pModule->szName, pszModule))
@@ -800,5 +727,5 @@
         if (!pModule)
         {
-            int rc = pdmR3LoadR0(pVM, NULL, pszModule);
+            int rc = pdmR3LoadR0U(pVM->pUVM, NULL, pszModule);
             AssertMsgRCReturn(rc, ("pszModule=%s rc=%Vrc\n", pszModule, rc), VERR_MODULE_NOT_FOUND);
         }
@@ -836,5 +763,5 @@
      * Find the module.
      */
-    for (PPDMMOD pModule = pVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    for (PPDMMOD pModule = pVM->pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
     {
         if (    pModule->eType == PDMMOD_TYPE_GC
@@ -891,5 +818,5 @@
         AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
         PPDMMOD pModule;
-        for (pModule = pVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+        for (pModule = pVM->pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
             if (    pModule->eType == PDMMOD_TYPE_GC
                 &&  !strcmp(pModule->szName, pszModule))
@@ -1078,5 +1005,5 @@
     int     rc = VERR_MODULE_NOT_FOUND;
     PPDMMOD pCur;
-    for (pCur = pVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    for (pCur = pVM->pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
     {
         /* Skip anything which isn't in GC. */
@@ -1195,5 +1122,5 @@
 {
     PPDMMOD pCur;
-    for (pCur = pVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    for (pCur = pVM->pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
     {
         int rc = pfnCallback(pVM,
Index: /trunk/src/VBox/VMM/STAM.cpp
===================================================================
--- /trunk/src/VBox/VMM/STAM.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/STAM.cpp	(revision 6796)
@@ -24,4 +24,5 @@
 #include "STAMInternal.h"
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 #include <VBox/err.h>
 #include <VBox/dbg.h>
@@ -72,5 +73,5 @@
     /** Pointer to the current buffer position. */
     char           *psz;
-    /** The VM handle (just in case). */
+    /** The VM handle. */
     PVM             pVM;
     /** The number of bytes allocated. */
@@ -104,6 +105,6 @@
 *   Internal Functions                                                         *
 *******************************************************************************/
-static int stamR3Register(PVM pVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
-                          STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc);
+static int stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                           STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc);
 static int stamR3ResetOne(PSTAMDESC pDesc, void *pvArg);
 static DECLCALLBACK(void) stamR3EnumLogPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
@@ -116,8 +117,8 @@
 static bool stamR3MultiMatch(const char * const *papszExpressions, unsigned cExpressions, unsigned *piExpression, const char *pszName);
 static char **stamR3SplitPattern(const char *pszPat, unsigned *pcExpressions, char **ppszCopy);
-static int stamR3Enum(PVM pVM, const char *pszPat, bool fUpdateRing0, int (pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg);
-static void stamR3Ring0StatsRegister(PVM pVM);
-static void stamR3Ring0StatsUpdate(PVM pVM, const char *pszPat);
-static void stamR3Ring0StatsUpdateMulti(PVM pVM, const char * const *papszExpressions, unsigned cExpressions);
+static int stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0, int (pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg);
+static void stamR3Ring0StatsRegisterU(PUVM pUVM);
+static void stamR3Ring0StatsUpdateU(PUVM pUVM, const char *pszPat);
+static void stamR3Ring0StatsUpdateMultiU(PUVM pUVM, const char * const *papszExpressions, unsigned cExpressions);
 
 #ifdef VBOX_WITH_DEBUGGER
@@ -188,5 +189,5 @@
  * @param   pVM         The VM to operate on.
  */
-STAMR3DECL(int) STAMR3Init(PVM pVM)
+STAMR3DECL(int) STAMR3InitUVM(PUVM pUVM)
 {
     LogFlow(("STAMR3Init\n"));
@@ -195,20 +196,17 @@
      * Assert alignment and sizes.
      */
-    AssertRelease(!(RT_OFFSETOF(VM, stam.s) & 31));
-    AssertRelease(sizeof(pVM->stam.s) <= sizeof(pVM->stam.padding));
+    AssertCompile(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
+    AssertRelease(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
 
     /*
      * Setup any fixed pointers and offsets.
      */
-    pVM->stam.s.offVM = RT_OFFSETOF(VM, stam);
-    int rc = RTSemRWCreate(&pVM->stam.s.RWSem);
-    AssertRC(rc);
-    if (VBOX_FAILURE(rc))
-        return rc;
+    int rc = RTSemRWCreate(&pUVM->stam.s.RWSem);
+    AssertRCReturn(rc, rc);
 
     /*
      * Register the ring-0 statistics (GVMM/GMM).
      */
-    stamR3Ring0StatsRegister(pVM);
+    stamR3Ring0StatsRegisterU(pUVM);
 
 #ifdef VBOX_WITH_DEBUGGER
@@ -230,32 +228,14 @@
 
 /**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM     The VM.
- */
-STAMR3DECL(void) STAMR3Relocate(PVM pVM)
-{
-    LogFlow(("STAMR3Relocate\n"));
-    NOREF(pVM);
-}
-
-
-/**
  * Terminates the STAM.
  *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-STAMR3DECL(int) STAMR3Term(PVM pVM)
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+STAMR3DECL(void) STAMR3TermUVM(PUVM pUVM)
 {
     /*
      * Free used memory and the RWLock.
      */
-    PSTAMDESC   pCur = pVM->stam.s.pHead;
+    PSTAMDESC pCur = pUVM->stam.s.pHead;
     while (pCur)
     {
@@ -264,14 +244,10 @@
         RTMemFree(pvFree);
     }
-    pVM->stam.s.pHead = NULL;
-
-    /* careful here as we might be called twice in on some failure paths (?) */
-    if (pVM->stam.s.RWSem != NIL_RTSEMRW)
-        RTSemRWDestroy(pVM->stam.s.RWSem);
-    pVM->stam.s.RWSem = NIL_RTSEMRW;
-    return VINF_SUCCESS;
-}
-
-
+    pUVM->stam.s.pHead = NULL;
+
+    Assert(pUVM->stam.s.RWSem != NIL_RTSEMRW);
+    RTSemRWDestroy(pUVM->stam.s.RWSem);
+    pUVM->stam.s.RWSem = NIL_RTSEMRW;
+}
 
 
@@ -279,9 +255,39 @@
  * Registers a sample with the statistics mamanger.
  *
- * Statistics are maintained on a per VM basis and should therefore
- * be registered during the VM init stage. However, there is not problem
- * registering temporary samples or samples for hotpluggable devices. Samples
- * can be deregisterd using the STAMR3Deregister() function, but note that
- * this is only necessary for temporary samples or hotpluggable devices.
+ * Statistics are maintained on a per VM basis and is normally registered
+ * during the VM init stage, but there is nothing preventing you from
+ * register them at runtime.
+ *
+ * Use STAMR3Deregister() to deregister statistics at runtime, however do
+ * not bother calling at termination time.
+ *
+ * It is not possible to register the same sample twice.
+ *
+ * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
+ *                      Further nesting is possible.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ */
+STAMR3DECL(int)  STAMR3RegisterU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+    return stamR3RegisterU(pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc);
+}
+
+
+/**
+ * Registers a sample with the statistics mamanger.
+ *
+ * Statistics are maintained on a per VM basis and is normally registered
+ * during the VM init stage, but there is nothing preventing you from
+ * register them at runtime.
+ *
+ * Use STAMR3Deregister() to deregister statistics at runtime, however do
+ * not bother calling at termination time.
  *
  * It is not possible to register the same sample twice.
@@ -300,5 +306,30 @@
 {
     AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
-    return stamR3Register(pVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc);
+    return stamR3RegisterU(pVM->pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc);
+}
+
+
+/**
+ * Same as STAMR3RegisterU except that the name is specified in a
+ * RTStrPrintf like fashion.
+ *
+ * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ */
+STAMR3DECL(int)  STAMR3RegisterFU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                  const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterVU(pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
 }
 
@@ -323,5 +354,5 @@
     va_list args;
     va_start(args, pszName);
-    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    int rc = STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
     va_end(args);
     return rc;
@@ -343,6 +374,6 @@
  * @param   args        Arguments to the format string.
  */
-STAMR3DECL(int)  STAMR3RegisterV(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
-                                 const char *pszDesc, const char *pszName, va_list args)
+STAMR3DECL(int)  STAMR3RegisterVU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                  const char *pszDesc, const char *pszName, va_list args)
 {
     AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
@@ -353,7 +384,28 @@
         return VERR_NO_MEMORY;
 
-    int rc = STAMR3Register(pVM, pvSample, enmType, enmVisibility, pszFormattedName, enmUnit, pszDesc);
+    int rc = STAMR3RegisterU(pUVM, pvSample, enmType, enmVisibility, pszFormattedName, enmUnit, pszDesc);
     RTStrFree(pszFormattedName);
     return rc;
+}
+
+
+/**
+ * Same as STAMR3Register except that the name is specified in a
+ * RTStrPrintfV like fashion.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ */
+STAMR3DECL(int)  STAMR3RegisterV(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                 const char *pszDesc, const char *pszName, va_list args)
+{
+    return STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
 }
 
@@ -411,5 +463,5 @@
         return VERR_NO_MEMORY;
 
-    int rc = stamR3Register(pVM, pvSample, pfnReset, pfnPrint, STAMTYPE_CALLBACK, enmVisibility, pszFormattedName, enmUnit, pszDesc);
+    int rc = stamR3RegisterU(pVM->pUVM, pvSample, pfnReset, pfnPrint, STAMTYPE_CALLBACK, enmVisibility, pszFormattedName, enmUnit, pszDesc);
     RTStrFree(pszFormattedName);
     return rc;
@@ -421,5 +473,5 @@
  *
  * @returns VBox status.
- * @param   pVM         The VM handle.
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   pvSample    Pointer to the sample.
  * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
@@ -433,8 +485,8 @@
  * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
  */
-static int stamR3Register(PVM pVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
-                          STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
-{
-    STAM_LOCK_WR(pVM);
+static int stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                           STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
+{
+    STAM_LOCK_WR(pUVM);
 
     /*
@@ -442,5 +494,5 @@
      */
     PSTAMDESC   pPrev = NULL;
-    PSTAMDESC   pCur = pVM->stam.s.pHead;
+    PSTAMDESC   pCur = pUVM->stam.s.pHead;
     while (pCur)
     {
@@ -452,5 +504,5 @@
         if (!iDiff)
         {
-            STAM_UNLOCK_WR(pVM);
+            STAM_UNLOCK_WR(pUVM);
             AssertMsgFailed(("Duplicate sample name: %s\n", pszName));
             return VERR_ALREADY_EXISTS;
@@ -491,7 +543,7 @@
             pPrev->pNext    = pNew;
         else
-            pVM->stam.s.pHead = pNew;
-
-        stamR3ResetOne(pNew, pVM);
+            pUVM->stam.s.pHead = pNew;
+
+        stamR3ResetOne(pNew, pUVM->pVM);
         rc = VINF_SUCCESS;
     }
@@ -499,5 +551,5 @@
         rc = VERR_NO_MEMORY;
 
-    STAM_UNLOCK_WR(pVM);
+    STAM_UNLOCK_WR(pUVM);
     return rc;
 }
@@ -511,10 +563,10 @@
  *
  * @returns VBox status.
- * @param   pVM         The VM handle.
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   pvSample    Pointer to the sample registered with STAMR3Register().
  */
-STAMR3DECL(int)  STAMR3Deregister(PVM pVM, void *pvSample)
-{
-    STAM_LOCK_WR(pVM);
+STAMR3DECL(int)  STAMR3DeregisterU(PUVM pUVM, void *pvSample)
+{
+    STAM_LOCK_WR(pUVM);
 
     /*
@@ -523,5 +575,5 @@
     int         rc = VERR_INVALID_HANDLE;
     PSTAMDESC   pPrev = NULL;
-    PSTAMDESC   pCur = pVM->stam.s.pHead;
+    PSTAMDESC   pCur = pUVM->stam.s.pHead;
     while (pCur)
     {
@@ -533,5 +585,5 @@
                 pPrev->pNext = pCur;
             else
-                pVM->stam.s.pHead = pCur;
+                pUVM->stam.s.pHead = pCur;
 
             RTMemFree(pvFree);
@@ -545,6 +597,22 @@
     }
 
-    STAM_UNLOCK_WR(pVM);
+    STAM_UNLOCK_WR(pUVM);
     return rc;
+}
+
+
+/**
+ * Deregisters a sample previously registered by STAR3Register().
+ *
+ * This is intended used for devices which can be unplugged and for
+ * temporary samples.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample registered with STAMR3Register().
+ */
+STAMR3DECL(int)  STAMR3Deregister(PVM pVM, void *pvSample)
+{
+    return STAMR3DeregisterU(pVM->pUVM, pvSample);
 }
 
@@ -560,5 +628,5 @@
  * @remarks Don't confuse this with the other 'XYZR3Reset' methods, it's not called at VM reset.
  */
-STAMR3DECL(int)  STAMR3Reset(PVM pVM, const char *pszPat)
+STAMR3DECL(int)  STAMR3ResetU(PUVM pUVM, const char *pszPat)
 {
     int rc = VINF_SUCCESS;
@@ -601,7 +669,8 @@
     }
 
-    STAM_LOCK_WR(pVM);
+    STAM_LOCK_WR(pUVM);
     if (fGVMMMatched)
     {
+        PVM pVM = pUVM->pVM;
         GVMMReq.Hdr.cbReq = sizeof(GVMMReq);
         GVMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
@@ -612,4 +681,5 @@
 //    if (fGMMMatched)
 //    {
+//        PVM pVM = pUVM->pVM;
 //        GMMReq.Hdr.cbReq = sizeof(Req);
 //        GMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
@@ -619,14 +689,30 @@
 
     /* and the reset */
-    stamR3Enum(pVM, pszPat, false /* fUpdateRing0 */, stamR3ResetOne, pVM);
-
-    STAM_UNLOCK_WR(pVM);
+    stamR3EnumU(pUVM, pszPat, false /* fUpdateRing0 */, stamR3ResetOne, pUVM->pVM);
+
+    STAM_UNLOCK_WR(pUVM);
     return rc;
 }
 
+/**
+ * Resets statistics for the specified VM.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status. (Basically, it cannot fail.)
+ * @param   pVM         The VM handle.
+ * @param   pszPat      The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                      If NULL all samples are reset.
+ * @remarks Don't confuse this with the other 'XYZR3Reset' methods, it's not called at VM reset.
+ */
+STAMR3DECL(int)  STAMR3Reset(PVM pVM, const char *pszPat)
+{
+    return STAMR3ResetU(pVM->pUVM, pszPat);
+}
+
+
 
 /**
  * Resets one statistics sample.
- * Callback for stamR3Enum().
+ * Callback for stamR3EnumU().
  *
  * @returns VINF_SUCCESS
@@ -706,4 +792,57 @@
  *
  * @returns VBox status. (Basically, it cannot fail.)
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ * @param   fWithDesc       Whether to include the descriptions.
+ * @param   ppszSnapshot    Where to store the pointer to the snapshot data.
+ *                          The format of the snapshot should be XML, but that will have to be discussed
+ *                          when this function is implemented.
+ *                          The returned pointer must be freed by calling STAMR3SnapshotFree().
+ * @param   pcchSnapshot    Where to store the size of the snapshot data. (Excluding the trailing '\0')
+ */
+STAMR3DECL(int) STAMR3SnapshotU(PUVM pUVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc)
+{
+    STAMR3SNAPSHOTONE State = { NULL, NULL, NULL, pUVM->pVM, 0, VINF_SUCCESS, fWithDesc };
+
+    /*
+     * Write the XML header.
+     */
+    /** @todo Make this proper & valid XML. */
+    stamR3SnapshotPrintf(&State, "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n");
+
+    /*
+     * Write the content.
+     */
+    stamR3SnapshotPrintf(&State, "<Statistics>\n");
+    STAM_LOCK_RD(pUVM);
+    int rc = stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3SnapshotOne, &State);
+    STAM_UNLOCK_RD(pUVM);
+    stamR3SnapshotPrintf(&State, "</Statistics>\n");
+
+    if (VBOX_SUCCESS(rc))
+        rc = State.rc;
+    else
+    {
+        RTMemFree(State.pszStart);
+        State.pszStart = State.pszEnd = State.psz = NULL;
+        State.cbAllocated = 0;
+    }
+
+    /*
+     * Done.
+     */
+    *ppszSnapshot = State.pszStart;
+    if (pcchSnapshot)
+        *pcchSnapshot = State.psz - State.pszStart;
+    return rc;
+}
+
+
+/**
+ * Get a snapshot of the statistics.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status. (Basically, it cannot fail.)
  * @param   pVM             The VM handle.
  * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
@@ -718,42 +857,10 @@
 STAMR3DECL(int) STAMR3Snapshot(PVM pVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc)
 {
-    STAMR3SNAPSHOTONE State = { NULL, NULL, NULL, pVM, 0, VINF_SUCCESS, fWithDesc };
-
-    /*
-     * Write the XML header.
-     */
-    /** @todo Make this proper & valid XML. */
-    stamR3SnapshotPrintf(&State, "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n");
-
-    /*
-     * Write the content.
-     */
-    stamR3SnapshotPrintf(&State, "<Statistics>\n");
-    STAM_LOCK_RD(pVM);
-    int rc = stamR3Enum(pVM, pszPat, true /* fUpdateRing0 */, stamR3SnapshotOne, &State);
-    STAM_UNLOCK_RD(pVM);
-    stamR3SnapshotPrintf(&State, "</Statistics>\n");
-
-    if (VBOX_SUCCESS(rc))
-        rc = State.rc;
-    else
-    {
-        RTMemFree(State.pszStart);
-        State.pszStart = State.pszEnd = State.psz = NULL;
-        State.cbAllocated = 0;
-    }
-
-    /*
-     * Done.
-     */
-    *ppszSnapshot = State.pszStart;
-    if (pcchSnapshot)
-        *pcchSnapshot = State.psz - State.pszStart;
-    return rc;
-}
-
-
-/**
- * stamR3Enum callback employed by STAMR3Snapshot.
+    return STAMR3SnapshotU(pVM->pUVM, pszPat, ppszSnapshot, pcchSnapshot, fWithDesc);
+}
+
+
+/**
+ * stamR3EnumU callback employed by STAMR3Snapshot.
  *
  * @returns VBox status code, but it's interpreted as 0 == success / !0 == failure by enmR3Enum.
@@ -986,4 +1093,20 @@
  *
  * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
+ *                          NULL is allowed.
+ */
+STAMR3DECL(int)  STAMR3SnapshotFreeU(PUVM pUVM, char *pszSnapshot)
+{
+    if (!pszSnapshot)
+        RTMemFree(pszSnapshot);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Releases a statistics snapshot returned by STAMR3Snapshot().
+ *
+ * @returns VBox status.
  * @param   pVM             The VM handle.
  * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
@@ -992,6 +1115,26 @@
 STAMR3DECL(int)  STAMR3SnapshotFree(PVM pVM, char *pszSnapshot)
 {
-    if (!pszSnapshot)
-        RTMemFree(pszSnapshot);
+    return STAMR3SnapshotFreeU(pVM->pUVM, pszSnapshot);
+}
+
+
+/**
+ * Dumps the selected statistics to the log.
+ *
+ * @returns VBox status.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+STAMR3DECL(int)  STAMR3DumpU(PUVM pUVM, const char *pszPat)
+{
+    STAMR3PRINTONEARGS Args;
+    Args.pVM = pUVM->pVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumLogPrintf;
+
+    STAM_LOCK_RD(pUVM);
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    STAM_UNLOCK_RD(pUVM);
     return VINF_SUCCESS;
 }
@@ -1008,13 +1151,5 @@
 STAMR3DECL(int)  STAMR3Dump(PVM pVM, const char *pszPat)
 {
-    STAMR3PRINTONEARGS Args;
-    Args.pVM = pVM;
-    Args.pvArg = NULL;
-    Args.pfnPrintf = stamR3EnumLogPrintf;
-
-    STAM_LOCK_RD(pVM);
-    stamR3Enum(pVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-    STAM_UNLOCK_RD(pVM);
-    return VINF_SUCCESS;
+    return STAMR3DumpU(pVM->pUVM, pszPat);
 }
 
@@ -1041,4 +1176,26 @@
  *
  * @returns VBox status.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+STAMR3DECL(int)  STAMR3DumpToReleaseLogU(PUVM pUVM, const char *pszPat)
+{
+    STAMR3PRINTONEARGS Args;
+    Args.pVM = pUVM->pVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumRelLogPrintf;
+
+    STAM_LOCK_RD(pUVM);
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    STAM_UNLOCK_RD(pUVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps the selected statistics to the release log.
+ *
+ * @returns VBox status.
  * @param   pVM             The VM handle.
  * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
@@ -1047,13 +1204,5 @@
 STAMR3DECL(int)  STAMR3DumpToReleaseLog(PVM pVM, const char *pszPat)
 {
-    STAMR3PRINTONEARGS Args;
-    Args.pVM = pVM;
-    Args.pvArg = NULL;
-    Args.pfnPrintf = stamR3EnumRelLogPrintf;
-
-    STAM_LOCK_RD(pVM);
-    stamR3Enum(pVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-    STAM_UNLOCK_RD(pVM);
-    return VINF_SUCCESS;
+    return STAMR3DumpToReleaseLogU(pVM->pUVM, pszPat);
 }
 
@@ -1084,15 +1233,29 @@
  *                          If NULL all samples are reset.
  */
-STAMR3DECL(int)  STAMR3Print(PVM pVM, const char *pszPat)
+STAMR3DECL(int)  STAMR3PrintU(PUVM pUVM, const char *pszPat)
 {
     STAMR3PRINTONEARGS Args;
-    Args.pVM = pVM;
+    Args.pVM = pUVM->pVM;
     Args.pvArg = NULL;
     Args.pfnPrintf = stamR3EnumPrintf;
 
-    STAM_LOCK_RD(pVM);
-    stamR3Enum(pVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-    STAM_UNLOCK_RD(pVM);
+    STAM_LOCK_RD(pUVM);
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    STAM_UNLOCK_RD(pUVM);
     return VINF_SUCCESS;
+}
+
+
+/**
+ * Prints the selected statistics to standard out.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ */
+STAMR3DECL(int)  STAMR3Print(PVM pVM, const char *pszPat)
+{
+    return STAMR3PrintU(pVM->pUVM, pszPat);
 }
 
@@ -1117,5 +1280,5 @@
 /**
  * Prints one sample.
- * Callback for stamR3Enum().
+ * Callback for stamR3EnumU().
  *
  * @returns VINF_SUCCESS
@@ -1235,4 +1398,28 @@
  * @returns Whatever the callback returns.
  *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPat      The pattern to match samples.
+ * @param   pfnEnum     The callback function.
+ * @param   pvUser      The pvUser argument of the callback function.
+ */
+STAMR3DECL(int) STAMR3EnumU(PUVM pUVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser)
+{
+    STAMR3ENUMONEARGS Args;
+    Args.pVM     = pUVM->pVM;
+    Args.pfnEnum = pfnEnum;
+    Args.pvUser  = pvUser;
+
+    STAM_LOCK_RD(pUVM);
+    int rc = stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3EnumOne, &Args);
+    STAM_UNLOCK_RD(pUVM);
+    return rc;
+}
+
+
+/**
+ * Enumerate the statistics by the means of a callback function.
+ *
+ * @returns Whatever the callback returns.
+ *
  * @param   pVM         The VM handle.
  * @param   pszPat      The pattern to match samples.
@@ -1242,13 +1429,5 @@
 STAMR3DECL(int) STAMR3Enum(PVM pVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser)
 {
-    STAMR3ENUMONEARGS Args;
-    Args.pVM     = pVM;
-    Args.pfnEnum = pfnEnum;
-    Args.pvUser  = pvUser;
-
-    STAM_LOCK_RD(pVM);
-    int rc = stamR3Enum(pVM, pszPat, true /* fUpdateRing0 */, stamR3EnumOne, &Args);
-    STAM_UNLOCK_RD(pVM);
-    return rc;
+    return STAMR3EnumU(pVM->pUVM, pszPat, pfnEnum, pvUser);
 }
 
@@ -1419,5 +1598,5 @@
  *
  * @returns The rc from the callback.
- * @param   pVM             VM handle
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   pszPat          Pattern.
  * @param   fUpdateRing0    Update the ring-0 .
@@ -1427,5 +1606,5 @@
  * @param   pvArg           User parameter for the callback.
  */
-static int stamR3Enum(PVM pVM, const char *pszPat, bool fUpdateRing0, int (*pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg)
+static int stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0, int (*pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg)
 {
     int rc = VINF_SUCCESS;
@@ -1437,7 +1616,7 @@
     {
         if (fUpdateRing0)
-            stamR3Ring0StatsUpdate(pVM, "*");
-
-        PSTAMDESC   pCur = pVM->stam.s.pHead;
+            stamR3Ring0StatsUpdateU(pUVM, "*");
+
+        PSTAMDESC   pCur = pUVM->stam.s.pHead;
         while (pCur)
         {
@@ -1457,10 +1636,10 @@
     {
         if (fUpdateRing0)
-            stamR3Ring0StatsUpdate(pVM, pszPat);
+            stamR3Ring0StatsUpdateU(pUVM, pszPat);
 
         /** @todo This needs to be optimized since the GUI is using this path for the VM info dialog.
          * Note that it's doing exact matching. Organizing the samples in a tree would speed up thing
          * no end (at least for debug and profile builds). */
-        for (PSTAMDESC pCur = pVM->stam.s.pHead; pCur; pCur = pCur->pNext)
+        for (PSTAMDESC pCur = pUVM->stam.s.pHead; pCur; pCur = pCur->pNext)
             if (stamR3Match(pszPat, pCur->pszName))
             {
@@ -1489,8 +1668,8 @@
          */
         if (fUpdateRing0)
-            stamR3Ring0StatsUpdateMulti(pVM, papszExpressions, cExpressions);
+            stamR3Ring0StatsUpdateMultiU(pUVM, papszExpressions, cExpressions);
 
         unsigned iExpression = 0;
-        for (PSTAMDESC pCur = pVM->stam.s.pHead; pCur; pCur = pCur->pNext)
+        for (PSTAMDESC pCur = pUVM->stam.s.pHead; pCur; pCur = pCur->pNext)
             if (stamR3MultiMatch(papszExpressions, cExpressions, &iExpression, pCur->pszName))
             {
@@ -1511,13 +1690,13 @@
  * Registers the ring-0 statistics.
  *
- * @param   pVM         Pointer to the shared VM structure.
- */
-static void stamR3Ring0StatsRegister(PVM pVM)
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+static void stamR3Ring0StatsRegisterU(PUVM pUVM)
 {
     /* GVMM */
     for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
-        stamR3Register(pVM, (uint8_t *)&pVM->stam.s.GVMMStats + g_aGVMMStats[i].offVar, NULL, NULL,
-                       g_aGVMMStats[i].enmType, STAMVISIBILITY_ALWAYS, g_aGVMMStats[i].pszName,
-                       g_aGVMMStats[i].enmUnit, g_aGVMMStats[i].pszDesc);
+        stamR3RegisterU(pUVM, (uint8_t *)&pUVM->stam.s.GVMMStats + g_aGVMMStats[i].offVar, NULL, NULL,
+                        g_aGVMMStats[i].enmType, STAMVISIBILITY_ALWAYS, g_aGVMMStats[i].pszName,
+                        g_aGVMMStats[i].enmUnit, g_aGVMMStats[i].pszDesc);
 }
 
@@ -1526,10 +1705,10 @@
  * Updates the ring-0 statistics (the copy).
  *
- * @param   pVM         Pointer to the shared VM structure.
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   pszPat      The pattern.
  */
-static void stamR3Ring0StatsUpdate(PVM pVM, const char *pszPat)
-{
-    stamR3Ring0StatsUpdateMulti(pVM, &pszPat, 1);
+static void stamR3Ring0StatsUpdateU(PUVM pUVM, const char *pszPat)
+{
+    stamR3Ring0StatsUpdateMultiU(pUVM, &pszPat, 1);
 }
 
@@ -1541,10 +1720,11 @@
  * must be copied when needed.
  *
- * @param   pVM         Pointer to the shared VM structure.
+ * @param   pUVM        Pointer to the user mode VM structure.
  * @param   pszPat      The pattern (for knowing when to skip).
  */
-static void stamR3Ring0StatsUpdateMulti(PVM pVM, const char * const *papszExpressions, unsigned cExpressions)
-{
-    if (!pVM->pSession)
+static void stamR3Ring0StatsUpdateMultiU(PUVM pUVM, const char * const *papszExpressions, unsigned cExpressions)
+{
+    PVM pVM = pUVM->pVM;
+    if (!pVM || !pVM->pSession)
         return;
 
@@ -1565,5 +1745,5 @@
         int rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_GVMM_QUERY_STATISTICS, 0, &Req.Hdr);
         if (RT_SUCCESS(rc))
-            pVM->stam.s.GVMMStats = Req.Stats;
+            pUVM->stam.s.GVMMStats = Req.Stats;
     }
 }
@@ -1622,5 +1802,6 @@
     if (!pVM)
         return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires VM to be selected.\n");
-    if (!pVM->stam.s.pHead)
+    PUVM pUVM = pVM->pUVM;
+    if (!pUVM->stam.s.pHead)
         return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no statistics present.\n");
 
@@ -1633,7 +1814,7 @@
     Args.pfnPrintf = stamR3EnumDbgfPrintf;
 
-    STAM_LOCK_RD(pVM);
-    int rc = stamR3Enum(pVM, cArgs ? paArgs[0].u.pszString : NULL, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-    STAM_UNLOCK_RD(pVM);
+    STAM_LOCK_RD(pUVM);
+    int rc = stamR3EnumU(pUVM, cArgs ? paArgs[0].u.pszString : NULL, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    STAM_UNLOCK_RD(pUVM);
 
     return rc;
@@ -1677,5 +1858,6 @@
     if (!pVM)
         return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires VM to be selected.\n");
-    if (!pVM->stam.s.pHead)
+    PUVM pUVM = pVM->pUVM;
+    if (!pUVM->stam.s.pHead)
         return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no statistics present.\n");
 
@@ -1683,5 +1865,5 @@
      * Execute reset.
      */
-    int rc = STAMR3Reset(pVM, cArgs ? paArgs[0].u.pszString : NULL);
+    int rc = STAMR3ResetU(pUVM, cArgs ? paArgs[0].u.pszString : NULL);
     if (VBOX_SUCCESS(rc))
         return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "info: Statistics reset.\n");
Index: /trunk/src/VBox/VMM/STAMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/STAMInternal.h	(revision 6795)
+++ /trunk/src/VBox/VMM/STAMInternal.h	(revision 6796)
@@ -95,21 +95,8 @@
 
 /**
- * Converts a STAM pointer into a VM pointer.
- * @returns Pointer to the VM structure the STAM is part of.
- * @param   pSTAM   Pointer to STAM instance data.
+ * STAM data kept in the UVM.
  */
-#define STAM2VM(pSTAM)  ( (PVM)((char*)pSTAM - pSTAM->offVM) )
-
-
-/**
- * The stam data in the VM.
- */
-struct STAM
+typedef struct STAMUSERPERVM
 {
-    /** Offset to the VM structure.
-     * See STAM2VM(). */
-    RTINT                   offVM;
-    /** Alignment padding. */
-    RTUINT                  uPadding;
     /** Pointer to the first sample. */
     R3PTRTYPE(PSTAMDESC)    pHead;
@@ -119,14 +106,17 @@
     /** The copy of the GVMM statistics. */
     GVMMSTATS               GVMMStats;
-};
+} STAMUSERPERVM;
+/** Pointer to the STAM data kept in the UVM. */
+typedef STAMUSERPERVM *PSTAMUSERPERVM;
+
 
 /** Locks the sample descriptors for reading. */
-#define STAM_LOCK_RD(pVM)       do { int rcSem = RTSemRWRequestRead(pVM->stam.s.RWSem, RT_INDEFINITE_WAIT);  AssertRC(rcSem); } while (0)
+#define STAM_LOCK_RD(pUVM)      do { int rcSem = RTSemRWRequestRead(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT);  AssertRC(rcSem); } while (0)
 /** Locks the sample descriptors for writing. */
-#define STAM_LOCK_WR(pVM)       do { int rcSem = RTSemRWRequestWrite(pVM->stam.s.RWSem, RT_INDEFINITE_WAIT); AssertRC(rcSem); } while (0)
+#define STAM_LOCK_WR(pUVM)      do { int rcSem = RTSemRWRequestWrite(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT); AssertRC(rcSem); } while (0)
 /** UnLocks the sample descriptors after reading. */
-#define STAM_UNLOCK_RD(pVM)     do { int rcSem = RTSemRWReleaseRead(pVM->stam.s.RWSem);  AssertRC(rcSem); } while (0)
+#define STAM_UNLOCK_RD(pUVM)    do { int rcSem = RTSemRWReleaseRead(pUVM->stam.s.RWSem);  AssertRC(rcSem); } while (0)
 /** UnLocks the sample descriptors after writing. */
-#define STAM_UNLOCK_WR(pVM)     do { int rcSem = RTSemRWReleaseWrite(pVM->stam.s.RWSem); AssertRC(rcSem); } while (0)
+#define STAM_UNLOCK_WR(pUVM)    do { int rcSem = RTSemRWReleaseWrite(pUVM->stam.s.RWSem); AssertRC(rcSem); } while (0)
 
 /** @} */
Index: /trunk/src/VBox/VMM/VM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VM.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/VM.cpp	(revision 6796)
@@ -43,4 +43,5 @@
 #include "VMInternal.h"
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 
 #include <VBox/sup.h>
@@ -82,8 +83,8 @@
 *******************************************************************************/
 /** Pointer to the list of VMs. */
-static PVM          g_pVMsHead;
+static PUVM         g_pUVMsHead = NULL;
 
 /** Pointer to the list of at VM destruction callbacks. */
-static PVMATDTOR    g_pVMAtDtorHead;
+static PVMATDTOR    g_pVMAtDtorHead = NULL;
 /** Lock the g_pVMAtDtorHead list. */
 #define VM_ATDTOR_LOCK() do { } while (0)
@@ -91,10 +92,11 @@
 #define VM_ATDTOR_UNLOCK() do { } while (0)
 
+
 /*******************************************************************************
 *   Internal Functions                                                         *
 *******************************************************************************/
-static int  vmR3Create(PVM pVM, PFNVMATERROR pfnVMAtError, void *pvUserVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM);
-static void vmR3CallVMAtError(PFNVMATERROR pfnVMAtError, void *pvUser, int rc, RT_SRC_POS_DECL, const char *pszError, ...);
-static int  vmR3InitRing3(PVM pVM);
+static int  vmR3CreateUVM(PUVM *ppUVM);
+static int  vmR3CreateU(PUVM pUVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM);
+static int  vmR3InitRing3(PVM pVM, PUVM pUVM);
 static int  vmR3InitRing0(PVM pVM);
 static int  vmR3InitGC(PVM pVM);
@@ -106,13 +108,15 @@
 static DECLCALLBACK(int) vmR3Load(PVM pVM, const char *pszFilename, PFNVMPROGRESS pfnProgress, void *pvUser);
 static DECLCALLBACK(int) vmR3PowerOff(PVM pVM);
+static void vmR3DestroyUVM(PUVM pUVM);
 static void vmR3AtDtor(PVM pVM);
-static int  vmR3AtReset(PVM pVM);
+static int  vmR3AtResetU(PUVM pUVM);
 static DECLCALLBACK(int) vmR3Reset(PVM pVM);
-static DECLCALLBACK(int) vmR3AtStateRegister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser);
-static DECLCALLBACK(int) vmR3AtStateDeregister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser);
-static DECLCALLBACK(int) vmR3AtErrorRegister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser);
-static DECLCALLBACK(int) vmR3AtErrorDeregister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser);
-static DECLCALLBACK(int) vmR3AtRuntimeErrorRegister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser);
-static DECLCALLBACK(int) vmR3AtRuntimeErrorDeregister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser);
+static DECLCALLBACK(int) vmR3AtStateRegisterU(PUVM pUVM, PFNVMATSTATE pfnAtState, void *pvUser);
+static DECLCALLBACK(int) vmR3AtStateDeregisterU(PUVM pUVM, PFNVMATSTATE pfnAtState, void *pvUser);
+static DECLCALLBACK(int) vmR3AtErrorRegisterU(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser);
+static DECLCALLBACK(int) vmR3AtErrorDeregisterU(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser);
+static int               vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...);
+static DECLCALLBACK(int) vmR3AtRuntimeErrorRegisterU(PUVM pUVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser);
+static DECLCALLBACK(int) vmR3AtRuntimeErrorDeregisterU(PUVM pUVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser);
 
 
@@ -127,6 +131,6 @@
      * Only once.
      */
-    static bool fDone = false;
-    if (fDone)
+    static bool volatile s_fDone = false;
+    if (s_fDone)
         return VINF_SUCCESS;
 
@@ -134,5 +138,5 @@
      * We're done.
      */
-    fDone = true;
+    s_fDone = true;
     return VINF_SUCCESS;
 }
@@ -175,203 +179,129 @@
 
     /*
-     * Init support library and load the VMMR0.r0 module.
-     */
-    PSUPDRVSESSION pSession = 0;
-    int rc = SUPInit(&pSession, 0);
-    if (VBOX_SUCCESS(rc))
-    {
-        /** @todo This is isn't very nice, it would be preferrable to move the loader bits
-         * out of the VM structure and into a ring-3 only thing. There's a big deal of the
-         * error path that we now won't unload the VMMR0.r0 module in. This isn't such a
-         * big deal right now, but I'll have to get back to this later. UVM (bird) */
-        void *pvVMMR0Opaque;
-        rc = PDMR3LdrLoadVMMR0(&pvVMMR0Opaque);
+     * Create the UVM so we can register the at-error callback
+     * and consoliate a bit of cleanup code.
+     */
+    PUVM pUVM;
+    int rc = vmR3CreateUVM(&pUVM);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (pfnVMAtError)
+        rc = VMR3AtErrorRegisterU(pUVM, pfnVMAtError, pvUserVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Initialize the support library creating the session for this v
+         */
+        rc = SUPInit(&pUVM->vm.s.pSession, 0);
         if (RT_SUCCESS(rc))
         {
             /*
-             * Request GVMM to create a new VM for us.
+             * Call vmR3CreateU in the EMT thread and wait for it to finish.
              */
-            GVMMCREATEVMREQ CreateVMReq;
-            CreateVMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-            CreateVMReq.Hdr.cbReq = sizeof(CreateVMReq);
-            CreateVMReq.pSession = pSession;
-            CreateVMReq.pVMR0 = NIL_RTR0PTR;
-            CreateVMReq.pVMR3 = NULL;
-            rc = SUPCallVMMR0Ex(NIL_RTR0PTR, VMMR0_DO_GVMM_CREATE_VM, 0, &CreateVMReq.Hdr);
+            PVMREQ pReq;
+            rc = VMR3ReqCallU(pUVM, &pReq, RT_INDEFINITE_WAIT, 0, (PFNRT)vmR3CreateU,
+                              3, pUVM, pfnCFGMConstructor, pvUserCFGM);
             if (RT_SUCCESS(rc))
             {
-                PVM pVM = CreateVMReq.pVMR3;
-                AssertRelease(VALID_PTR(pVM));
-                Log(("VMR3Create: Created pVM=%p pVMR0=%p\n", pVM, pVM->pVMR0));
-                PDMR3LdrLoadVMMR0Part2(pVM, pvVMMR0Opaque);
-
-                /*
-                 * Do basic init of the VM structure.
-                 */
-                pVM->vm.s.offVM = RT_OFFSETOF(VM, vm.s);
-                pVM->vm.s.ppAtResetNext = &pVM->vm.s.pAtReset;
-                pVM->vm.s.ppAtStateNext = &pVM->vm.s.pAtState;
-                pVM->vm.s.ppAtErrorNext = &pVM->vm.s.pAtError;
-                pVM->vm.s.ppAtRuntimeErrorNext = &pVM->vm.s.pAtRuntimeError;
-                pVM->vm.s.enmHaltMethod = VMHALTMETHOD_1;
-                rc = RTSemEventCreate(&pVM->vm.s.EventSemWait);
+                rc = pReq->iStatus;
+                VMR3ReqFree(pReq);
                 if (RT_SUCCESS(rc))
                 {
                     /*
-                     * Initialize STAM.
+                     * Success!
                      */
-                    rc = STAMR3Init(pVM);
-                    if (VBOX_SUCCESS(rc))
-                    {
-                        /*
-                         * Create the EMT thread, it will start up and wait for requests to process.
-                         */
-                        VMEMULATIONTHREADARGS Args;
-                        Args.pVM = pVM;
-                        rc = RTThreadCreate(&pVM->ThreadEMT, vmR3EmulationThread, &Args, _1M,
-                                            RTTHREADTYPE_EMULATION, RTTHREADFLAGS_WAITABLE, "EMT");
-                        if (VBOX_SUCCESS(rc))
-                        {
-                            /*
-                             * Issue a VM Create request and wait for it to complete.
-                             */
-                            PVMREQ pReq;
-                            rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3Create, 5,
-                                             pVM, pfnVMAtError, pvUserVM, pfnCFGMConstructor, pvUserCFGM);
-                            if (VBOX_SUCCESS(rc))
-                            {
-                                rc = pReq->iStatus;
-                                VMR3ReqFree(pReq);
-                                if (VBOX_SUCCESS(rc))
-                                {
-                                    *ppVM = pVM;
-                                    LogFlow(("VMR3Create: returns VINF_SUCCESS *ppVM=%p\n", pVM));
-                                    return VINF_SUCCESS;
-                                }
-
-                                NoDmik(AssertMsgFailed(("vmR3Create failed rc=%Vrc\n", rc)));
-                            }
-                            else
-                                AssertMsgFailed(("VMR3ReqCall failed rc=%Vrc\n", rc));
-
-                            /*
-                             * An error occurred during VM creation. Set the error message directly
-                             * using the initial callback, as the callback list doesn't exist yet.
-                             */
-                            const char *pszError = NULL;
-                            switch (rc)
-                            {
-                                case VERR_VMX_IN_VMX_ROOT_MODE:
-#ifdef RT_OS_LINUX
-                                    pszError = N_("VirtualBox can't operate in VMX root mode. "
-                                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
-#else
-                                    pszError = N_("VirtualBox can't operate in VMX root mode");
-#endif
-                                    break;
-                                default:
-                                    /* XXX check if there was already an error message set! */
-                                    pszError = N_("Unknown error creating VM");
-                                    NoDmik(AssertMsgFailed(("Add error message for rc=%d (%Vrc)\n", rc, rc)));
-                            }
-                            if (pszError)
-                                vmR3CallVMAtError(pfnVMAtError, pvUserVM, rc, RT_SRC_POS, pszError, rc);
-
-                            /* Forcefully terminate the emulation thread. */
-                            VM_FF_SET(pVM, VM_FF_TERMINATE);
-                            VMR3NotifyFF(pVM, false);
-                            RTThreadWait(pVM->ThreadEMT, 1000, NULL);
-                        }
-
-                        int rc2 = STAMR3Term(pVM);
-                        AssertRC(rc2);
-                    }
-
-                    RTSemEventDestroy(pVM->vm.s.EventSemWait);
+                    *ppVM = pUVM->pVM;
+                    LogFlow(("VMR3Create: returns VINF_SUCCESS *ppVM=%p\n", *ppVM));
+                    return VINF_SUCCESS;
                 }
-                /* cleanup the heap. */
-                int rc2 = MMR3Term(pVM);
-                AssertRC(rc2);
-
-                /* Tell GVMM that it can destroy the VM now. */
-                rc2 = SUPCallVMMR0Ex(CreateVMReq.pVMR0, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
-                AssertRC(rc2);
             }
             else
+                AssertMsgFailed(("VMR3ReqCall failed rc=%Vrc\n", rc));
+
+            /*
+             * An error occurred during VM creation. Set the error message directly
+             * using the initial callback, as the callback list doesn't exist yet.
+             */
+            const char *pszError = NULL;
+            switch (rc)
             {
-                PDMR3LdrLoadVMMR0Part2(NULL, pvVMMR0Opaque);
-                vmR3CallVMAtError(pfnVMAtError, pvUserVM, rc, RT_SRC_POS, N_("VM creation failed"));
-                AssertMsgFailed(("GMMR0CreateVMReq returned %Rrc\n", rc));
+                case VERR_VMX_IN_VMX_ROOT_MODE:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox can't operate in VMX root mode. "
+                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
+#else
+                    pszError = N_("VirtualBox can't operate in VMX root mode");
+#endif
+                    break;
+
+                default:
+                    pszError = N_("Unknown error creating VM");
+                    NoDmik(AssertMsgFailed(("Add error message for rc=%d (%Vrc)\n", rc, rc)));
+                    break;
             }
+            vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
         }
         else
         {
-            vmR3CallVMAtError(pfnVMAtError, pvUserVM, rc, RT_SRC_POS, N_("Failed to load VMMR0.r0"));
-            AssertMsgFailed(("PDMR3LdrLoadVMMR0 returned %Rrc\n", rc));
+            /*
+             * An error occurred at support library initialization time (before the
+             * VM could be created). Set the error message directly using the
+             * initial callback, as the callback list doesn't exist yet.
+             */
+            const char *pszError;
+            switch (rc)
+            {
+                case VERR_VM_DRIVER_LOAD_ERROR:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox kernel driver not loaded. The vboxdrv kernel module "
+                                  "was either not loaded or /dev/vboxdrv is not set up properly. "
+                                  "Re-setup the kernel module by executing "
+                                  "'/etc/init.d/vboxdrv setup' as root");
+#else
+                    pszError = N_("VirtualBox kernel driver not loaded");
+#endif
+                    break;
+                case VERR_VM_DRIVER_OPEN_ERROR:
+                    pszError = N_("VirtualBox kernel driver cannot be opened");
+                    break;
+                case VERR_VM_DRIVER_NOT_ACCESSIBLE:
+#ifdef RT_OS_LINUX
+                    pszError = N_("The VirtualBox kernel driver is not accessible to the current "
+                                  "user. Make sure that the user has write permissions for "
+                                  "/dev/vboxdrv by adding them to the vboxusers groups. You "
+                                  "will need to logout for the change to take effect.");
+#else
+                    pszError = N_("VirtualBox kernel driver not accessible, permission problem");
+#endif
+                    break;
+                case VERR_VM_DRIVER_NOT_INSTALLED:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox kernel driver not installed. The vboxdrv kernel module "
+                                  "was either not loaded or /dev/vboxdrv was not created for some "
+                                  "reason. Re-setup the kernel module by executing "
+                                  "'/etc/init.d/vboxdrv setup' as root");
+#else
+                    pszError = N_("VirtualBox kernel driver not installed");
+#endif
+                    break;
+                case VERR_NO_MEMORY:
+                    pszError = N_("VirtualBox support library out of memory");
+                    break;
+                case VERR_VERSION_MISMATCH:
+                case VERR_VM_DRIVER_VERSION_MISMATCH:
+                    pszError = N_("The VirtualBox support driver which is running is from a different "
+                                  "version of VirtualBox.  You can correct this by stopping all "
+                                  "running instances of VirtualBox and reinstalling the software.");
+                    break;
+                default:
+                    pszError = N_("Unknown error initializing kernel driver");
+                    AssertMsgFailed(("Add error message for rc=%d (%Vrc)\n", rc, rc));
+            }
+            vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
         }
-
-        /* terminate SUPLib */
-        int rc2 = SUPTerm(false);
-        AssertRC(rc2);
-    }
-    else
-    {
-        /*
-         * An error occurred at support library initialization time (before the
-         * VM could be created). Set the error message directly using the
-         * initial callback, as the callback list doesn't exist yet.
-         */
-        const char *pszError;
-        switch (rc)
-        {
-            case VERR_VM_DRIVER_LOAD_ERROR:
-#ifdef RT_OS_LINUX
-                pszError = N_("VirtualBox kernel driver not loaded. The vboxdrv kernel module "
-		              "was either not loaded or /dev/vboxdrv is not set up properly. "
-			      "Re-setup the kernel module by executing "
-			      "'/etc/init.d/vboxdrv setup' as root");
-#else
-                pszError = N_("VirtualBox kernel driver not loaded");
-#endif
-                break;
-            case VERR_VM_DRIVER_OPEN_ERROR:
-                pszError = N_("VirtualBox kernel driver cannot be opened");
-                break;
-            case VERR_VM_DRIVER_NOT_ACCESSIBLE:
-#ifdef RT_OS_LINUX
-                pszError = N_("The VirtualBox kernel driver is not accessible to the current "
-		              "user. Make sure that the user has write permissions for "
-			      "/dev/vboxdrv by adding them to the vboxusers groups. You "
-			      "will need to logout for the change to take effect.");
-#else
-                pszError = N_("VirtualBox kernel driver not accessible, permission problem");
-#endif
-                break;
-            case VERR_VM_DRIVER_NOT_INSTALLED:
-#ifdef RT_OS_LINUX
-                pszError = N_("VirtualBox kernel driver not installed. The vboxdrv kernel module "
-		              "was either not loaded or /dev/vboxdrv was not created for some "
-			      "reason. Re-setup the kernel module by executing "
-			      "'/etc/init.d/vboxdrv setup' as root");
-#else
-                pszError = N_("VirtualBox kernel driver not installed");
-#endif
-                break;
-            case VERR_NO_MEMORY:
-                pszError = N_("VirtualBox support library out of memory");
-                break;
-            case VERR_VERSION_MISMATCH:
-            case VERR_VM_DRIVER_VERSION_MISMATCH:
-                pszError = N_("The VirtualBox support driver which is running is from a different "
-                              "version of VirtualBox.  You can correct this by stopping all "
-                              "running instances of VirtualBox and reinstalling the software.");
-                break;
-            default:
-                pszError = N_("Unknown error initializing kernel driver");
-                AssertMsgFailed(("Add error message for rc=%d (%Vrc)\n", rc, rc));
-        }
-        vmR3CallVMAtError(pfnVMAtError, pvUserVM, rc, RT_SRC_POS, pszError, rc);
-    }
-
+    }
+
+    /* cleanup */
+    vmR3DestroyUVM(pUVM);
     LogFlow(("VMR3Create: returns %Vrc\n", rc));
     return rc;
@@ -380,29 +310,109 @@
 
 /**
- * Wrapper for getting a correct va_list.
- */
-static void vmR3CallVMAtError(PFNVMATERROR pfnVMAtError, void *pvUser, int rc, RT_SRC_POS_DECL, const char *pszError, ...)
-{
-    if (!pfnVMAtError)
-        return;
-    va_list va;
-    va_start(va, pszError);
-    pfnVMAtError(NULL, pvUser, rc, RT_SRC_POS_ARGS, pszError, va);
-    va_end(va);
-}
-
-
-/**
- * Initializes the VM.
- */
-static int vmR3Create(PVM pVM, PFNVMATERROR pfnVMAtError, void *pvUserVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM)
+ * Creates the UVM.
+ *
+ * This will not initialize the support library even if vmR3DestroyUVM
+ * will terminate that.
+ *
+ * @returns VBox status code.
+ * @param   ppUVM   Where to store the UVM pointer.
+ */
+static int vmR3CreateUVM(PUVM *ppUVM)
+{
+    /*
+     * Create the UVM, initialize the fundamental stuff (VM+MMR3Heap+STAM)
+     * and start the emulation thread (EMT).
+     */
+    PUVM pUVM = (PUVM)RTMemAllocZ(sizeof(*pUVM));
+    AssertReturn(pUVM, VERR_NO_MEMORY);
+    pUVM->u32Magic = UVM_MAGIC;
+
+    AssertCompile(sizeof(pUVM->vm.s) <= sizeof(pUVM->vm.padding));
+    AssertRelease(sizeof(pUVM->vm.s) <= sizeof(pUVM->vm.padding));
+
+    pUVM->vm.s.ppAtResetNext = &pUVM->vm.s.pAtReset;
+    pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
+    pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
+    pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
+    pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_BOOTSTRAP;
+    int rc = RTSemEventCreate(&pUVM->vm.s.EventSemWait);
+    if (RT_SUCCESS(rc))
+    {
+        rc = STAMR3InitUVM(pUVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = MMR3InitUVM(pUVM);
+            if (RT_SUCCESS(rc))
+            {
+                rc = PDMR3InitUVM(pUVM);
+                if (RT_SUCCESS(rc))
+                {
+
+                    rc = RTThreadCreate(&pUVM->vm.s.ThreadEMT, vmR3EmulationThread, pUVM, _1M,
+                                        RTTHREADTYPE_EMULATION, RTTHREADFLAGS_WAITABLE, "EMT");
+                    if (RT_SUCCESS(rc))
+                    {
+                        *ppUVM = pUVM;
+                        return VINF_SUCCESS;
+                    }
+
+                    /* bail out. */
+                    PDMR3TermUVM(pUVM);
+                }
+                MMR3TermUVM(pUVM);
+            }
+            STAMR3TermUVM(pUVM);
+        }
+        RTSemEventDestroy(pUVM->vm.s.EventSemWait);
+    }
+    RTMemFree(pUVM);
+    return rc;
+}
+
+
+/**
+ * Creates and initializes the VM.
+ *
+ * @thread EMT
+ */
+static int vmR3CreateU(PUVM pUVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM)
 {
     int rc = VINF_SUCCESS;
 
-    /* Register error callback if specified. */
-    if (pfnVMAtError)
-        rc = VMR3AtErrorRegister(pVM, pfnVMAtError, pvUserVM);
-    if (VBOX_SUCCESS(rc))
-    {
+    /*
+     * Load the VMMR0.r0 module so that we can call GVMMR0CreateVM.
+     */
+    rc = PDMR3LdrLoadVMMR0U(pUVM);
+    if (RT_FAILURE(rc))
+        return vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("Failed to load VMMR0.r0"));
+
+    /*
+     * Request GVMM to create a new VM for us.
+     */
+    GVMMCREATEVMREQ CreateVMReq;
+    CreateVMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    CreateVMReq.Hdr.cbReq = sizeof(CreateVMReq);
+    CreateVMReq.pSession = pUVM->vm.s.pSession;
+    CreateVMReq.pVMR0 = NIL_RTR0PTR;
+    CreateVMReq.pVMR3 = NULL;
+    rc = SUPCallVMMR0Ex(NIL_RTR0PTR, VMMR0_DO_GVMM_CREATE_VM, 0, &CreateVMReq.Hdr);
+    if (RT_SUCCESS(rc))
+    {
+        PVM pVM = pUVM->pVM = CreateVMReq.pVMR3;
+        AssertRelease(VALID_PTR(pVM));
+        AssertRelease(pVM->pVMR0 == CreateVMReq.pVMR0);
+        AssertRelease(pVM->pSession == pUVM->vm.s.pSession);
+
+        Log(("VMR3Create: Created pUVM=%p pVM=%p pVMR0=%p hSelf=%#x \n", pUVM, pVM, pVM->pVMR0, pVM->hSelf));
+
+        /*
+         * Initialize the VM structure and our internal data (VMINT).
+         */
+        pVM->pUVM = pUVM;
+        pVM->ThreadEMT = pUVM->vm.s.ThreadEMT;
+        pVM->NativeThreadEMT = pUVM->vm.s.NativeThreadEMT;
+
+        pVM->vm.s.offVM = RT_OFFSETOF(VM, vm.s);
+
         /*
          * Init the configuration.
@@ -424,71 +434,64 @@
 
             /*
-             * Check if the required minimum of resources are available.
+             * Init the Ring-3 components and do a round of relocations with 0 delta.
              */
-            /** @todo Check if the required minimum of resources are available. */
+            rc = vmR3InitRing3(pVM, pUVM);
             if (VBOX_SUCCESS(rc))
             {
+                VMR3Relocate(pVM, 0);
+                LogFlow(("Ring-3 init succeeded\n"));
+
                 /*
-                 * Init the Ring-3 components and do a round of relocations with 0 delta.
+                 * Init the Ring-0 components.
                  */
-                rc = vmR3InitRing3(pVM);
+                rc = vmR3InitRing0(pVM);
                 if (VBOX_SUCCESS(rc))
                 {
+                    /* Relocate again, because some switcher fixups depends on R0 init results. */
                     VMR3Relocate(pVM, 0);
-                    LogFlow(("Ring-3 init succeeded\n"));
-
+
+#ifdef VBOX_WITH_DEBUGGER
                     /*
-                     * Init the Ring-0 components.
+                     * Init the tcp debugger console if we're building
+                     * with debugger support.
                      */
-                    rc = vmR3InitRing0(pVM);
-                    if (VBOX_SUCCESS(rc))
+                    void *pvUser = NULL;
+                    rc = DBGCTcpCreate(pVM, &pvUser);
+                    if (    VBOX_SUCCESS(rc)
+                        ||  rc == VERR_NET_ADDRESS_IN_USE)
                     {
-                        /* Relocate again, because some switcher fixups depends on R0 init results. */
-                        VMR3Relocate(pVM, 0);
-
-#ifdef VBOX_WITH_DEBUGGER
+                        pUVM->vm.s.pvDBGC = pvUser;
+#endif
                         /*
-                         * Init the tcp debugger console if we're building
-                         * with debugger support.
+                         * Init the Guest Context components.
                          */
-                        void *pvUser = NULL;
-                        rc = DBGCTcpCreate(pVM, &pvUser);
-                        if (    VBOX_SUCCESS(rc)
-                            ||  rc == VERR_NET_ADDRESS_IN_USE)
+                        rc = vmR3InitGC(pVM);
+                        if (VBOX_SUCCESS(rc))
                         {
-                            pVM->vm.s.pvDBGC = pvUser;
-#endif
                             /*
-                             * Init the Guest Context components.
+                             * Now we can safely set the VM halt method to default.
                              */
-                            rc = vmR3InitGC(pVM);
-                            if (VBOX_SUCCESS(rc))
+                            rc = vmR3SetHaltMethodU(pUVM, VMHALTMETHOD_DEFAULT);
+                            if (RT_SUCCESS(rc))
                             {
                                 /*
-                                 * Now we can safely set the VM halt method to default.
+                                 * Set the state and link into the global list.
                                  */
-                                rc = vmR3SetHaltMethod(pVM, VMHALTMETHOD_DEFAULT);
-                                if (RT_SUCCESS(rc))
-                                {
-                                    /*
-                                     * Set the state and link into the global list.
-                                     */
-                                    vmR3SetState(pVM, VMSTATE_CREATED);
-                                    pVM->pNext = g_pVMsHead;
-                                    g_pVMsHead = pVM;
-                                    return VINF_SUCCESS;
-                                }
+                                vmR3SetState(pVM, VMSTATE_CREATED);
+                                pUVM->pNext = g_pUVMsHead;
+                                g_pUVMsHead = pUVM;
+                                return VINF_SUCCESS;
                             }
+                        }
 #ifdef VBOX_WITH_DEBUGGER
-                            DBGCTcpTerminate(pVM, pVM->vm.s.pvDBGC);
-                            pVM->vm.s.pvDBGC = NULL;
-                        }
+                        DBGCTcpTerminate(pVM, pUVM->vm.s.pvDBGC);
+                        pUVM->vm.s.pvDBGC = NULL;
+                    }
 #endif
-                        //..
-                    }
-                    vmR3Destroy(pVM);
+                    //..
                 }
-                //..
+                vmR3Destroy(pVM);
             }
+            //..
 
             /* Clean CFGM. */
@@ -496,8 +499,13 @@
             AssertRC(rc2);
         }
-        //..
-    }
-
-    LogFlow(("vmR3Create: returns %Vrc\n", rc));
+
+        /* Tell GVMM that it can destroy the VM now. */
+        int rc2 = SUPCallVMMR0Ex(CreateVMReq.pVMR0, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
+        AssertRC(rc2);
+    }
+    else
+        vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("VM creation failed (GVMM)"));
+
+    LogFlow(("vmR3Create: returns %Rrc\n", rc));
     return rc;
 }
@@ -508,5 +516,5 @@
  * Initializes all R3 components of the VM
  */
-static int vmR3InitRing3(PVM pVM)
+static int vmR3InitRing3(PVM pVM, PUVM pUVM)
 {
     int rc;
@@ -521,10 +529,4 @@
         STAM_REG(pVM, &pVM->StatSwitcherToGC,       STAMTYPE_PROFILE_ADV, "/PROF/VM/SwitchToGC",    STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
         STAM_REG(pVM, &pVM->StatSwitcherToHC,       STAMTYPE_PROFILE_ADV, "/PROF/VM/SwitchToHC",    STAMUNIT_TICKS_PER_CALL,    "Profiling switching to HC.");
-
-        STAM_REL_REG(pVM, &pVM->vm.s.StatHaltYield, STAMTYPE_PROFILE,     "/PROF/VM/Halt/Yield",    STAMUNIT_TICKS_PER_CALL,    "Profiling halted state yielding.");
-        STAM_REL_REG(pVM, &pVM->vm.s.StatHaltBlock, STAMTYPE_PROFILE,     "/PROF/VM/Halt/Block",    STAMUNIT_TICKS_PER_CALL,    "Profiling halted state blocking.");
-        STAM_REL_REG(pVM, &pVM->vm.s.StatHaltTimers,STAMTYPE_PROFILE,     "/PROF/VM/Halt/Timers",   STAMUNIT_TICKS_PER_CALL,    "Profiling halted state timer tasks.");
-        STAM_REL_REG(pVM, &pVM->vm.s.StatHaltPoll,  STAMTYPE_PROFILE,     "/PROF/VM/Halt/Poll",     STAMUNIT_TICKS_PER_CALL,    "Profiling halted state poll tasks.");
-
         STAM_REG(pVM, &pVM->StatSwitcherSaveRegs,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/SaveRegs", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
         STAM_REG(pVM, &pVM->StatSwitcherSysEnter,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/SysEnter", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
@@ -539,9 +541,14 @@
         STAM_REG(pVM, &pVM->StatSwitcherRstrRegs,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/RstrRegs", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
 
-        STAM_REG(pVM, &pVM->vm.s.StatReqAllocNew,   STAMTYPE_COUNTER,     "/VM/Req/AllocNew",       STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a new packet.");
-        STAM_REG(pVM, &pVM->vm.s.StatReqAllocRaces, STAMTYPE_COUNTER,     "/VM/Req/AllocRaces",     STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc causing races.");
-        STAM_REG(pVM, &pVM->vm.s.StatReqAllocRecycled, STAMTYPE_COUNTER,  "/VM/Req/AllocRecycled",  STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a recycled packet.");
-        STAM_REG(pVM, &pVM->vm.s.StatReqFree,       STAMTYPE_COUNTER,     "/VM/Req/Free",           STAMUNIT_OCCURENCES,        "Number of VMR3ReqFree calls.");
-        STAM_REG(pVM, &pVM->vm.s.StatReqFreeOverflow, STAMTYPE_COUNTER,   "/VM/Req/FreeOverflow",   STAMUNIT_OCCURENCES,        "Number of times the request was actually freed.");
+        STAM_REL_REG(pVM, &pUVM->vm.s.StatHaltYield, STAMTYPE_PROFILE,     "/PROF/VM/Halt/Yield",    STAMUNIT_TICKS_PER_CALL,    "Profiling halted state yielding.");
+        STAM_REL_REG(pVM, &pUVM->vm.s.StatHaltBlock, STAMTYPE_PROFILE,     "/PROF/VM/Halt/Block",    STAMUNIT_TICKS_PER_CALL,    "Profiling halted state blocking.");
+        STAM_REL_REG(pVM, &pUVM->vm.s.StatHaltTimers,STAMTYPE_PROFILE,     "/PROF/VM/Halt/Timers",   STAMUNIT_TICKS_PER_CALL,    "Profiling halted state timer tasks.");
+        STAM_REL_REG(pVM, &pUVM->vm.s.StatHaltPoll,  STAMTYPE_PROFILE,     "/PROF/VM/Halt/Poll",     STAMUNIT_TICKS_PER_CALL,    "Profiling halted state poll tasks.");
+
+        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocNew,   STAMTYPE_COUNTER,     "/VM/Req/AllocNew",       STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a new packet.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRaces, STAMTYPE_COUNTER,     "/VM/Req/AllocRaces",     STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc causing races.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRecycled, STAMTYPE_COUNTER,  "/VM/Req/AllocRecycled",  STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a recycled packet.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqFree,       STAMTYPE_COUNTER,     "/VM/Req/Free",           STAMUNIT_OCCURENCES,        "Number of VMR3ReqFree calls.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqFreeOverflow, STAMTYPE_COUNTER,   "/VM/Req/FreeOverflow",   STAMUNIT_OCCURENCES,        "Number of times the request was actually freed.");
 
         rc = CPUMR3Init(pVM);
@@ -756,5 +763,5 @@
      */
     PGMR3Relocate(pVM, offDelta);
-    PDMR3LdrRelocate(pVM, offDelta);
+    PDMR3LdrRelocateU(pVM->pUVM, offDelta);
     PGMR3Relocate(pVM, 0);              /* Repeat after PDM relocation. */
     CPUMR3Relocate(pVM);
@@ -1332,6 +1339,5 @@
  * The VM handle is destroyed and can no longer be used up successful return.
  *
- * @returns 0 on success.
- * @returns VBox error code on failure.
+ * @returns VBox status code.
  * @param   pVM     VM which should be destroyed.
  * @thread      Any thread but the emulation thread.
@@ -1348,34 +1354,29 @@
     if (!pVM)
         return VERR_INVALID_PARAMETER;
-    if (    pVM->enmVMState != VMSTATE_OFF
-        &&  pVM->enmVMState != VMSTATE_CREATED)
-    {
-        AssertMsgFailed(("Invalid VM state %d\n", pVM->enmVMState));
-        return VERR_VM_INVALID_VM_STATE;
-    }
-
-    /*
-     * Unlink the VM and change it's state to destroying.
-     */
-/** @todo lock this when we start having multiple machines in a process... */
-    PVM pPrev = NULL;
-    PVM pCur = g_pVMsHead;
-    while (pCur && pCur != pVM)
-    {
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-    if (!pCur)
-    {
-        AssertMsgFailed(("pVM=%p is INVALID!\n", pVM));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (pPrev)
-        pPrev->pNext = pCur->pNext;
+    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
+    AssertMsgReturn(   pVM->enmVMState == VMSTATE_OFF
+                    || pVM->enmVMState == VMSTATE_CREATED,
+                    ("Invalid VM state %d\n", pVM->enmVMState),
+                    VERR_VM_INVALID_VM_STATE);
+
+    /*
+     * Change VM state to destroying and unlink the VM.
+     */
+    vmR3SetState(pVM, VMSTATE_DESTROYING);
+
+    /** @todo lock this when we start having multiple machines in a process... */
+    PUVM pUVM = pVM->pUVM; AssertPtr(pUVM);
+    if (g_pUVMsHead == pUVM)
+        g_pUVMsHead = pUVM->pNext;
     else
-        g_pVMsHead = pCur->pNext;
-
-    vmR3SetState(pVM, VMSTATE_DESTROYING);
-
+    {
+        PUVM pPrev = g_pUVMsHead;
+        while (pPrev && pPrev->pNext != pUVM)
+            pPrev = pPrev->pNext;
+        AssertMsgReturn(pPrev, ("pUVM=%p / pVM=%p  is INVALID!\n", pUVM, pVM), VERR_INVALID_PARAMETER);
+
+        pPrev->pNext = pUVM->pNext;
+    }
+    pUVM->pNext = NULL;
 
     /*
@@ -1385,7 +1386,4 @@
     vmR3AtDtor(pVM);
 
-    pVM->pNext = g_pVMsHead;
-    g_pVMsHead = pVM;
-
     /*
      * If we are the EMT we'll delay the cleanup till later.
@@ -1393,5 +1391,6 @@
     if (VM_IS_EMT(pVM))
     {
-        pVM->vm.s.fEMTDoesTheCleanup = true;
+        pUVM->vm.s.fEMTDoesTheCleanup = true;
+        pUVM->vm.s.fTerminateEMT = true;
         VM_FF_SET(pVM, VM_FF_TERMINATE);
     }
@@ -1402,9 +1401,8 @@
          */
         PVMREQ pReq = NULL;
-        int rc = VMR3ReqCall(pVM, &pReq, 0, (PFNRT)vmR3Destroy, 1, pVM);
-        while (rc == VERR_TIMEOUT)
-            rc = VMR3ReqWait(pReq, RT_INDEFINITE_WAIT);
-        if (VBOX_SUCCESS(rc))
+        int rc = VMR3ReqCallU(pUVM, &pReq, RT_INDEFINITE_WAIT, 0, (PFNRT)vmR3Destroy, 1, pVM);
+        if (RT_SUCCESS(rc))
             rc = pReq->iStatus;
+        AssertRC(rc);
         VMR3ReqFree(pReq);
 
@@ -1412,18 +1410,12 @@
          * Wait for the EMT thread to terminate.
          */
-        VM_FF_SET(pVM, VM_FF_TERMINATE);
-        uint64_t u64Start = RTTimeMilliTS();
-        do
-        {
-            VMR3NotifyFF(pVM, false);
-            rc = RTThreadWait(pVM->ThreadEMT, 1000, NULL);
-        } while (   RTTimeMilliTS() - u64Start < 30000 /* 30 sec */
-                 && rc == VERR_TIMEOUT);
-        AssertMsgRC(rc, ("EMT thread wait failed, rc=%Vrc\n", rc));
+        Assert(pUVM->vm.s.fTerminateEMT);
+        rc = RTThreadWait(pUVM->vm.s.ThreadEMT, 30000, NULL);
+        AssertMsgRC(rc, ("EMT thread wait failed, rc=%Rrc\n", rc));
 
         /*
          * Now do the final bit where the heap and VM structures are freed up.
          */
-        vmR3DestroyFinalBit(pVM);
+        vmR3DestroyUVM(pUVM);
     }
 
@@ -1434,6 +1426,7 @@
 
 /**
- * Internal destruction worker. This will do nearly all of the
- * job, including quitting the emulation thread.
+ * Internal destruction worker.
+ *
+ * This will do nearly all of the job, including sacking the EMT.
  *
  * @returns VBox status.
@@ -1442,5 +1435,6 @@
 DECLCALLBACK(int) vmR3Destroy(PVM pVM)
 {
-    LogFlow(("vmR3Destroy: pVM=%p\n", pVM));
+    PUVM pUVM = pVM->pUVM;
+    LogFlow(("vmR3Destroy: pVM=%p pUVM=%p\n", pVM, pUVM));
     VM_ASSERT_EMT(pVM);
 
@@ -1467,6 +1461,6 @@
     AssertRC(rc);
 #ifdef VBOX_WITH_DEBUGGER
-    rc = DBGCTcpTerminate(pVM, pVM->vm.s.pvDBGC);
-    pVM->vm.s.pvDBGC = NULL;
+    rc = DBGCTcpTerminate(pVM, pUVM->vm.s.pvDBGC);
+    pVM->pUVM->vm.s.pvDBGC = NULL;
 #endif
     AssertRC(rc);
@@ -1497,14 +1491,14 @@
     rc = CPUMR3Term(pVM);
     AssertRC(rc);
-    rc = STAMR3Term(pVM);
-    AssertRC(rc);
     rc = PDMR3CritSectTerm(pVM);
     AssertRC(rc);
-    /* MM is destroyed later in vmR3DestroyFinalBit() for heap reasons. */
-
-    /*
-     * We're done in this thread.
-     */
-    pVM->fForcedActions = VM_FF_TERMINATE;
+    rc = MMR3Term(pVM);
+    AssertRC(rc);
+
+    /*
+     * We're done in this thread (EMT).
+     */
+    ASMAtomicUoWriteBool(&pVM->pUVM->vm.s.fTerminateEMT, true);
+    ASMAtomicWriteU32(&pVM->fForcedActions, VM_FF_TERMINATE);
     LogFlow(("vmR3Destroy: returning %Vrc\n", VINF_EM_TERMINATE));
     return VINF_EM_TERMINATE;
@@ -1513,19 +1507,75 @@
 
 /**
- * Does the final part of the VM destruction.
- * This is called by EMT in it's final stage or by the VMR3Destroy caller.
+ * Destroys the shared VM structure, leaving only UVM behind.
+ *
+ * This is called by EMT right before terminating.
+ *
+ * @param   pUVM        The UVM handle.
+ */
+void vmR3DestroyFinalBitFromEMT(PUVM pUVM)
+{
+    if (pUVM->pVM)
+    {
+        /*
+         * Modify state and then terminate MM.
+         * (MM must be delayed until this point so we don't destroy the callbacks and the request packet.)
+         */
+        vmR3SetState(pUVM->pVM, VMSTATE_TERMINATED);
+
+        /*
+         * Tell GVMM to destroy the VM and free its resources.
+         */
+        int rc = SUPCallVMMR0Ex(pUVM->pVM->pVMR0, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
+        AssertRC(rc);
+        pUVM->pVM = NULL;
+    }
+
+    /*
+     * Did EMT call VMR3Destroy and have to do it all?
+     */
+    if (pUVM->vm.s.fEMTDoesTheCleanup)
+        vmR3DestroyUVM(pUVM);
+}
+
+
+/**
+ * Destroys the UVM portion.
+ *
+ * This is called as the final step in the VM destruction or as the cleanup
+ * in case of a creation failure. If EMT called VMR3Destroy, meaning
+ * VMINTUSERPERVM::fEMTDoesTheCleanup is true, it will call this as
+ * vmR3DestroyFinalBitFromEMT completes.
  *
  * @param   pVM     VM Handle.
  */
-void vmR3DestroyFinalBit(PVM pVM)
-{
+static void vmR3DestroyUVM(PUVM pUVM)
+{
+    /*
+     * Terminate the EMT if still running (creation failure only).
+     */
+    if (!pUVM->vm.s.fTerminateEMT)
+    {
+        ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+        if (pUVM->pVM)
+        {
+            VM_FF_SET(pUVM->pVM, VM_FF_TERMINATE);
+            VMR3NotifyFF(pUVM->pVM, false);
+        }
+        RTSemEventSignal(pUVM->vm.s.EventSemWait);
+
+        int rc2 = RTThreadWait(pUVM->vm.s.ThreadEMT, 2000, NULL);
+        AssertRC(rc2);
+    }
+    RTSemEventDestroy(pUVM->vm.s.EventSemWait);
+    pUVM->vm.s.EventSemWait = NIL_RTSEMEVENT;
+
     /*
      * Free the event semaphores associated with the request packets.
      */
     unsigned cReqs = 0;
-    for (unsigned i = 0; i < ELEMENTS(pVM->vm.s.apReqFree); i++)
-    {
-        PVMREQ pReq = pVM->vm.s.apReqFree[i];
-        pVM->vm.s.apReqFree[i] = NULL;
+    for (unsigned i = 0; i < RT_ELEMENTS(pUVM->vm.s.apReqFree); i++)
+    {
+        PVMREQ pReq = pUVM->vm.s.apReqFree[i];
+        pUVM->vm.s.apReqFree[i] = NULL;
         for (; pReq; pReq = pReq->pNext, cReqs++)
         {
@@ -1534,5 +1584,5 @@
         }
     }
-    Assert(cReqs == pVM->vm.s.cReqFree); NOREF(cReqs);
+    Assert(cReqs == pUVM->vm.s.cReqFree); NOREF(cReqs);
 
     /*
@@ -1541,5 +1591,5 @@
     for (unsigned i = 0; i < 10; i++)
     {
-        PVMREQ pReqHead = (PVMREQ)ASMAtomicXchgPtr((void *volatile *)&pVM->vm.s.pReqs, NULL);
+        PVMREQ pReqHead = (PVMREQ)ASMAtomicXchgPtr((void *volatile *)&pUVM->vm.s.pReqs, NULL);
         AssertMsg(!pReqHead, ("This isn't supposed to happen! VMR3Destroy caller has to serialize this.\n"));
         if (!pReqHead)
@@ -1547,6 +1597,6 @@
         for (PVMREQ pReq = pReqHead; pReq; pReq = pReq->pNext)
         {
-            ASMAtomicXchgSize(&pReq->iStatus, VERR_INTERNAL_ERROR);
-            ASMAtomicXchgSize(&pReq->enmState, VMREQSTATE_INVALID);
+            ASMAtomicUoWriteSize(&pReq->iStatus, VERR_INTERNAL_ERROR);
+            ASMAtomicWriteSize(&pReq->enmState, VMREQSTATE_INVALID);
             RTSemEventSignal(pReq->EventSem);
             RTThreadSleep(2);
@@ -1558,18 +1608,26 @@
 
     /*
-     * Modify state and then terminate MM.
-     * (MM must be delayed until this point so we don't destroy the callbacks and the request packet.)
-     */
-    vmR3SetState(pVM, VMSTATE_TERMINATED);
-    int rc = MMR3Term(pVM);
-    AssertRC(rc);
-
-    /*
-     * Tell GVMM that it can destroy the VM now.
-     */
-    rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
-    AssertRC(rc);
-    rc = SUPTerm();
-    AssertRC(rc);
+     * Make sure the VMMR0.r0 module and whatever else is unloaded.
+     */
+    PDMR3TermUVM(pUVM);
+
+    /*
+     * Terminate the support library if initialized.
+     */
+    if (pUVM->vm.s.pSession)
+    {
+        int rc = SUPTerm();
+        AssertRC(rc);
+        pUVM->vm.s.pSession = NIL_RTR0PTR;
+    }
+
+    /*
+     * Destroy the MM heap and free the UVM structure.
+     */
+    MMR3TermUVM(pUVM);
+    STAMR3TermUVM(pUVM);
+
+    ASMAtomicUoWriteU32(&pUVM->u32Magic, UINT32_MAX);
+    RTMemFree(pUVM);
 
     RTLogFlush(NULL);
@@ -1591,7 +1649,10 @@
      * destroyed during the enumeration.
      */
+    PUVM pNext;
     if (pVMPrev)
-        return pVMPrev->pNext;
-    return g_pVMsHead;
+        pNext = pVMPrev->pUVM->pNext;
+    else
+        pNext = g_pUVMsHead;
+    return pNext ? pNext->pVM : NULL;
 }
 
@@ -1781,5 +1842,5 @@
     SELMR3Reset(pVM);
     TRPMR3Reset(pVM);
-    vmR3AtReset(pVM);
+    vmR3AtResetU(pVM->pUVM);
     REMR3Reset(pVM);
     IOMR3Reset(pVM);
@@ -1813,7 +1874,7 @@
  * @returns VBox status code.
  *          Any failure is fatal.
- * @param   pVM     The VM which is being reset.
- */
-static int vmR3AtReset(PVM pVM)
+ * @param   pUVM        Pointe to the user mode VM structure.
+ */
+static int vmR3AtResetU(PUVM pUVM)
 {
     /*
@@ -1821,5 +1882,5 @@
      */
     int rc = VINF_SUCCESS;
-    for (PVMATRESET pCur = pVM->vm.s.pAtReset; pCur; pCur = pCur->pNext)
+    for (PVMATRESET pCur = pUVM->vm.s.pAtReset; pCur; pCur = pCur->pNext)
     {
         /* do the call */
@@ -1830,5 +1891,5 @@
                 break;
             case VMATRESETTYPE_INTERNAL:
-                rc = pCur->u.Internal.pfnCallback(pVM, pCur->pvUser);
+                rc = pCur->u.Internal.pfnCallback(pUVM->pVM, pCur->pvUser);
                 break;
             case VMATRESETTYPE_EXTERNAL:
@@ -1854,10 +1915,10 @@
  * Internal registration function
  */
-static int vmr3AtResetRegister(PVM pVM, void *pvUser, const char *pszDesc, PVMATRESET *ppNew)
+static int vmr3AtResetRegisterU(PUVM pUVM, void *pvUser, const char *pszDesc, PVMATRESET *ppNew)
 {
     /*
      * Allocate restration structure.
      */
-    PVMATRESET pNew = (PVMATRESET)MMR3HeapAlloc(pVM, MM_TAG_VM,  sizeof(*pNew));
+    PVMATRESET pNew = (PVMATRESET)MMR3HeapAllocU(pUVM, MM_TAG_VM,  sizeof(*pNew));
     if (pNew)
     {
@@ -1868,6 +1929,6 @@
 
         /* insert */
-        *pVM->vm.s.ppAtResetNext = pNew;
-        pVM->vm.s.ppAtResetNext = &pNew->pNext;
+        *pUVM->vm.s.ppAtResetNext = pNew;
+        pUVM->vm.s.ppAtResetNext = &pNew->pNext;
 
         *ppNew = pNew;
@@ -1903,5 +1964,5 @@
      */
     PVMATRESET pNew;
-    int rc = vmr3AtResetRegister(pVM, pvUser, pszDesc, &pNew);
+    int rc = vmr3AtResetRegisterU(pVM->pUVM, pvUser, pszDesc, &pNew);
     if (VBOX_SUCCESS(rc))
     {
@@ -1942,5 +2003,5 @@
      */
     PVMATRESET pNew;
-    int rc = vmr3AtResetRegister(pVM, pvUser, pszDesc, &pNew);
+    int rc = vmr3AtResetRegisterU(pVM->pUVM, pvUser, pszDesc, &pNew);
     if (VBOX_SUCCESS(rc))
     {
@@ -1980,5 +2041,5 @@
      */
     PVMATRESET pNew;
-    int rc = vmr3AtResetRegister(pVM, pvUser, pszDesc, &pNew);
+    int rc = vmr3AtResetRegisterU(pVM->pUVM, pvUser, pszDesc, &pNew);
     if (VBOX_SUCCESS(rc))
     {
@@ -1998,9 +2059,9 @@
  *
  * @returns Pointer to the next callback structure.
- * @param   pVM             The VM.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pCur            The one to free.
  * @param   pPrev           The one before pCur.
  */
-static PVMATRESET vmr3AtResetFree(PVM pVM, PVMATRESET pCur, PVMATRESET pPrev)
+static PVMATRESET vmr3AtResetFreeU(PUVM pUVM, PVMATRESET pCur, PVMATRESET pPrev)
 {
     /*
@@ -2012,11 +2073,11 @@
         pPrev->pNext = pNext;
         if (!pNext)
-            pVM->vm.s.ppAtResetNext = &pPrev->pNext;
+            pUVM->vm.s.ppAtResetNext = &pPrev->pNext;
     }
     else
     {
-        pVM->vm.s.pAtReset = pNext;
+        pUVM->vm.s.pAtReset = pNext;
         if (!pNext)
-            pVM->vm.s.ppAtResetNext = &pVM->vm.s.pAtReset;
+            pUVM->vm.s.ppAtResetNext = &pUVM->vm.s.pAtReset;
     }
 
@@ -2042,5 +2103,5 @@
     int         rc = VERR_VM_ATRESET_NOT_FOUND;
     PVMATRESET  pPrev = NULL;
-    PVMATRESET  pCur = pVM->vm.s.pAtReset;
+    PVMATRESET  pCur = pVM->pUVM->vm.s.pAtReset;
     while (pCur)
     {
@@ -2049,5 +2110,5 @@
             &&  (!pfnCallback || pCur->u.Dev.pfnCallback == pfnCallback))
         {
-            pCur = vmr3AtResetFree(pVM, pCur, pPrev);
+            pCur = vmr3AtResetFreeU(pVM->pUVM, pCur, pPrev);
             rc = VINF_SUCCESS;
         }
@@ -2075,5 +2136,5 @@
     int         rc = VERR_VM_ATRESET_NOT_FOUND;
     PVMATRESET  pPrev = NULL;
-    PVMATRESET  pCur = pVM->vm.s.pAtReset;
+    PVMATRESET  pCur = pVM->pUVM->vm.s.pAtReset;
     while (pCur)
     {
@@ -2081,5 +2142,5 @@
             &&  pCur->u.Internal.pfnCallback == pfnCallback)
         {
-            pCur = vmr3AtResetFree(pVM, pCur, pPrev);
+            pCur = vmr3AtResetFreeU(pVM->pUVM, pCur, pPrev);
             rc = VINF_SUCCESS;
         }
@@ -2107,5 +2168,5 @@
     int         rc = VERR_VM_ATRESET_NOT_FOUND;
     PVMATRESET  pPrev = NULL;
-    PVMATRESET  pCur = pVM->vm.s.pAtReset;
+    PVMATRESET  pCur = pVM->pUVM->vm.s.pAtReset;
     while (pCur)
     {
@@ -2113,5 +2174,5 @@
             &&  pCur->u.External.pfnCallback == pfnCallback)
         {
-            pCur = vmr3AtResetFree(pVM, pCur, pPrev);
+            pCur = vmr3AtResetFreeU(pVM->pUVM, pCur, pPrev);
             rc = VINF_SUCCESS;
         }
@@ -2186,5 +2247,5 @@
      * Call the at state change callbacks.
      */
-    for (PVMATSTATE pCur = pVM->vm.s.pAtState; pCur; pCur = pCur->pNext)
+    for (PVMATSTATE pCur = pVM->pUVM->vm.s.pAtState; pCur; pCur = pCur->pNext)
     {
         pCur->pfnAtState(pVM, enmStateNew, enmStateOld, pCur->pvUser);
@@ -2229,5 +2290,5 @@
      */
     PVMREQ pReq;
-    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtStateRegister, 3, pVM, pfnAtState, pvUser);
+    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtStateRegisterU, 3, pVM->pUVM, pfnAtState, pvUser);
     if (VBOX_FAILURE(rc))
         return rc;
@@ -2244,10 +2305,10 @@
  *
  * @returns VBox status code.
- * @param   pVM             VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pfnAtState      Pointer to callback.
  * @param   pvUser          User argument.
  * @thread  EMT
  */
-static DECLCALLBACK(int) vmR3AtStateRegister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser)
+static DECLCALLBACK(int) vmR3AtStateRegisterU(PUVM pUVM, PFNVMATSTATE pfnAtState, void *pvUser)
 {
     /*
@@ -2255,5 +2316,5 @@
      */
 
-    PVMATSTATE pNew = (PVMATSTATE)MMR3HeapAlloc(pVM, MM_TAG_VM, sizeof(*pNew));
+    PVMATSTATE pNew = (PVMATSTATE)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
     if (!pNew)
         return VERR_NO_MEMORY;
@@ -2265,6 +2326,6 @@
 
     /* insert */
-    *pVM->vm.s.ppAtStateNext = pNew;
-    pVM->vm.s.ppAtStateNext = &pNew->pNext;
+    *pUVM->vm.s.ppAtStateNext = pNew;
+    pUVM->vm.s.ppAtStateNext = &pNew->pNext;
 
     return VINF_SUCCESS;
@@ -2298,5 +2359,5 @@
      */
     PVMREQ pReq;
-    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtStateDeregister, 3, pVM, pfnAtState, pvUser);
+    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtStateDeregisterU, 3, pVM->pUVM, pfnAtState, pvUser);
     if (VBOX_FAILURE(rc))
         return rc;
@@ -2313,12 +2374,12 @@
  *
  * @returns VBox status code.
- * @param   pVM             VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pfnAtState      Pointer to callback.
  * @param   pvUser          User argument.
  * @thread  EMT
  */
-static DECLCALLBACK(int) vmR3AtStateDeregister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser)
-{
-    LogFlow(("vmR3AtStateDeregister: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
+static DECLCALLBACK(int) vmR3AtStateDeregisterU(PUVM pUVM, PFNVMATSTATE pfnAtState, void *pvUser)
+{
+    LogFlow(("vmR3AtStateDeregisterU: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
 
     /*
@@ -2326,5 +2387,5 @@
      */
     PVMATSTATE pPrev = NULL;
-    PVMATSTATE pCur = pVM->vm.s.pAtState;
+    PVMATSTATE pCur = pUVM->vm.s.pAtState;
     while (     pCur
            &&   pCur->pfnAtState == pfnAtState
@@ -2347,11 +2408,11 @@
         pPrev->pNext = pCur->pNext;
         if (!pCur->pNext)
-            pVM->vm.s.ppAtStateNext = &pPrev->pNext;
+            pUVM->vm.s.ppAtStateNext = &pPrev->pNext;
     }
     else
     {
-        pVM->vm.s.pAtState = pCur->pNext;
+        pUVM->vm.s.pAtState = pCur->pNext;
         if (!pCur->pNext)
-            pVM->vm.s.ppAtStateNext = &pVM->vm.s.pAtState;
+            pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
     }
 
@@ -2378,14 +2439,21 @@
 VMR3DECL(int)   VMR3AtErrorRegister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser)
 {
+    return VMR3AtErrorRegisterU(pVM->pUVM, pfnAtError, pvUser);
+}
+
+
+/**
+ * Registers a VM error callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The VM handle.
+ * @param   pfnAtError      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMR3DECL(int)   VMR3AtErrorRegisterU(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser)
+{
     LogFlow(("VMR3AtErrorRegister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
-
-    /*
-     * Validate input.
-     */
-    if (!pfnAtError)
-    {
-        AssertMsgFailed(("callback is required\n"));
-        return VERR_INVALID_PARAMETER;
-    }
+    AssertPtrReturn(pfnAtError, VERR_INVALID_PARAMETER);
 
     /*
@@ -2393,5 +2461,5 @@
      */
     PVMREQ pReq;
-    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtErrorRegister, 3, pVM, pfnAtError, pvUser);
+    int rc = VMR3ReqCallU(pUVM, &pReq, RT_INDEFINITE_WAIT, 0, (PFNRT)vmR3AtErrorRegisterU, 3, pUVM, pfnAtError, pvUser);
     if (VBOX_FAILURE(rc))
         return rc;
@@ -2408,10 +2476,10 @@
  *
  * @returns VBox status code.
- * @param   pVM             The VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pfnAtError      Pointer to callback.
  * @param   pvUser          User argument.
  * @thread  EMT
  */
-static DECLCALLBACK(int)    vmR3AtErrorRegister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser)
+static DECLCALLBACK(int) vmR3AtErrorRegisterU(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser)
 {
     /*
@@ -2419,5 +2487,5 @@
      */
 
-    PVMATERROR pNew = (PVMATERROR)MMR3HeapAlloc(pVM, MM_TAG_VM, sizeof(*pNew));
+    PVMATERROR pNew = (PVMATERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
     if (!pNew)
         return VERR_NO_MEMORY;
@@ -2429,6 +2497,6 @@
 
     /* insert */
-    *pVM->vm.s.ppAtErrorNext = pNew;
-    pVM->vm.s.ppAtErrorNext = &pNew->pNext;
+    *pUVM->vm.s.ppAtErrorNext = pNew;
+    pUVM->vm.s.ppAtErrorNext = &pNew->pNext;
 
     return VINF_SUCCESS;
@@ -2462,5 +2530,5 @@
      */
     PVMREQ pReq;
-    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtErrorDeregister, 3, pVM, pfnAtError, pvUser);
+    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtErrorDeregisterU, 3, pVM->pUVM, pfnAtError, pvUser);
     if (VBOX_FAILURE(rc))
         return rc;
@@ -2477,12 +2545,12 @@
  *
  * @returns VBox status code.
- * @param   pVM             The VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pfnAtError      Pointer to callback.
  * @param   pvUser          User argument.
  * @thread  EMT
  */
-static DECLCALLBACK(int)    vmR3AtErrorDeregister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser)
-{
-    LogFlow(("vmR3AtErrorDeregister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
+static DECLCALLBACK(int)    vmR3AtErrorDeregisterU(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser)
+{
+    LogFlow(("vmR3AtErrorDeregisterU: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
 
     /*
@@ -2490,5 +2558,5 @@
      */
     PVMATERROR pPrev = NULL;
-    PVMATERROR pCur = pVM->vm.s.pAtError;
+    PVMATERROR pCur = pUVM->vm.s.pAtError;
     while (     pCur
            &&   pCur->pfnAtError == pfnAtError
@@ -2511,11 +2579,11 @@
         pPrev->pNext = pCur->pNext;
         if (!pCur->pNext)
-            pVM->vm.s.ppAtErrorNext = &pPrev->pNext;
+            pUVM->vm.s.ppAtErrorNext = &pPrev->pNext;
     }
     else
     {
-        pVM->vm.s.pAtError = pCur->pNext;
+        pUVM->vm.s.pAtError = pCur->pNext;
         if (!pCur->pNext)
-            pVM->vm.s.ppAtErrorNext = &pVM->vm.s.pAtError;
+            pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
     }
 
@@ -2583,5 +2651,5 @@
      * Call the at error callbacks.
      */
-    for (PVMATERROR pCur = pVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
+    for (PVMATERROR pCur = pVM->pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
         vmR3SetErrorWorkerDoCall(pVM, pCur, rc, RT_SRC_POS_ARGS, "%s", pszMessage);
 }
@@ -2589,7 +2657,28 @@
 
 /**
+ * Creation time wrapper for vmR3SetErrorUV.
+ *
+ * @returns rc.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   rc              The VBox status code.
+ * @param   RT_SRC_POS_DECL The source position of this error.
+ * @param   pszFormat       Format string.
+ * @param   ...             The arguments.
+ * @thread  Any thread.
+ */
+static int vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    vmR3SetErrorUV(pUVM, rc, pszFile, iLine, pszFunction, pszFormat, &va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
  * Worker which calls everyone listening to the VM error messages.
  *
- * @param   pVM             The VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   rc              The VBox status code.
  * @param   RT_SRC_POS_DECL The source position of this error.
@@ -2598,5 +2687,5 @@
  * @thread  EMT
  */
-DECLCALLBACK(void) vmR3SetErrorV(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *pArgs)
+DECLCALLBACK(void) vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *pArgs)
 {
 #ifdef LOG_ENABLED
@@ -2615,14 +2704,15 @@
      * Make a copy of the message.
      */
-    vmSetErrorCopy(pVM, rc, RT_SRC_POS_ARGS, pszFormat, *pArgs);
+    if (pUVM->pVM)
+        vmSetErrorCopy(pUVM->pVM, rc, RT_SRC_POS_ARGS, pszFormat, *pArgs);
 
     /*
      * Call the at error callbacks.
      */
-    for (PVMATERROR pCur = pVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
+    for (PVMATERROR pCur = pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
     {
         va_list va2;
         va_copy(va2, *pArgs);
-        pCur->pfnAtError(pVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va2);
+        pCur->pfnAtError(pUVM->pVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va2);
         va_end(va2);
     }
@@ -2656,5 +2746,5 @@
      */
     PVMREQ pReq;
-    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtRuntimeErrorRegister, 3, pVM, pfnAtRuntimeError, pvUser);
+    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtRuntimeErrorRegisterU, 3, pVM->pUVM, pfnAtRuntimeError, pvUser);
     if (VBOX_FAILURE(rc))
         return rc;
@@ -2671,10 +2761,10 @@
  *
  * @returns VBox status code.
- * @param   pVM                 The VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pfnAtRuntimeError   Pointer to callback.
  * @param   pvUser              User argument.
  * @thread  EMT
  */
-static DECLCALLBACK(int)    vmR3AtRuntimeErrorRegister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
+static DECLCALLBACK(int)    vmR3AtRuntimeErrorRegisterU(PUVM pUVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
 {
     /*
@@ -2682,5 +2772,5 @@
      */
 
-    PVMATRUNTIMEERROR pNew = (PVMATRUNTIMEERROR)MMR3HeapAlloc(pVM, MM_TAG_VM, sizeof(*pNew));
+    PVMATRUNTIMEERROR pNew = (PVMATRUNTIMEERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
     if (!pNew)
         return VERR_NO_MEMORY;
@@ -2692,6 +2782,6 @@
 
     /* insert */
-    *pVM->vm.s.ppAtRuntimeErrorNext = pNew;
-    pVM->vm.s.ppAtRuntimeErrorNext = &pNew->pNext;
+    *pUVM->vm.s.ppAtRuntimeErrorNext = pNew;
+    pUVM->vm.s.ppAtRuntimeErrorNext = &pNew->pNext;
 
     return VINF_SUCCESS;
@@ -2725,5 +2815,5 @@
      */
     PVMREQ pReq;
-    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtRuntimeErrorDeregister, 3, pVM, pfnAtRuntimeError, pvUser);
+    int rc = VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3AtRuntimeErrorDeregisterU, 3, pVM->pUVM, pfnAtRuntimeError, pvUser);
     if (VBOX_FAILURE(rc))
         return rc;
@@ -2740,12 +2830,12 @@
  *
  * @returns VBox status code.
- * @param   pVM                 The VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   pfnAtRuntimeError   Pointer to callback.
  * @param   pvUser              User argument.
  * @thread  EMT
  */
-static DECLCALLBACK(int)    vmR3AtRuntimeErrorDeregister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
-{
-    LogFlow(("vmR3AtRuntimeErrorDeregister: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
+static DECLCALLBACK(int)    vmR3AtRuntimeErrorDeregisterU(PUVM pUVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
+{
+    LogFlow(("vmR3AtRuntimeErrorDeregisterU: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
 
     /*
@@ -2753,5 +2843,5 @@
      */
     PVMATRUNTIMEERROR pPrev = NULL;
-    PVMATRUNTIMEERROR pCur = pVM->vm.s.pAtRuntimeError;
+    PVMATRUNTIMEERROR pCur = pUVM->vm.s.pAtRuntimeError;
     while (     pCur
            &&   pCur->pfnAtRuntimeError == pfnAtRuntimeError
@@ -2774,11 +2864,11 @@
         pPrev->pNext = pCur->pNext;
         if (!pCur->pNext)
-            pVM->vm.s.ppAtRuntimeErrorNext = &pPrev->pNext;
+            pUVM->vm.s.ppAtRuntimeErrorNext = &pPrev->pNext;
     }
     else
     {
-        pVM->vm.s.pAtRuntimeError = pCur->pNext;
+        pUVM->vm.s.pAtRuntimeError = pCur->pNext;
         if (!pCur->pNext)
-            pVM->vm.s.ppAtRuntimeErrorNext = &pVM->vm.s.pAtRuntimeError;
+            pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
     }
 
@@ -2844,5 +2934,5 @@
      * Call the at runtime error callbacks.
      */
-    for (PVMATRUNTIMEERROR pCur = pVM->vm.s.pAtRuntimeError; pCur; pCur = pCur->pNext)
+    for (PVMATRUNTIMEERROR pCur = pVM->pUVM->vm.s.pAtRuntimeError; pCur; pCur = pCur->pNext)
         vmR3SetRuntimeErrorWorkerDoCall(pVM, pCur, fFatal, pszErrorID, "%s", pszMessage);
 }
@@ -2871,5 +2961,5 @@
      * Call the at error callbacks.
      */
-    for (PVMATRUNTIMEERROR pCur = pVM->vm.s.pAtRuntimeError; pCur; pCur = pCur->pNext)
+    for (PVMATRUNTIMEERROR pCur = pVM->pUVM->vm.s.pAtRuntimeError; pCur; pCur = pCur->pNext)
     {
         va_list va2;
Index: /trunk/src/VBox/VMM/VMEmt.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMEmt.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/VMEmt.cpp	(revision 6796)
@@ -28,4 +28,5 @@
 #include "VMInternal.h"
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 
 #include <VBox/err.h>
@@ -46,31 +47,183 @@
  * @returns Thread exit code.
  * @param   ThreadSelf  The handle to the executing thread.
- * @param   pvArgs      Pointer to a VMEMULATIONTHREADARGS structure.
+ * @param   pvArgs      Pointer to the user mode VM structure (UVM).
  */
 DECLCALLBACK(int) vmR3EmulationThread(RTTHREAD ThreadSelf, void *pvArgs)
 {
-    PVMEMULATIONTHREADARGS pArgs = (PVMEMULATIONTHREADARGS)pvArgs;
-    AssertReleaseMsg(pArgs && pArgs->pVM, ("Invalid arguments to the emulation thread!\n"));
+    PUVM pUVM = (PUVM)pvArgs;
+    AssertReleaseMsg(VALID_PTR(pUVM) && pUVM->u32Magic == UVM_MAGIC,
+                     ("Invalid arguments to the emulation thread!\n"));
 
     /*
      * Init the native thread member.
      */
-    PVM pVM = pArgs->pVM;
-    pVM->NativeThreadEMT = RTThreadGetNative(ThreadSelf);
+    pUVM->vm.s.NativeThreadEMT = RTThreadGetNative(ThreadSelf);
 
     /*
      * The request loop.
      */
+    int     rc = VINF_SUCCESS;
+    VMSTATE enmBefore = VMSTATE_CREATING;
+    Log(("vmR3EmulationThread: Emulation thread starting the days work... Thread=%#x pUVM=%p\n", ThreadSelf, pUVM));
+    for (;;)
+    {
+        /* Requested to exit the EMT thread out of sync? (currently only VMR3WaitForResume) */
+        if (setjmp(pUVM->vm.s.emtJumpEnv) != 0)
+        {
+            rc = VINF_SUCCESS;
+            break;
+        }
+
+        /*
+         * During early init there is no pVM, so make a special path
+         * for that to keep things clearly separate.
+         */
+        if (!pUVM->pVM)
+        {
+            /*
+             * Check for termination first.
+             */
+            if (pUVM->vm.s.fTerminateEMT)
+            {
+                rc = VINF_EM_TERMINATE;
+                break;
+            }
+            if (pUVM->vm.s.pReqs)
+            {
+                /*
+                 * Service execute in EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM);
+                Log(("vmR3EmulationThread: Req rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING));
+            }
+            else
+            {
+                /*
+                 * Nothing important is pending, so wait for something.
+                 */
+                rc = VMR3WaitU(pUVM);
+                if (VBOX_FAILURE(rc))
+                    break;
+            }
+        }
+        else
+        {
+
+            /*
+             * Pending requests which needs servicing?
+             *
+             * We check for state changes in addition to status codes when
+             * servicing requests. (Look after the ifs.)
+             */
+            PVM pVM = pUVM->pVM;
+            enmBefore = pVM->enmVMState;
+            if (    VM_FF_ISSET(pVM, VM_FF_TERMINATE)
+                ||  pUVM->vm.s.fTerminateEMT)
+            {
+                rc = VINF_EM_TERMINATE;
+                break;
+            }
+            if (pUVM->vm.s.pReqs)
+            {
+                /*
+                 * Service execute in EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM);
+                Log(("vmR3EmulationThread: Req rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pVM->enmVMState));
+            }
+            else if (VM_FF_ISSET(pVM, VM_FF_DBGF))
+            {
+                /*
+                 * Service the debugger request.
+                 */
+                rc = DBGFR3VMMForcedAction(pVM);
+                Log(("vmR3EmulationThread: Dbg rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pVM->enmVMState));
+            }
+            else if (VM_FF_ISSET(pVM, VM_FF_RESET))
+            {
+                /*
+                 * Service a delayed reset request.
+                 */
+                rc = VMR3Reset(pVM);
+                VM_FF_CLEAR(pVM, VM_FF_RESET);
+                Log(("vmR3EmulationThread: Reset rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pVM->enmVMState));
+            }
+            else
+            {
+                /*
+                 * Nothing important is pending, so wait for something.
+                 */
+                rc = VMR3WaitU(pUVM);
+                if (VBOX_FAILURE(rc))
+                    break;
+            }
+
+            /*
+             * Check for termination requests, these have extremely high priority.
+             */
+            if (    rc == VINF_EM_TERMINATE
+                ||  VM_FF_ISSET(pVM, VM_FF_TERMINATE)
+                ||  pUVM->vm.s.fTerminateEMT)
+                break;
+
+            /*
+             * Some requests (both VMR3Req* and the DBGF) can potentially
+             * resume or start the VM, in that case we'll get a change in
+             * VM status indicating that we're now running.
+             */
+            if (    VBOX_SUCCESS(rc)
+                &&  enmBefore != pVM->enmVMState
+                &&  (pVM->enmVMState == VMSTATE_RUNNING))
+            {
+                rc = EMR3ExecuteVM(pVM);
+                Log(("vmR3EmulationThread: EMR3ExecuteVM() -> rc=%Vrc, enmVMState=%d\n", rc, pVM->enmVMState));
+                if (EMGetState(pVM) == EMSTATE_GURU_MEDITATION)
+                    vmR3SetState(pVM, VMSTATE_GURU_MEDITATION);
+            }
+        }
+    } /* forever */
+
+
+    /*
+     * Exiting.
+     */
+    Log(("vmR3EmulationThread: Terminating emulation thread! Thread=%#x pUVM=%p rc=%Vrc enmBefore=%d enmVMState=%d\n",
+         ThreadSelf, pUVM, rc, enmBefore, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_TERMINATED));
+    if (pUVM->vm.s.fEMTDoesTheCleanup)
+    {
+        Log(("vmR3EmulationThread: executing delayed Destroy\n"));
+        Assert(pUVM->pVM);
+        vmR3Destroy(pUVM->pVM);
+        vmR3DestroyFinalBitFromEMT(pUVM);
+    }
+    else
+    {
+        vmR3DestroyFinalBitFromEMT(pUVM);
+
+        /* we don't reset ThreadEMT here because it's used in waiting. */
+        pUVM->vm.s.NativeThreadEMT = NIL_RTNATIVETHREAD;
+    }
+    Log(("vmR3EmulationThread: EMT is terminated.\n"));
+    return rc;
+}
+
+
+/**
+ * Wait for VM to be resumed. Handle events like vmR3EmulationThread does.
+ * In case the VM is stopped, clean up and long jump to the main EMT loop.
+ *
+ * @returns VINF_SUCCESS or doesn't return
+ * @param   pVM             VM handle.
+ */
+VMR3DECL(int) VMR3WaitForResume(PVM pVM)
+{
+    /*
+     * The request loop.
+     */
+    PUVM    pUVM = pVM->pUVM;
     VMSTATE enmBefore;
     int     rc;
-    Log(("vmR3EmulationThread: Emulation thread starting the days work... Thread=%#x pVM=%p\n", ThreadSelf, pVM));
     for (;;)
     {
-        /* Requested to exit the EMT thread out of sync? (currently only VMR3WaitForResume) */
-        if (setjmp(pVM->vm.s.emtJumpEnv) != 0)
-        {
-            rc = VINF_SUCCESS;
-            break;
-        }
 
         /*
@@ -81,125 +234,16 @@
          */
         enmBefore = pVM->enmVMState;
-        if (VM_FF_ISSET(pVM, VM_FF_TERMINATE))
+        if (    VM_FF_ISSET(pVM, VM_FF_TERMINATE)
+            ||  pUVM->vm.s.fTerminateEMT)
         {
             rc = VINF_EM_TERMINATE;
             break;
         }
-        if (pVM->vm.s.pReqs)
+        else if (pUVM->vm.s.pReqs)
         {
             /*
              * Service execute in EMT request.
              */
-            rc = VMR3ReqProcess(pVM);
-            Log(("vmR3EmulationThread: Req rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pVM->enmVMState));
-        }
-        else if (VM_FF_ISSET(pVM, VM_FF_DBGF))
-        {
-            /*
-             * Service the debugger request.
-             */
-            rc = DBGFR3VMMForcedAction(pVM);
-            Log(("vmR3EmulationThread: Dbg rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pVM->enmVMState));
-        }
-        else if (VM_FF_ISSET(pVM, VM_FF_RESET))
-        {
-            /*
-             * Service a delayed reset request.
-             */
-            rc = VMR3Reset(pVM);
-            VM_FF_CLEAR(pVM, VM_FF_RESET);
-            Log(("vmR3EmulationThread: Reset rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pVM->enmVMState));
-        }
-        else
-        {
-            /*
-             * Nothing important is pending, so wait for something.
-             */
-            rc = VMR3Wait(pVM);
-            if (VBOX_FAILURE(rc))
-                break;
-        }
-
-        /*
-         * Check for termination requests, these are extremely high priority.
-         */
-        if (    rc == VINF_EM_TERMINATE
-            ||  VM_FF_ISSET(pVM, VM_FF_TERMINATE))
-            break;
-
-        /*
-         * Some requests (both VMR3Req* and the DBGF) can potentially
-         * resume or start the VM, in that case we'll get a change in
-         * VM status indicating that we're now running.
-         */
-        if (    VBOX_SUCCESS(rc)
-            &&  enmBefore != pVM->enmVMState
-            &&  (pVM->enmVMState == VMSTATE_RUNNING))
-        {
-            rc = EMR3ExecuteVM(pVM);
-            Log(("vmR3EmulationThread: EMR3ExecuteVM() -> rc=%Vrc, enmVMState=%d\n", rc, pVM->enmVMState));
-            if (EMGetState(pVM) == EMSTATE_GURU_MEDITATION)
-                vmR3SetState(pVM, VMSTATE_GURU_MEDITATION);
-        }
-
-    } /* forever */
-
-
-    /*
-     * Exiting.
-     */
-    Log(("vmR3EmulationThread: Terminating emulation thread! Thread=%#x pVM=%p rc=%Vrc enmBefore=%d enmVMState=%d\n",
-         ThreadSelf, pVM, rc, enmBefore, pVM->enmVMState));
-    if (pVM->vm.s.fEMTDoesTheCleanup)
-    {
-        Log(("vmR3EmulationThread: executing delayed Destroy\n"));
-        vmR3Destroy(pVM);
-        vmR3DestroyFinalBit(pVM);
-        Log(("vmR3EmulationThread: EMT is terminated.\n"));
-    }
-    else
-    {
-        /* we don't reset ThreadEMT here because it's used in waiting. */
-        pVM->NativeThreadEMT = NIL_RTNATIVETHREAD;
-    }
-    return rc;
-}
-
-
-/**
- * Wait for VM to be resumed. Handle events like vmR3EmulationThread does.
- * In case the VM is stopped, clean up and long jump to the main EMT loop.
- *
- * @returns VINF_SUCCESS or doesn't return
- * @param   pVM             VM handle.
- */
-VMR3DECL(int) VMR3WaitForResume(PVM pVM)
-{
-    /*
-     * The request loop.
-     */
-    VMSTATE enmBefore;
-    int     rc;
-    for (;;)
-    {
-
-        /*
-         * Pending requests which needs servicing?
-         *
-         * We check for state changes in addition to status codes when
-         * servicing requests. (Look after the ifs.)
-         */
-        enmBefore = pVM->enmVMState;
-        if (VM_FF_ISSET(pVM, VM_FF_TERMINATE))
-        {
-            rc = VINF_EM_TERMINATE;
-            break;
-        }
-        else if (pVM->vm.s.pReqs)
-        {
-            /*
-             * Service execute in EMT request.
-             */
-            rc = VMR3ReqProcess(pVM);
+            rc = VMR3ReqProcessU(pUVM);
             Log(("vmR3EmulationThread: Req rc=%Vrc, VM state %d -> %d\n", rc, enmBefore, pVM->enmVMState));
         }
@@ -226,5 +270,5 @@
              * Nothing important is pending, so wait for something.
              */
-            rc = VMR3Wait(pVM);
+            rc = VMR3WaitU(pUVM);
             if (VBOX_FAILURE(rc))
                 break;
@@ -235,5 +279,6 @@
          */
         if (    rc == VINF_EM_TERMINATE
-            ||  VM_FF_ISSET(pVM, VM_FF_TERMINATE))
+            ||  VM_FF_ISSET(pVM, VM_FF_TERMINATE)
+            ||  pUVM->vm.s.fTerminateEMT)
             break;
 
@@ -245,5 +290,5 @@
         if (    VBOX_SUCCESS(rc)
             &&  enmBefore != pVM->enmVMState
-            &&  (pVM->enmVMState == VMSTATE_RUNNING))
+            &&  pVM->enmVMState == VMSTATE_RUNNING)
         {
             /* Only valid exit reason. */
@@ -254,5 +299,5 @@
 
     /* Return to the main loop in vmR3EmulationThread, which will clean up for us. */
-    longjmp(pVM->vm.s.emtJumpEnv, 1);
+    longjmp(pUVM->vm.s.emtJumpEnv, 1);
 }
 
@@ -268,10 +313,11 @@
     switch (enmMethod)
     {
-        case VMHALTMETHOD_DEFAULT:  return "default";
-        case VMHALTMETHOD_OLD:      return "old";
-        case VMHALTMETHOD_1:        return "method1";
-        //case VMHALTMETHOD_2:        return "method2";
-        case VMHALTMETHOD_GLOBAL_1: return "global1";
-        default:                    return "unknown";
+        case VMHALTMETHOD_BOOTSTRAP:    return "bootstrap";
+        case VMHALTMETHOD_DEFAULT:      return "default";
+        case VMHALTMETHOD_OLD:          return "old";
+        case VMHALTMETHOD_1:            return "method1";
+        //case VMHALTMETHOD_2:            return "method2";
+        case VMHALTMETHOD_GLOBAL_1:     return "global1";
+        default:                        return "unknown";
     }
 }
@@ -280,12 +326,15 @@
 /**
  * The old halt loop.
- */
-static DECLCALLBACK(int) vmR3HaltOldDoHalt(PVM pVM, const uint32_t fMask, uint64_t /* u64Now*/)
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltOldDoHalt(PUVM pUVM, const uint32_t fMask, uint64_t /* u64Now*/)
 {
     /*
      * Halt loop.
      */
+    PVM pVM = pUVM->pVM;
     int rc = VINF_SUCCESS;
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 1);
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, true);
     //unsigned cLoops = 0;
     for (;;)
@@ -296,10 +345,10 @@
          * addition to perhaps set an FF.
          */
-        STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltPoll, a);
+        STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltPoll, a);
         PDMR3Poll(pVM);
-        STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltPoll, a);
-        STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltTimers, b);
+        STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltPoll, a);
+        STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltTimers, b);
         TMR3TimerQueuesDo(pVM);
-        STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltTimers, b);
+        STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltTimers, b);
         if (VM_FF_ISPENDING(pVM, fMask))
             break;
@@ -324,21 +373,21 @@
             {
                 //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d yield", u64NanoTS, cLoops++);
-                STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltYield, a);
+                STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltYield, a);
                 RTThreadYield(); /* this is the best we can do here */
-                STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltYield, a);
+                STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltYield, a);
             }
             else if (u64NanoTS < 2000000)
             {
                 //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep 1ms", u64NanoTS, cLoops++);
-                STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltBlock, a);
-                rc = RTSemEventWait(pVM->vm.s.EventSemWait, 1);
-                STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltBlock, a);
+                STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltBlock, a);
+                rc = RTSemEventWait(pUVM->vm.s.EventSemWait, 1);
+                STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltBlock, a);
             }
             else
             {
                 //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep %dms", u64NanoTS, cLoops++, (uint32_t)RT_MIN((u64NanoTS - 500000) / 1000000, 15));
-                STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltBlock, a);
-                rc = RTSemEventWait(pVM->vm.s.EventSemWait, RT_MIN((u64NanoTS - 1000000) / 1000000, 15));
-                STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltBlock, a);
+                STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltBlock, a);
+                rc = RTSemEventWait(pUVM->vm.s.EventSemWait, RT_MIN((u64NanoTS - 1000000) / 1000000, 15));
+                STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltBlock, a);
             }
             //uint64_t u64Slept = RTTimeNanoTS() - u64Start;
@@ -351,4 +400,5 @@
             AssertRC(rc != VERR_INTERRUPTED);
             AssertMsgFailed(("RTSemEventWait->%Vrc\n", rc));
+            ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
             VM_FF_SET(pVM, VM_FF_TERMINATE);
             rc = VERR_INTERNAL_ERROR;
@@ -357,4 +407,5 @@
     }
 
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, false);
     return rc;
 }
@@ -367,5 +418,5 @@
  * @param   pVM     The VM handle.
  */
-static int vmR3HaltMethod12ReadConfig(PVM pVM)
+static int vmR3HaltMethod12ReadConfigU(PUVM pUVM)
 {
     /*
@@ -373,15 +424,15 @@
      */
 #if 1 /* DEBUGGING STUFF - REMOVE LATER */
-    pVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
-    pVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   2*1000000;
-    pVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg =  75*1000000;
-    pVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  30*1000000;
-    pVM->vm.s.Halt.Method12.u32StopSpinningCfg     =  20*1000000;
+    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
+    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   2*1000000;
+    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg =  75*1000000;
+    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  30*1000000;
+    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =  20*1000000;
 #else
-    pVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
-    pVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   5*1000000;
-    pVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = 200*1000000;
-    pVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  20*1000000;
-    pVM->vm.s.Halt.Method12.u32StopSpinningCfg     =   2*1000000;
+    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
+    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   5*1000000;
+    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = 200*1000000;
+    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  20*1000000;
+    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =   2*1000000;
 #endif
 
@@ -392,24 +443,24 @@
      * here right now. sorry.
      */
-    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/VMM/HaltedMethod1");
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pUVM->pVM), "/VMM/HaltedMethod1");
     if (pCfg)
     {
         uint32_t u32;
         if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "LagBlockIntervalDivisor", &u32)))
-            pVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = u32;
+            pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = u32;
         if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MinBlockInterval", &u32)))
-            pVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg = u32;
+            pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg = u32;
         if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MaxBlockInterval", &u32)))
-            pVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = u32;
+            pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = u32;
         if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StartSpinning", &u32)))
-            pVM->vm.s.Halt.Method12.u32StartSpinningCfg = u32;
+            pUVM->vm.s.Halt.Method12.u32StartSpinningCfg = u32;
         if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StopSpinning", &u32)))
-            pVM->vm.s.Halt.Method12.u32StopSpinningCfg = u32;
+            pUVM->vm.s.Halt.Method12.u32StopSpinningCfg = u32;
         LogRel(("HaltedMethod1 config: %d/%d/%d/%d/%d\n",
-                pVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
-                pVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
-                pVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg,
-                pVM->vm.s.Halt.Method12.u32StartSpinningCfg,
-                pVM->vm.s.Halt.Method12.u32StopSpinningCfg));
+                pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
+                pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
+                pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg,
+                pUVM->vm.s.Halt.Method12.u32StartSpinningCfg,
+                pUVM->vm.s.Halt.Method12.u32StopSpinningCfg));
     }
 
@@ -422,9 +473,9 @@
  *
  * @return VBox status code.
- * @param   pVM     The VM handle.
- */
-static DECLCALLBACK(int) vmR3HaltMethod1Init(PVM pVM)
-{
-    return vmR3HaltMethod12ReadConfig(pVM);
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltMethod1Init(PUVM pUVM)
+{
+    return vmR3HaltMethod12ReadConfigU(pUVM);
 }
 
@@ -435,6 +486,8 @@
  * the lag has been eliminated.
  */
-static DECLCALLBACK(int) vmR3HaltMethod1Halt(PVM pVM, const uint32_t fMask, uint64_t u64Now)
-{
+static DECLCALLBACK(int) vmR3HaltMethod1Halt(PUVM pUVM, const uint32_t fMask, uint64_t u64Now)
+{
+    PVM pVM = pUVM->pVM;
+
     /*
      * To simplify things, we decide up-front whether we should switch to spinning or
@@ -448,32 +501,32 @@
     if (u32CatchUpPct /* non-zero if catching up */)
     {
-        if (pVM->vm.s.Halt.Method12.u64StartSpinTS)
-        {
-            fSpinning = TMVirtualSyncGetLag(pVM) >= pVM->vm.s.Halt.Method12.u32StopSpinningCfg;
+        if (pUVM->vm.s.Halt.Method12.u64StartSpinTS)
+        {
+            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StopSpinningCfg;
             if (fSpinning)
             {
                 uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
-                fBlockOnce = u64Now - pVM->vm.s.Halt.Method12.u64LastBlockTS
-                           > RT_MAX(pVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
-                                    RT_MIN(u64Lag / pVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
-                                           pVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg));
+                fBlockOnce = u64Now - pUVM->vm.s.Halt.Method12.u64LastBlockTS
+                           > RT_MAX(pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
+                                    RT_MIN(u64Lag / pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
+                                           pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg));
             }
             else
             {
-                //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pVM->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
-                pVM->vm.s.Halt.Method12.u64StartSpinTS = 0;
+                //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVM->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
+                pUVM->vm.s.Halt.Method12.u64StartSpinTS = 0;
             }
         }
         else
         {
-            fSpinning = TMVirtualSyncGetLag(pVM) >= pVM->vm.s.Halt.Method12.u32StartSpinningCfg;
+            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StartSpinningCfg;
             if (fSpinning)
-                pVM->vm.s.Halt.Method12.u64StartSpinTS = u64Now;
-        }
-    }
-    else if (pVM->vm.s.Halt.Method12.u64StartSpinTS)
-    {
-        //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pVM->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
-        pVM->vm.s.Halt.Method12.u64StartSpinTS = 0;
+                pUVM->vm.s.Halt.Method12.u64StartSpinTS = u64Now;
+        }
+    }
+    else if (pUVM->vm.s.Halt.Method12.u64StartSpinTS)
+    {
+        //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVM->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
+        pUVM->vm.s.Halt.Method12.u64StartSpinTS = 0;
     }
 
@@ -482,5 +535,5 @@
      */
     int rc = VINF_SUCCESS;
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 1);
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, true);
     unsigned cLoops = 0;
     for (;; cLoops++)
@@ -489,10 +542,10 @@
          * Work the timers and check if we can exit.
          */
-        STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltPoll, a);
+        STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltPoll, a);
         PDMR3Poll(pVM);
-        STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltPoll, a);
-        STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltTimers, b);
+        STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltPoll, a);
+        STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltTimers, b);
         TMR3TimerQueuesDo(pVM);
-        STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltTimers, b);
+        STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltTimers, b);
         if (VM_FF_ISPENDING(pVM, fMask))
             break;
@@ -516,16 +569,16 @@
 #endif
         {
-            const uint64_t Start = pVM->vm.s.Halt.Method12.u64LastBlockTS = RTTimeNanoTS();
+            const uint64_t Start = pUVM->vm.s.Halt.Method12.u64LastBlockTS = RTTimeNanoTS();
             VMMR3YieldStop(pVM);
 
             uint32_t cMilliSecs = RT_MIN(u64NanoTS / 1000000, 15);
-            if (cMilliSecs <= pVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg)
+            if (cMilliSecs <= pUVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg)
                 cMilliSecs = 1;
             else
-                cMilliSecs -= pVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg;
+                cMilliSecs -= pUVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg;
             //RTLogRelPrintf("u64NanoTS=%RI64 cLoops=%3d sleep %02dms (%7RU64) ", u64NanoTS, cLoops, cMilliSecs, u64NanoTS);
-            STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltBlock, a);
-            rc = RTSemEventWait(pVM->vm.s.EventSemWait, cMilliSecs);
-            STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltBlock, a);
+            STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltBlock, a);
+            rc = RTSemEventWait(pUVM->vm.s.EventSemWait, cMilliSecs);
+            STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltBlock, a);
             if (rc == VERR_TIMEOUT)
                 rc = VINF_SUCCESS;
@@ -534,4 +587,5 @@
                 AssertRC(rc != VERR_INTERRUPTED);
                 AssertMsgFailed(("RTSemEventWait->%Vrc\n", rc));
+                ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
                 VM_FF_SET(pVM, VM_FF_TERMINATE);
                 rc = VERR_INTERNAL_ERROR;
@@ -544,15 +598,15 @@
              */
             const uint64_t Elapsed = RTTimeNanoTS() - Start;
-            pVM->vm.s.Halt.Method12.cNSBlocked += Elapsed;
+            pUVM->vm.s.Halt.Method12.cNSBlocked += Elapsed;
             if (Elapsed > u64NanoTS)
-                pVM->vm.s.Halt.Method12.cNSBlockedTooLong += Elapsed - u64NanoTS;
-            pVM->vm.s.Halt.Method12.cBlocks++;
-            if (!(pVM->vm.s.Halt.Method12.cBlocks & 0xf))
-            {
-                pVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg = pVM->vm.s.Halt.Method12.cNSBlockedTooLong / pVM->vm.s.Halt.Method12.cBlocks;
-                if (!(pVM->vm.s.Halt.Method12.cBlocks & 0x3f))
+                pUVM->vm.s.Halt.Method12.cNSBlockedTooLong += Elapsed - u64NanoTS;
+            pUVM->vm.s.Halt.Method12.cBlocks++;
+            if (!(pUVM->vm.s.Halt.Method12.cBlocks & 0xf))
+            {
+                pUVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg = pUVM->vm.s.Halt.Method12.cNSBlockedTooLong / pUVM->vm.s.Halt.Method12.cBlocks;
+                if (!(pUVM->vm.s.Halt.Method12.cBlocks & 0x3f))
                 {
-                    pVM->vm.s.Halt.Method12.cNSBlockedTooLong = pVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg * 0x40;
-                    pVM->vm.s.Halt.Method12.cBlocks = 0x40;
+                    pUVM->vm.s.Halt.Method12.cNSBlockedTooLong = pUVM->vm.s.Halt.Method12.cNSBlockedTooLongAvg * 0x40;
+                    pUVM->vm.s.Halt.Method12.cBlocks = 0x40;
                 }
             }
@@ -569,4 +623,5 @@
     //if (fSpinning) RTLogRelPrintf("spun for %RU64 ns %u loops; lag=%RU64 pct=%d\n", RTTimeNanoTS() - u64Now, cLoops, TMVirtualSyncGetLag(pVM), u32CatchUpPct);
 
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, false);
     return rc;
 }
@@ -577,7 +632,7 @@
  *
  * @return VBox status code.
- * @param   pVM     The VM handle.
- */
-static DECLCALLBACK(int) vmR3HaltGlobal1Init(PVM pVM)
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Init(PUVM pUVM)
 {
     return VINF_SUCCESS;
@@ -589,11 +644,13 @@
  * try take care of the global scheduling of EMT threads.
  */
-static DECLCALLBACK(int) vmR3HaltGlobal1Halt(PVM pVM, const uint32_t fMask, uint64_t u64Now)
-{
+static DECLCALLBACK(int) vmR3HaltGlobal1Halt(PUVM pUVM, const uint32_t fMask, uint64_t u64Now)
+{
+    PVM pVM = pUVM->pVM;
+
     /*
      * Halt loop.
      */
     int rc = VINF_SUCCESS;
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 1);
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, true);
     unsigned cLoops = 0;
     for (;; cLoops++)
@@ -602,10 +659,10 @@
          * Work the timers and check if we can exit.
          */
-        STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltPoll, a);
+        STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltPoll, a);
         PDMR3Poll(pVM);
-        STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltPoll, a);
-        STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltTimers, b);
+        STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltPoll, a);
+        STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltTimers, b);
         TMR3TimerQueuesDo(pVM);
-        STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltTimers, b);
+        STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltTimers, b);
         if (VM_FF_ISPENDING(pVM, fMask))
             break;
@@ -625,12 +682,11 @@
         {
             VMMR3YieldStop(pVM);
-            ASMAtomicXchgU32(&pVM->vm.s.fWait, 1);
             if (VM_FF_ISPENDING(pVM, fMask))
                 break;
 
             //RTLogRelPrintf("u64NanoTS=%RI64 cLoops=%3d sleep %02dms (%7RU64) ", u64NanoTS, cLoops, cMilliSecs, u64NanoTS);
-            STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltBlock, c);
+            STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltBlock, c);
             rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_GVMM_SCHED_HALT, u64GipTime, NULL);
-            STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltBlock, c);
+            STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltBlock, c);
             if (rc == VERR_INTERRUPTED)
                 rc = VINF_SUCCESS;
@@ -638,4 +694,5 @@
             {
                 AssertMsgFailed(("VMMR0_DO_GVMM_SCHED_HALT->%Vrc\n", rc));
+                ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
                 VM_FF_SET(pVM, VM_FF_TERMINATE);
                 rc = VERR_INTERNAL_ERROR;
@@ -649,11 +706,12 @@
         else if (!(cLoops & 0x1fff))
         {
-            STAM_REL_PROFILE_START(&pVM->vm.s.StatHaltYield, d);
+            STAM_REL_PROFILE_START(&pUVM->vm.s.StatHaltYield, d);
             rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_GVMM_SCHED_POLL, false /* don't yield */, NULL);
-            STAM_REL_PROFILE_STOP(&pVM->vm.s.StatHaltYield, d);
+            STAM_REL_PROFILE_STOP(&pUVM->vm.s.StatHaltYield, d);
         }
     }
     //if (fSpinning) RTLogRelPrintf("spun for %RU64 ns %u loops; lag=%RU64 pct=%d\n", RTTimeNanoTS() - u64Now, cLoops, TMVirtualSyncGetLag(pVM), u32CatchUpPct);
 
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, false);
     return rc;
 }
@@ -664,10 +722,12 @@
  *
  * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-static DECLCALLBACK(int) vmR3HaltGlobal1Wait(PVM pVM)
-{
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Wait(PUVM pUVM)
+{
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, true);
+
+    PVM pVM = pUVM->pVM;
     int rc = VINF_SUCCESS;
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 1);
     for (;;)
     {
@@ -688,4 +748,5 @@
         {
             AssertMsgFailed(("RTSemEventWait->%Vrc\n", rc));
+            ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
             VM_FF_SET(pVM, VM_FF_TERMINATE);
             rc = VERR_INTERNAL_ERROR;
@@ -694,5 +755,6 @@
 
     }
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 0);
+
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, false);
     return rc;
 }
@@ -702,29 +764,30 @@
  * The global 1 halt method - VMR3NotifyFF() worker.
  *
- * @param   pVM             The VM handle.
- * @param   fNotifiedREM    Se VMR3NotifyFF().
- */
-static DECLCALLBACK(void) vmR3HaltGlobal1NotifyFF(PVM pVM, bool fNotifiedREM)
-{
-    if (pVM->vm.s.fWait)
-    {
-        int rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_GVMM_SCHED_WAKE_UP, 0, NULL);
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   fNotifiedREM    See VMR3NotifyFF().
+ */
+static DECLCALLBACK(void) vmR3HaltGlobal1NotifyFF(PUVM pUVM, bool fNotifiedREM)
+{
+    if (pUVM->vm.s.fWait)
+    {
+        int rc = SUPCallVMMR0Ex(pUVM->pVM->pVMR0, VMMR0_DO_GVMM_SCHED_WAKE_UP, 0, NULL);
         AssertRC(rc);
     }
     else if (!fNotifiedREM)
-        REMR3NotifyFF(pVM);
-}
-
-
-/**
- * Default VMR3Wait() worker.
+        REMR3NotifyFF(pUVM->pVM);
+}
+
+
+/**
+ * Bootstrap VMR3Wait() worker.
  *
  * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-static DECLCALLBACK(int) vmR3DefaultWait(PVM pVM)
-{
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3BootstrapWait(PUVM pUVM)
+{
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, true);
+
     int rc = VINF_SUCCESS;
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 1);
     for (;;)
     {
@@ -732,5 +795,8 @@
          * Check Relevant FFs.
          */
-        if (VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK))
+        if (pUVM->vm.s.pReqs)
+            break;
+        if (    pUVM->pVM
+            &&  VM_FF_ISPENDING(pUVM->pVM, VM_FF_EXTERNAL_SUSPENDED_MASK))
             break;
 
@@ -739,5 +805,5 @@
          * anything needs our attention.
          */
-        rc = RTSemEventWait(pVM->vm.s.EventSemWait, 1000);
+        rc = RTSemEventWait(pUVM->vm.s.EventSemWait, 1000);
         if (rc == VERR_TIMEOUT)
             rc = VINF_SUCCESS;
@@ -745,4 +811,66 @@
         {
             AssertMsgFailed(("RTSemEventWait->%Vrc\n", rc));
+            ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+            if (pUVM->pVM)
+                VM_FF_SET(pUVM->pVM, VM_FF_TERMINATE);
+            rc = VERR_INTERNAL_ERROR;
+            break;
+        }
+
+    }
+
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Bootstrap VMR3NotifyFF() worker.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   fNotifiedREM    See VMR3NotifyFF().
+ */
+static DECLCALLBACK(void) vmR3BootstrapNotifyFF(PUVM pUVM, bool fNotifiedREM)
+{
+    if (pUVM->vm.s.fWait)
+    {
+        int rc = RTSemEventSignal(pUVM->vm.s.EventSemWait);
+        AssertRC(rc);
+    }
+}
+
+
+
+/**
+ * Default VMR3Wait() worker.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3DefaultWait(PUVM pUVM)
+{
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, true);
+
+    PVM pVM = pUVM->pVM;
+    int rc = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Check Relevant FFs.
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK))
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        rc = RTSemEventWait(pUVM->vm.s.EventSemWait, 1000);
+        if (rc == VERR_TIMEOUT)
+            rc = VINF_SUCCESS;
+        else if (VBOX_FAILURE(rc))
+        {
+            AssertMsgFailed(("RTSemEventWait->%Vrc\n", rc));
+            ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
             VM_FF_SET(pVM, VM_FF_TERMINATE);
             rc = VERR_INTERNAL_ERROR;
@@ -751,5 +879,6 @@
 
     }
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 0);
+
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fWait, false);
     return rc;
 }
@@ -759,16 +888,16 @@
  * Default VMR3NotifyFF() worker.
  *
- * @param   pVM             The VM handle.
- * @param   fNotifiedREM    Se VMR3NotifyFF().
- */
-static DECLCALLBACK(void) vmR3DefaultNotifyFF(PVM pVM, bool fNotifiedREM)
-{
-    if (pVM->vm.s.fWait)
-    {
-        int rc = RTSemEventSignal(pVM->vm.s.EventSemWait);
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   fNotifiedREM    See VMR3NotifyFF().
+ */
+static DECLCALLBACK(void) vmR3DefaultNotifyFF(PUVM pUVM, bool fNotifiedREM)
+{
+    if (pUVM->vm.s.fWait)
+    {
+        int rc = RTSemEventSignal(pUVM->vm.s.EventSemWait);
         AssertRC(rc);
     }
     else if (!fNotifiedREM)
-        REMR3NotifyFF(pVM);
+        REMR3NotifyFF(pUVM->pVM);
 }
 
@@ -783,15 +912,16 @@
     VMHALTMETHOD enmHaltMethod;
     /** The init function for loading config and initialize variables. */
-    DECLR3CALLBACKMEMBER(int,  pfnInit,(PVM pVM));
+    DECLR3CALLBACKMEMBER(int,  pfnInit,(PUVM pUVM));
     /** The term function. */
-    DECLR3CALLBACKMEMBER(void, pfnTerm,(PVM pVM));
+    DECLR3CALLBACKMEMBER(void, pfnTerm,(PUVM pUVM));
     /** The halt function. */
-    DECLR3CALLBACKMEMBER(int,  pfnHalt,(PVM pVM, const uint32_t fMask, uint64_t u64Now));
+    DECLR3CALLBACKMEMBER(int,  pfnHalt,(PUVM pUVM, const uint32_t fMask, uint64_t u64Now));
     /** The wait function. */
-    DECLR3CALLBACKMEMBER(int,  pfnWait,(PVM pVM));
+    DECLR3CALLBACKMEMBER(int,  pfnWait,(PUVM pUVM));
     /** The notifyFF function. */
-    DECLR3CALLBACKMEMBER(void, pfnNotifyFF,(PVM pVM, bool fNotifiedREM));
+    DECLR3CALLBACKMEMBER(void, pfnNotifyFF,(PUVM pUVM, bool fNotifiedREM));
 } g_aHaltMethods[] =
 {
+    { VMHALTMETHOD_BOOTSTRAP, NULL,                 NULL,                   NULL,                   vmR3BootstrapWait,      vmR3BootstrapNotifyFF },
     { VMHALTMETHOD_OLD,     NULL,                   NULL,                   vmR3HaltOldDoHalt,      vmR3DefaultWait,        vmR3DefaultNotifyFF },
     { VMHALTMETHOD_1,       vmR3HaltMethod1Init,    NULL,                   vmR3HaltMethod1Halt,    vmR3DefaultWait,        vmR3DefaultNotifyFF },
@@ -814,5 +944,23 @@
 {
     LogFlow(("VMR3NotifyFF:\n"));
-    g_aHaltMethods[pVM->vm.s.iHaltMethod].pfnNotifyFF(pVM, fNotifiedREM);
+    PUVM pUVM = pVM->pUVM;
+    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyFF(pUVM, fNotifiedREM);
+}
+
+
+/**
+ * Notify the emulation thread (EMT) about pending Forced Action (FF).
+ *
+ * This function is called by thread other than EMT to make
+ * sure EMT wakes up and promptly service an FF request.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   fNotifiedREM    Set if REM have already been notified. If clear the
+ *                          generic REMR3NotifyFF() method is called.
+ */
+VMR3DECL(void) VMR3NotifyFFU(PUVM pUVM, bool fNotifiedREM)
+{
+    LogFlow(("VMR3NotifyFF:\n"));
+    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyFF(pUVM, fNotifiedREM);
 }
 
@@ -852,32 +1000,32 @@
      * Record halt averages for the last second.
      */
+    PUVM pUVM = pVM->pUVM;
     uint64_t u64Now = RTTimeNanoTS();
-    int64_t off = u64Now - pVM->vm.s.u64HaltsStartTS;
+    int64_t off = u64Now - pUVM->vm.s.u64HaltsStartTS;
     if (off > 1000000000)
     {
-        if (off > _4G || !pVM->vm.s.cHalts)
-        {
-            pVM->vm.s.HaltInterval = 1000000000 /* 1 sec */;
-            pVM->vm.s.HaltFrequency = 1;
+        if (off > _4G || !pUVM->vm.s.cHalts)
+        {
+            pUVM->vm.s.HaltInterval = 1000000000 /* 1 sec */;
+            pUVM->vm.s.HaltFrequency = 1;
         }
         else
         {
-            pVM->vm.s.HaltInterval = (uint32_t)off / pVM->vm.s.cHalts;
-            pVM->vm.s.HaltFrequency = ASMMultU64ByU32DivByU32(pVM->vm.s.cHalts, 1000000000, (uint32_t)off);
-        }
-        pVM->vm.s.u64HaltsStartTS = u64Now;
-        pVM->vm.s.cHalts = 0;
-    }
-    pVM->vm.s.cHalts++;
+            pUVM->vm.s.HaltInterval = (uint32_t)off / pUVM->vm.s.cHalts;
+            pUVM->vm.s.HaltFrequency = ASMMultU64ByU32DivByU32(pUVM->vm.s.cHalts, 1000000000, (uint32_t)off);
+        }
+        pUVM->vm.s.u64HaltsStartTS = u64Now;
+        pUVM->vm.s.cHalts = 0;
+    }
+    pUVM->vm.s.cHalts++;
 
     /*
      * Do the halt.
      */
-    int rc = g_aHaltMethods[pVM->vm.s.iHaltMethod].pfnHalt(pVM, fMask, u64Now);
-
-    /*
-     * Resume the yielder and tell the world we're not blocking.
-     */
-    ASMAtomicXchgU32(&pVM->vm.s.fWait, 0);
+    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnHalt(pUVM, fMask, u64Now);
+
+    /*
+     * Resume the yielder.
+     */
     VMMR3YieldResume(pVM);
 
@@ -894,15 +1042,17 @@
  * @returns VINF_SUCCESS unless a fatal error occured. In the latter
  *          case an appropriate status code is returned.
- * @param   pVM         VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @thread  The emulation thread.
  */
-VMR3DECL(int) VMR3Wait(PVM pVM)
-{
-    LogFlow(("VMR3Wait:\n"));
+VMR3DECL(int) VMR3WaitU(PUVM pUVM)
+{
+    LogFlow(("VMR3WaitU:\n"));
 
     /*
      * Check Relevant FFs.
      */
-    if (VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK))
+    PVM pVM = pUVM->pVM;
+    if (    pVM
+        &&  VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK))
     {
         LogFlow(("VMR3Wait: returns VINF_SUCCESS (FF %#x)\n", pVM->fForcedActions));
@@ -914,6 +1064,6 @@
      * doesn't have to special case anything).
      */
-    int rc = g_aHaltMethods[pVM->vm.s.iHaltMethod].pfnWait(pVM);
-    LogFlow(("VMR3Wait: returns %Vrc (FF %#x)\n", rc, pVM->fForcedActions));
+    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnWait(pUVM);
+    LogFlow(("VMR3WaitU: returns %Vrc (FF %#x)\n", rc, pVM ? pVM->fForcedActions : 0));
     return rc;
 }
@@ -924,10 +1074,11 @@
  *
  * @returns VBox status code.
- * @param   pVM             The VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   enmHaltMethod   The new halt method.
  * @thread  EMT.
  */
-int vmR3SetHaltMethod(PVM pVM, VMHALTMETHOD enmHaltMethod)
-{
+int vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod)
+{
+    PVM pVM = pUVM->pVM; Assert(pVM);
     VM_ASSERT_EMT(pVM);
     AssertReturn(enmHaltMethod > VMHALTMETHOD_INVALID && enmHaltMethod < VMHALTMETHOD_END, VERR_INVALID_PARAMETER);
@@ -967,9 +1118,9 @@
      * Terminate the old one.
      */
-    if (    pVM->vm.s.enmHaltMethod != VMHALTMETHOD_INVALID
-        &&  g_aHaltMethods[pVM->vm.s.iHaltMethod].pfnTerm)
-    {
-        g_aHaltMethods[pVM->vm.s.iHaltMethod].pfnTerm(pVM);
-        pVM->vm.s.enmHaltMethod = VMHALTMETHOD_INVALID;
+    if (    pUVM->vm.s.enmHaltMethod != VMHALTMETHOD_INVALID
+        &&  g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm)
+    {
+        g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm(pUVM);
+        pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_INVALID;
     }
 
@@ -977,12 +1128,13 @@
      * Init the new one.
      */
-    memset(&pVM->vm.s.Halt, 0, sizeof(pVM->vm.s.Halt));
+    memset(&pUVM->vm.s.Halt, 0, sizeof(pUVM->vm.s.Halt));
     if (g_aHaltMethods[i].pfnInit)
     {
-        int rc = g_aHaltMethods[i].pfnInit(pVM);
+        int rc = g_aHaltMethods[i].pfnInit(pUVM);
         AssertRCReturn(rc, rc);
     }
-    pVM->vm.s.enmHaltMethod = enmHaltMethod;
-    ASMAtomicXchgU32(&pVM->vm.s.iHaltMethod, i);
+    pUVM->vm.s.enmHaltMethod = enmHaltMethod;
+
+    ASMAtomicWriteU32(&pUVM->vm.s.iHaltMethod, i);
     return VINF_SUCCESS;
 }
Index: /trunk/src/VBox/VMM/VMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/VMInternal.h	(revision 6795)
+++ /trunk/src/VBox/VMM/VMInternal.h	(revision 6796)
@@ -192,4 +192,6 @@
     /** The usual invalid value. */
     VMHALTMETHOD_INVALID = 0,
+    /** Use the method used during bootstrapping. */
+    VMHALTMETHOD_BOOTSTRAP,
     /** Use the default method. */
     VMHALTMETHOD_DEFAULT,
@@ -223,58 +225,57 @@
      * See VMINT2VM(). */
     RTINT                           offVM;
-
-    /** List of registered reset callbacks. */
-    R3PTRTYPE(PVMATRESET)           pAtReset;
-    /** List of registered reset callbacks. */
-    R3PTRTYPE(PVMATRESET *)         ppAtResetNext;
-
-    /** List of registered state change callbacks. */
-    R3PTRTYPE(PVMATSTATE)           pAtState;
-    /** List of registered state change callbacks. */
-    R3PTRTYPE(PVMATSTATE *)         ppAtStateNext;
-
-    /** List of registered error callbacks. */
-    R3PTRTYPE(PVMATERROR)           pAtError;
-    /** List of registered error callbacks. */
-    R3PTRTYPE(PVMATERROR *)         ppAtErrorNext;
-
-    /** List of registered error callbacks. */
-    R3PTRTYPE(PVMATRUNTIMEERROR)    pAtRuntimeError;
-    /** List of registered error callbacks. */
-    R3PTRTYPE(PVMATRUNTIMEERROR *)  ppAtRuntimeErrorNext;
-
+    /** VM Error Message. */
+    R3PTRTYPE(PVMERROR)             pErrorR3;
+    /** VM Runtime Error Message. */
+    R3PTRTYPE(PVMRUNTIMEERROR)      pRuntimeErrorR3;
+    /** Set by VMR3SuspendNoSave; cleared by VMR3Resume; signals the VM is in an inconsistent state and saving is not allowed. */
+    bool                            fPreventSaveState;
+} VMINT, *PVMINT;
+
+
+/**
+ * VM internal data kept in the UVM.
+ */
+typedef struct VMINTUSERPERVM
+{
     /** Head of the request queue. Atomic. */
-    volatile R3PTRTYPE(PVMREQ)      pReqs;
+    volatile PVMREQ                 pReqs;
     /** The last index used during alloc/free. */
     volatile uint32_t               iReqFree;
-    /** Array of pointers to lists of free request packets. Atomic. */
-    volatile R3PTRTYPE(PVMREQ)      apReqFree[9];
     /** Number of free request packets. */
     volatile uint32_t               cReqFree;
-
+    /** Array of pointers to lists of free request packets. Atomic. */
+    volatile PVMREQ                 apReqFree[9];
+
+#ifdef VBOX_WITH_STATISTICS
+    /** Number of VMR3ReqAlloc returning a new packet. */
+    STAMCOUNTER                     StatReqAllocNew;
+    /** Number of VMR3ReqAlloc causing races. */
+    STAMCOUNTER                     StatReqAllocRaces;
+    /** Number of VMR3ReqAlloc returning a recycled packet. */
+    STAMCOUNTER                     StatReqAllocRecycled;
+    /** Number of VMR3ReqFree calls. */
+    STAMCOUNTER                     StatReqFree;
+    /** Number of times the request was actually freed. */
+    STAMCOUNTER                     StatReqFreeOverflow;
+#endif
+
+    /** Pointer to the support library session.
+     * Mainly for creation and destruction.. */
+    PSUPDRVSESSION                  pSession;
+
+    /** The handle to the EMT thread. */
+    RTTHREAD                        ThreadEMT;
+    /** The native of the EMT thread. */
+    RTNATIVETHREAD                  NativeThreadEMT;
+    /** Wait event semaphore. */
+    RTSEMEVENT                      EventSemWait;
     /** Wait/Idle indicator. */
-    volatile uint32_t               fWait;
-    /** Wait event semaphore. */
-    R3PTRTYPE(RTSEMEVENT)           EventSemWait;
-
-    /** VM Error Message. */
-    R3PTRTYPE(PVMERROR)             pErrorR3;
-
-    /** VM Runtime Error Message. */
-    R3PTRTYPE(PVMRUNTIMEERROR)      pRuntimeErrorR3;
-
-    /** Pointer to the DBGC instance data. */
-    R3PTRTYPE(void *)               pvDBGC;
-
+    bool volatile                   fWait;
+    /** Force EMT to terminate. */
+    bool volatile                   fTerminateEMT;
     /** If set the EMT does the final VM cleanup when it exits.
      * If clear the VMR3Destroy() caller does so. */
     bool                            fEMTDoesTheCleanup;
-
-    /** Set by VMR3SuspendNoSave; cleared by VMR3Resume; signals the VM is in an inconsistent state and saving is not allowed. */
-    bool                            fPreventSaveState;
-
-    /** vmR3EmulationThread longjmp buffer
-     * @todo r=bird: requires union with padding. See EMInternal.h. */
-    jmp_buf                         emtJumpEnv;
 
     /** @name Generic Halt data
@@ -374,44 +375,53 @@
     }                               Halt;
 
-    /** @} */
-
-    /** Number of VMR3ReqAlloc returning a new packet. */
-    STAMCOUNTER                     StatReqAllocNew;
-    /** Number of VMR3ReqAlloc causing races. */
-    STAMCOUNTER                     StatReqAllocRaces;
-    /** Number of VMR3ReqAlloc returning a recycled packet. */
-    STAMCOUNTER                     StatReqAllocRecycled;
-    /** Number of VMR3ReqFree calls. */
-    STAMCOUNTER                     StatReqFree;
-    /** Number of times the request was actually freed. */
-    STAMCOUNTER                     StatReqFreeOverflow;
-
-    /** Profiling the halted state; yielding vs blocking. */
+    /** Profiling the halted state; yielding vs blocking.
+     * @{ */
     STAMPROFILE                     StatHaltYield;
     STAMPROFILE                     StatHaltBlock;
     STAMPROFILE                     StatHaltTimers;
     STAMPROFILE                     StatHaltPoll;
-} VMINT, *PVMINT;
-
-
-/**
- * Emulation thread arguments.
- */
-typedef struct VMEMULATIONTHREADARGS
-{
-    /** Pointer to the VM structure. */
-    PVM     pVM;
-} VMEMULATIONTHREADARGS;
-/** Pointer to the emulation thread arguments. */
-typedef VMEMULATIONTHREADARGS *PVMEMULATIONTHREADARGS;
+    /** @} */
+
+
+    /** List of registered reset callbacks. */
+    PVMATRESET                      pAtReset;
+    /** List of registered reset callbacks. */
+    PVMATRESET                     *ppAtResetNext;
+
+    /** List of registered state change callbacks. */
+    PVMATSTATE                      pAtState;
+    /** List of registered state change callbacks. */
+    PVMATSTATE                     *ppAtStateNext;
+
+    /** List of registered error callbacks. */
+    PVMATERROR                      pAtError;
+    /** List of registered error callbacks. */
+    PVMATERROR                     *ppAtErrorNext;
+
+    /** List of registered error callbacks. */
+    PVMATRUNTIMEERROR               pAtRuntimeError;
+    /** List of registered error callbacks. */
+    PVMATRUNTIMEERROR              *ppAtRuntimeErrorNext;
+
+    /** Pointer to the DBGC instance data. */
+    void                           *pvDBGC;
+
+
+    /** vmR3EmulationThread longjmp buffer. Must be last in the structure. */
+    jmp_buf                         emtJumpEnv;
+} VMINTUSERPERVM;
+
+/** Pointer to the VM internal data kept in the UVM. */
+typedef VMINTUSERPERVM *PVMINTUSERPERVM;
+
 
 DECLCALLBACK(int) vmR3EmulationThread(RTTHREAD ThreadSelf, void *pvArg);
-int vmR3SetHaltMethod(PVM pVM, VMHALTMETHOD enmHaltMethod);
+int vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod);
 DECLCALLBACK(int) vmR3Destroy(PVM pVM);
-DECLCALLBACK(void) vmR3SetErrorV(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *args);
+DECLCALLBACK(void) vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *args);
 void vmSetErrorCopy(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args);
 DECLCALLBACK(void) vmR3SetRuntimeErrorV(PVM pVM, bool fFatal, const char *pszErrorID, const char *pszFormat, va_list *args);
 void vmSetRuntimeErrorCopy(PVM pVM, bool fFatal, const char *pszErrorID, const char *pszFormat, va_list args);
-void vmR3DestroyFinalBit(PVM pVM);
+void vmR3DestroyFinalBitFromEMT(PUVM pUVM);
 void vmR3SetState(PVM pVM, VMSTATE enmStateNew);
 
Index: /trunk/src/VBox/VMM/VMMAll/VMAll.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/VMAll.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/VMMAll/VMAll.cpp	(revision 6796)
@@ -78,6 +78,6 @@
     va_copy(va2, args); /* Have to make a copy here or GCC will break. */
     PVMREQ pReq;
-    VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3SetErrorV, 7,  /* ASSUMES 3 source pos args! */
-                pVM, rc, RT_SRC_POS_ARGS, pszFormat, &va2);
+    VMR3ReqCall(pVM, &pReq, RT_INDEFINITE_WAIT, (PFNRT)vmR3SetErrorUV, 7,   /* ASSUMES 3 source pos args! */
+                pVM->pUVM, rc, RT_SRC_POS_ARGS, pszFormat, &va2);
     VMR3ReqFree(pReq);
     va_end(va2);
@@ -202,5 +202,5 @@
  * containig possible actions (for example, Retry/Ignore). For this reason,
  * an error ID assigned once to some particular error condition should not
- * change in the future. The format of this parameter is "someErrorCondition". 
+ * change in the future. The format of this parameter is "someErrorCondition".
  *
  * @param   pVM             VM handle. Must be non-NULL.
@@ -214,9 +214,9 @@
  *
  * @thread  Any
- * @todo    r=bird: The pausing/suspending of the VM should be done here, we'll just end 
- *                  up duplicating code all over the place otherwise. In the case of 
- *                  devices/drivers/etc they might not be trusted to pause/suspend the 
+ * @todo    r=bird: The pausing/suspending of the VM should be done here, we'll just end
+ *                  up duplicating code all over the place otherwise. In the case of
+ *                  devices/drivers/etc they might not be trusted to pause/suspend the
  *                  vm even. Change fFatal to fFlags and define action flags and a fatal flag.
- *                      
+ *
  *                  Also, why a string ID and not an enum?
  */
Index: /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp	(revision 6796)
@@ -83,7 +83,7 @@
     if (RT_SUCCESS(rc))
     {
-#ifdef VBOX_WITH_NEW_PHYS_CODE /* need to test on windows, solaris and darwin. */
+//#ifdef VBOX_WITH_NEW_PHYS_CODE /* need to test on windows, solaris and darwin. */
         rc = GMMR0Init();
-#endif
+//#endif
         if (RT_SUCCESS(rc))
         {
@@ -134,7 +134,7 @@
      * Destroy the GMM and GVMM instances.
      */
-#ifdef VBOX_WITH_NEW_PHYS_CODE
+//#ifdef VBOX_WITH_NEW_PHYS_CODE
     GMMR0Term();
-#endif
+//#endif
     GVMMR0Term();
 
Index: /trunk/src/VBox/VMM/VMReq.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMReq.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/VMReq.cpp	(revision 6796)
@@ -25,4 +25,5 @@
 #include "VMInternal.h"
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 
 #include <VBox/err.h>
@@ -40,5 +41,38 @@
 *   Internal Functions                                                         *
 *******************************************************************************/
-static int  vmR3ReqProcessOne(PVM pVM, PVMREQ pReq);
+static int  vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq);
+
+
+/**
+ * Allocate and queue a call request.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completation. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happends.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ *                          Not possible to pass 64-bit arguments!
+ * @param   ...             Function arguments.
+ */
+VMR3DECL(int) VMR3ReqCallU(PUVM pUVM, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, ppReq, cMillies, VMREQFLAGS_VBOX_STATUS, pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
 
 
@@ -70,5 +104,38 @@
     va_list va;
     va_start(va, cArgs);
-    int rc = VMR3ReqCallV(pVM, ppReq, cMillies, VMREQFLAGS_VBOX_STATUS, pfnFunction, cArgs, va);
+    int rc = VMR3ReqCallVU(pVM->pUVM, ppReq, cMillies, VMREQFLAGS_VBOX_STATUS, pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Allocate and queue a call request to a void function.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completation. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happends.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ *                          Not possible to pass 64-bit arguments!
+ * @param   ...             Function arguments.
+ */
+VMR3DECL(int) VMR3ReqCallVoidU(PUVM pUVM, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, ppReq, cMillies, VMREQFLAGS_VOID, pfnFunction, cArgs, va);
     va_end(va);
     return rc;
@@ -103,5 +170,5 @@
     va_list va;
     va_start(va, cArgs);
-    int rc = VMR3ReqCallV(pVM, ppReq, cMillies, VMREQFLAGS_VOID, pfnFunction, cArgs, va);
+    int rc = VMR3ReqCallVU(pVM->pUVM, ppReq, cMillies, VMREQFLAGS_VOID, pfnFunction, cArgs, va);
     va_end(va);
     return rc;
@@ -138,5 +205,5 @@
     va_list va;
     va_start(va, cArgs);
-    int rc = VMR3ReqCallV(pVM, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
+    int rc = VMR3ReqCallVU(pVM->pUVM, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
     va_end(va);
     return rc;
@@ -145,5 +212,5 @@
 
 /**
- * Allocate and queue a call request.
+ * Allocate and queue a call request to a void function.
  *
  * If it's desired to poll on the completion of the request set cMillies
@@ -156,5 +223,40 @@
  * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
  *
- * @param   pVM             The VM handle.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happends, unless fFlags
+ *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   fFlags          A combination of the VMREQFLAGS values.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ *                          Not possible to pass 64-bit arguments!
+ * @param   ...             Function arguments.
+ */
+VMR3DECL(int) VMR3ReqCallU(PUVM pUVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Allocate and queue a call request.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completation. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
  * @param   ppReq           Where to store the pointer to the request.
  *                          This will be NULL or a valid request pointer not matter what happends, unless fFlags
@@ -169,32 +271,28 @@
  * @param   Args            Argument vector.
  */
-VMR3DECL(int) VMR3ReqCallV(PVM pVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, va_list Args)
+VMR3DECL(int) VMR3ReqCallVU(PUVM pUVM, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, va_list Args)
 {
     LogFlow(("VMR3ReqCallV: cMillies=%d fFlags=%#x pfnFunction=%p cArgs=%d\n", cMillies, fFlags, pfnFunction, cArgs));
 
     /*
-     * Check input.
-     */
-    if (!pfnFunction || !pVM || (fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT)))
-    {
-        AssertFailed();
-        return VERR_INVALID_PARAMETER;
-    }
+     * Validate input.
+     */
+    AssertPtrReturn(pfnFunction, VERR_INVALID_POINTER);
+    AssertPtrReturn(pUVM, VERR_INVALID_POINTER);
+    AssertReturn(!(fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT)), VERR_INVALID_PARAMETER);
     if (!(fFlags & VMREQFLAGS_NO_WAIT) || ppReq)
     {
-        Assert(ppReq);
+        AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
         *ppReq = NULL;
     }
     PVMREQ pReq = NULL;
-    if (cArgs * sizeof(uintptr_t) > sizeof(pReq->u.Internal.aArgs))
-    {
-        AssertMsgFailed(("cArg=%d\n", cArgs));
-        return VERR_TOO_MUCH_DATA;
-    }
+    AssertMsgReturn(cArgs * sizeof(uintptr_t) <= sizeof(pReq->u.Internal.aArgs),
+                    ("cArg=%d\n", cArgs),
+                    VERR_TOO_MUCH_DATA);
 
     /*
      * Allocate request
      */
-    int rc = VMR3ReqAlloc(pVM, &pReq, VMREQTYPE_INTERNAL);
+    int rc = VMR3ReqAllocU(pUVM, &pReq, VMREQTYPE_INTERNAL);
     if (VBOX_FAILURE(rc))
         return rc;
@@ -260,5 +358,5 @@
  * Joins the list pList with whatever is linked up at *pHead.
  */
-static void vmr3ReqJoinFree(PVMINT pVMInt, PVMREQ pList)
+static void vmr3ReqJoinFree(PVMINTUSERPERVM pVMInt, PVMREQ pList)
 {
     /*
@@ -272,13 +370,13 @@
         {
             const uint32_t i = pVMInt->iReqFree;
-            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2) % ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
+            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
 
             pTail->pNext = NULL;
-            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2 + (i == pVMInt->iReqFree)) % ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
+            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2 + (i == pVMInt->iReqFree)) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
             return;
         }
         pTail = pTail->pNext;
     }
-    vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(pVMInt->iReqFree + 2) % ELEMENTS(pVMInt->apReqFree)], pList);
+    vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(pVMInt->iReqFree + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pList);
 }
 
@@ -298,14 +396,30 @@
 VMR3DECL(int) VMR3ReqAlloc(PVM pVM, PVMREQ *ppReq, VMREQTYPE enmType)
 {
+    return VMR3ReqAllocU(pVM->pUVM, ppReq, enmType);
+}
+
+
+/**
+ * Allocates a request packet.
+ *
+ * The caller allocates a request packet, fills in the request data
+ * union and queues the request.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   ppReq           Where to store the pointer to the allocated packet.
+ * @param   enmType         Package type.
+ */
+VMR3DECL(int) VMR3ReqAllocU(PUVM pUVM, PVMREQ *ppReq, VMREQTYPE enmType)
+{
     /*
      * Validate input.
      */
-    if (    enmType < VMREQTYPE_INVALID
-        ||  enmType > VMREQTYPE_MAX)
-    {
-        AssertMsgFailed(("Invalid package type %d valid range %d-%d inclusivly.\n",
-                         enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1));
-        return VERR_VM_REQUEST_INVALID_TYPE;
-    }
+    AssertMsgReturn(enmType > VMREQTYPE_INVALID && enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusivly.\n",
+                     enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
+    AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
 
     /*
@@ -314,8 +428,8 @@
      * of mine to avoid locks.
      */
-    int cTries = ELEMENTS(pVM->vm.s.apReqFree) * 2;
+    int cTries = RT_ELEMENTS(pUVM->vm.s.apReqFree) * 2;
     while (--cTries >= 0)
     {
-        PVMREQ volatile *ppHead = &pVM->vm.s.apReqFree[ASMAtomicIncU32(&pVM->vm.s.iReqFree) % ELEMENTS(pVM->vm.s.apReqFree)];
+        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
 #if 0 /* sad, but this won't work safely because the reading of pReq->pNext. */
         PVMREQ pNext = NULL;
@@ -337,9 +451,9 @@
                 &&  !ASMAtomicCmpXchgPtr((void * volatile *)ppHead, pNext, NULL))
             {
-                STAM_COUNTER_INC(&pVM->vm.s.StatReqAllocRaces);
-                vmr3ReqJoinFree(&pVM->vm.s, pReq->pNext);
+                STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRaces);
+                vmr3ReqJoinFree(&pUVM->vm.s, pReq->pNext);
             }
 #endif
-            ASMAtomicDecU32(&pVM->vm.s.cReqFree);
+            ASMAtomicDecU32(&pUVM->vm.s.cReqFree);
 
             /*
@@ -372,5 +486,5 @@
             Assert(pReq->enmType == VMREQTYPE_INVALID);
             Assert(pReq->enmState == VMREQSTATE_FREE);
-            Assert(pReq->pVM == pVM);
+            Assert(pReq->pUVM == pUVM);
             ASMAtomicXchgSize(&pReq->pNext, NULL);
             pReq->enmState = VMREQSTATE_ALLOCATED;
@@ -380,5 +494,5 @@
 
             *ppReq = pReq;
-            STAM_COUNTER_INC(&pVM->vm.s.StatReqAllocRecycled);
+            STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRecycled);
             LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p recycled\n", pReq));
             return VINF_SUCCESS;
@@ -389,5 +503,5 @@
      * Ok allocate one.
      */
-    PVMREQ pReq = (PVMREQ)MMR3HeapAlloc(pVM, MM_TAG_VM_REQ, sizeof(*pReq));
+    PVMREQ pReq = (PVMREQ)MMR3HeapAllocU(pUVM, MM_TAG_VM_REQ, sizeof(*pReq));
     if (!pReq)
         return VERR_NO_MEMORY;
@@ -408,5 +522,5 @@
      */
     pReq->pNext    = NULL;
-    pReq->pVM      = pVM;
+    pReq->pUVM     = pUVM;
     pReq->enmState = VMREQSTATE_ALLOCATED;
     pReq->iStatus  = VERR_VM_REQUEST_STATUS_STILL_PENDING;
@@ -416,5 +530,5 @@
 
     *ppReq = pReq;
-    STAM_COUNTER_INC(&pVM->vm.s.StatReqAllocNew);
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocNew);
     LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p new\n", pReq));
     return VINF_SUCCESS;
@@ -437,6 +551,4 @@
     if (!pReq)
         return VINF_SUCCESS;
-
-    STAM_COUNTER_INC(&pReq->pVM->vm.s.StatReqFree);
 
     /*
@@ -460,9 +572,11 @@
     pReq->enmType  = VMREQTYPE_INVALID;
 
-    PVM pVM = pReq->pVM;
-    if (pVM->vm.s.cReqFree < 128)
-    {
-        ASMAtomicIncU32(&pVM->vm.s.cReqFree);
-        PVMREQ volatile *ppHead = &pVM->vm.s.apReqFree[ASMAtomicIncU32(&pVM->vm.s.iReqFree) % ELEMENTS(pVM->vm.s.apReqFree)];
+    PUVM pUVM = pReq->pUVM;
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqFree);
+
+    if (pUVM->vm.s.cReqFree < 128)
+    {
+        ASMAtomicIncU32(&pUVM->vm.s.cReqFree);
+        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
         PVMREQ pNext;
         do
@@ -474,5 +588,5 @@
     else
     {
-        STAM_COUNTER_INC(&pReq->pVM->vm.s.StatReqFreeOverflow);
+        STAM_COUNTER_INC(&pReq->pUVM->vm.s.StatReqFreeOverflow);
         RTSemEventDestroy(pReq->EventSem);
         MMR3HeapFree(pReq);
@@ -506,23 +620,15 @@
      * Verify the supplied package.
      */
-    if (pReq->enmState != VMREQSTATE_ALLOCATED)
-    {
-        AssertMsgFailed(("Invalid state %d\n", pReq->enmState));
-        return VERR_VM_REQUEST_STATE;
-    }
-    if (   !pReq->pVM
-        ||  pReq->pNext
-        ||  !pReq->EventSem)
-    {
-        AssertMsgFailed(("Invalid request package! Anyone cooking their own packages???\n"));
-        return VERR_VM_REQUEST_INVALID_PACKAGE;
-    }
-    if (    pReq->enmType < VMREQTYPE_INVALID
-        || pReq->enmType > VMREQTYPE_MAX)
-    {
-        AssertMsgFailed(("Invalid package type %d valid range %d-%d inclusivly. This was verified on alloc too...\n",
-                         pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1));
-        return VERR_VM_REQUEST_INVALID_TYPE;
-    }
+    AssertMsgReturn(pReq->enmState == VMREQSTATE_ALLOCATED, ("%d\n", pReq->enmState), VERR_VM_REQUEST_STATE);
+    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
+                    &&  !pReq->pNext
+                    &&  pReq->EventSem != NIL_RTSEMEVENT,
+                    ("Invalid request package! Anyone cooking their own packages???\n"),
+                    VERR_VM_REQUEST_INVALID_PACKAGE);
+    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
+                    &&  pReq->enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusivly. This was verified on alloc too...\n",
+                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
 
     /*
@@ -531,6 +637,6 @@
      */
     int rc = VINF_SUCCESS;
-    PVM pVM = ((VMREQ volatile *)pReq)->pVM;                    /* volatile paranoia */
-    if (pVM->NativeThreadEMT != RTThreadNativeSelf())
+    PUVM pUVM = ((VMREQ volatile *)pReq)->pUVM;                 /* volatile paranoia */
+    if (pUVM->vm.s.NativeThreadEMT != RTThreadNativeSelf())
     {
         unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags;     /* volatile paranoia */
@@ -543,13 +649,14 @@
         do
         {
-            pNext = pVM->vm.s.pReqs;
+            pNext = pUVM->vm.s.pReqs;
             pReq->pNext = pNext;
-        } while (!ASMAtomicCmpXchgPtr((void * volatile *)&pVM->vm.s.pReqs, (void *)pReq, (void *)pNext));
+        } while (!ASMAtomicCmpXchgPtr((void * volatile *)&pUVM->vm.s.pReqs, (void *)pReq, (void *)pNext));
 
         /*
          * Notify EMT.
          */
-        VM_FF_SET(pVM, VM_FF_REQUEST);
-        VMR3NotifyFF(pVM, false);
+        if (pUVM->pVM)
+            VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
+        VMR3NotifyFFU(pUVM, false);
 
         /*
@@ -566,5 +673,5 @@
          */
         pReq->enmState = VMREQSTATE_QUEUED;
-        rc = vmR3ReqProcessOne(pVM, pReq);
+        rc = vmR3ReqProcessOneU(pUVM, pReq);
         LogFlow(("VMR3ReqQueue: returns %Vrc (processed)\n", rc));
     }
@@ -590,29 +697,25 @@
      * Verify the supplied package.
      */
-    if (    pReq->enmState != VMREQSTATE_QUEUED
-        &&  pReq->enmState != VMREQSTATE_PROCESSING
-        &&  pReq->enmState != VMREQSTATE_COMPLETED)
-    {
-        AssertMsgFailed(("Invalid state %d\n", pReq->enmState));
-        return VERR_VM_REQUEST_STATE;
-    }
-    if (    !pReq->pVM
-        ||  !pReq->EventSem)
-    {
-        AssertMsgFailed(("Invalid request package! Anyone cooking their own packages???\n"));
-        return VERR_VM_REQUEST_INVALID_PACKAGE;
-    }
-    if (    pReq->enmType < VMREQTYPE_INVALID
-        ||  pReq->enmType > VMREQTYPE_MAX)
-    {
-        AssertMsgFailed(("Invalid package type %d valid range %d-%d inclusivly. This was verified on alloc and queue too...\n",
-                         pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1));
-        return VERR_VM_REQUEST_INVALID_TYPE;
-    }
+    AssertMsgReturn(    pReq->enmState == VMREQSTATE_QUEUED
+                    ||  pReq->enmState == VMREQSTATE_PROCESSING
+                    ||  pReq->enmState == VMREQSTATE_COMPLETED,
+                    ("Invalid state %d\n", pReq->enmState),
+                    VERR_VM_REQUEST_STATE);
+    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
+                    &&  pReq->EventSem != NIL_RTSEMEVENT,
+                    ("Invalid request package! Anyone cooking their own packages???\n"),
+                    VERR_VM_REQUEST_INVALID_PACKAGE);
+    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
+                    &&  pReq->enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusivly. This was verified on alloc too...\n",
+                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
 
     /*
      * Check for deadlock condition
      */
-    AssertMsg(!VMMR3LockIsOwner(pReq->pVM), ("Waiting for EMT to process a request, but we own the global VM lock!?!?!?!\n"));
+    PUVM pUVM = pReq->pUVM;
+    AssertMsg(!pUVM->pVM || !VMMR3LockIsOwner(pUVM->pVM),
+              ("Waiting for EMT to process a request, but we own the global VM lock!?!?!?!\n"));
 
     /*
@@ -644,14 +747,14 @@
  * Process pending request(s).
  *
- * This function is called from a forced action handler in
- * the EMT.
- *
- * @returns VBox status code.
- *
- * @param   pVM         VM handle.
- */
-VMR3DECL(int) VMR3ReqProcess(PVM pVM)
-{
-    LogFlow(("VMR3ReqProcess: (enmVMState=%d)\n", pVM->enmVMState));
+ * This function is called from a forced action handler in the EMT
+ * or from one of the EMT loops.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+VMR3DECL(int) VMR3ReqProcessU(PUVM pUVM)
+{
+    LogFlow(("VMR3ReqProcessU: (enmVMState=%d)\n", pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING));
 
     /*
@@ -667,6 +770,7 @@
          * Get pending requests.
          */
-        VM_FF_CLEAR(pVM, VM_FF_REQUEST);
-        PVMREQ pReqs = (PVMREQ)ASMAtomicXchgPtr((void * volatile *)&pVM->vm.s.pReqs, NULL);
+        if (RT_LIKELY(pUVM->pVM))
+            VM_FF_CLEAR(pUVM->pVM, VM_FF_REQUEST);
+        PVMREQ pReqs = (PVMREQ)ASMAtomicXchgPtr((void * volatile *)&pUVM->vm.s.pReqs, NULL);
         if (!pReqs)
             break;
@@ -682,5 +786,5 @@
         {
             Assert(pReq->enmState == VMREQSTATE_QUEUED);
-            Assert(pReq->pVM == pVM);
+            Assert(pReq->pUVM == pUVM);
             PVMREQ pCur = pReq;
             pReq = pReq->pNext;
@@ -704,5 +808,5 @@
 
             /* Process the request */
-            int rc2 = vmR3ReqProcessOne(pVM, pReq);
+            int rc2 = vmR3ReqProcessOneU(pUVM, pReq);
 
             /*
@@ -718,5 +822,5 @@
     }
 
-    LogFlow(("VMR3ReqProcess: returns %Vrc (enmVMState=%d)\n", rc, pVM->enmVMState));
+    LogFlow(("VMR3ReqProcess: returns %Vrc (enmVMState=%d)\n", rc, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING));
     return rc;
 }
@@ -731,5 +835,5 @@
  * @param   pReq        Request packet to process.
  */
-static int  vmR3ReqProcessOne(PVM pVM, PVMREQ pReq)
+static int  vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq)
 {
     LogFlow(("vmR3ReqProcessOne: pReq=%p type=%d fFlags=%#x\n", pReq, pReq->enmType, pReq->fFlags));
Index: /trunk/src/VBox/VMM/testcase/tstCFGM.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstCFGM.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/testcase/tstCFGM.cpp	(revision 6796)
@@ -25,8 +25,9 @@
 #include <VBox/mm.h>
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 
 #include <VBox/err.h>
 #include <VBox/param.h>
-#include <iprt/runtime.h>
+#include <iprt/initterm.h>
 #include <iprt/stream.h>
 #include <iprt/string.h>
@@ -46,5 +47,5 @@
     int rc = SUPInit(NULL);
     if (VBOX_SUCCESS(rc))
-        rc = SUPPageAlloc((sizeof(*pVM) + PAGE_SIZE - 1) >> PAGE_SHIFT, (void **)&pVM);
+        rc = SUPPageAlloc(RT_ALIGN_Z(sizeof(*pVM), PAGE_SIZE) >> PAGE_SHIFT, (void **)&pVM);
     if (VBOX_FAILURE(rc))
     {
@@ -53,5 +54,17 @@
     }
 
-    rc = STAMR3Init(pVM);
+    static UVM s_UVM;
+    PUVM pUVM = &s_UVM;
+    pUVM->pVM = pVM;
+    pVM->pUVM = pUVM;
+
+    rc = STAMR3InitUVM(pUVM);
+    if (VBOX_FAILURE(rc))
+    {
+        RTPrintf("FAILURE: STAMR3Init failed. rc=%Vrc\n", rc);
+        return 1;
+    }
+
+    rc = MMR3InitUVM(pUVM);
     if (VBOX_FAILURE(rc))
     {
Index: /trunk/src/VBox/VMM/testcase/tstMMHyperHeap.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstMMHyperHeap.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/testcase/tstMMHyperHeap.cpp	(revision 6796)
@@ -22,4 +22,5 @@
 #include <VBox/stam.h>
 #include <VBox/vm.h>
+#include <VBox/uvm.h>
 #include <VBox/sup.h>
 #include <VBox/param.h>
@@ -27,5 +28,6 @@
 
 #include <VBox/log.h>
-#include <iprt/runtime.h>
+#include <iprt/initterm.h>
+#include <iprt/mem.h>
 #include <iprt/assert.h>
 #include <iprt/stream.h>
@@ -33,5 +35,5 @@
 
 
-int main(int argc, char *argv[])
+int main(int argc, char **argv)
 {
 
@@ -45,7 +47,9 @@
      */
     PVM         pVM;
+    RTR0PTR     pvR0;
+    SUPPAGE     aPages[RT_ALIGN_Z(sizeof(*pVM), PAGE_SIZE) >> PAGE_SHIFT];
     int rc = SUPInit(NULL);
     if (VBOX_SUCCESS(rc))
-        rc = SUPPageAlloc((sizeof(*pVM) + PAGE_SIZE - 1) >> PAGE_SHIFT, (void **)&pVM);
+        rc = SUPLowAlloc(RT_ELEMENTS(aPages), (void **)&pVM, &pvR0, &aPages[0]);
     if (VBOX_FAILURE(rc))
     {
@@ -53,10 +57,27 @@
         return 1;
     }
-    rc = STAMR3Init(pVM);
-    if (VBOX_FAILURE(rc))
-    {
-        RTPrintf("Fatal error: STAMR3Init failed! rc=%Vrc\n", rc);
-        return 1;
-    }
+    memset(pVM, 0, sizeof(*pVM)); /* wtf? */
+    pVM->paVMPagesR3 = aPages;
+    pVM->pVMR0 = pvR0;
+
+    static UVM s_UVM;
+    PUVM pUVM = &s_UVM;
+    pUVM->pVM = pVM;
+    pVM->pUVM = pUVM;
+
+    rc = STAMR3InitUVM(pUVM);
+    if (VBOX_FAILURE(rc))
+    {
+        RTPrintf("FAILURE: STAMR3Init failed. rc=%Vrc\n", rc);
+        return 1;
+    }
+
+    rc = MMR3InitUVM(pUVM);
+    if (VBOX_FAILURE(rc))
+    {
+        RTPrintf("FAILURE: STAMR3Init failed. rc=%Vrc\n", rc);
+        return 1;
+    }
+
     rc = MMR3Init(pVM);
     if (VBOX_FAILURE(rc))
Index: /trunk/src/VBox/VMM/testcase/tstSSM.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstSSM.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/testcase/tstSSM.cpp	(revision 6796)
@@ -573,5 +573,5 @@
     }
 
-    rc = STAMR3Init(pVM);
+///@todo    rc = STAMR3Init(pVM);
     if (VBOX_FAILURE(rc))
     {
Index: /trunk/src/VBox/VMM/testcase/tstVMStructGC.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstVMStructGC.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/testcase/tstVMStructGC.cpp	(revision 6796)
@@ -187,5 +187,4 @@
     GEN_CHECK_OFF(MM, cShadowPages);
     GEN_CHECK_OFF(MM, cFixedPages);
-    GEN_CHECK_OFF(MM, pHeap);
     GEN_CHECK_SIZE(MMHYPERSTAT);
     GEN_CHECK_SIZE(MMHYPERCHUNK);
@@ -218,5 +217,4 @@
     GEN_CHECK_SIZE(PDM);
     GEN_CHECK_OFF(PDM, offVM);
-    GEN_CHECK_OFF(PDM, pModules);
     GEN_CHECK_OFF(PDM, pDevs);
     GEN_CHECK_OFF(PDM, pDevInstances);
@@ -660,6 +658,4 @@
     GEN_CHECK_OFF(SELM, StatUpdateFromCPUM);
 
-    GEN_CHECK_SIZE(STAM);
-
     GEN_CHECK_SIZE(TM);
     GEN_CHECK_OFF(TM, offVM);
Index: /trunk/src/VBox/VMM/testcase/tstVMStructSize.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstVMStructSize.cpp	(revision 6795)
+++ /trunk/src/VBox/VMM/testcase/tstVMStructSize.cpp	(revision 6796)
@@ -133,5 +133,4 @@
     CHECK_PADDING_VM(pgm);
     CHECK_PADDING_VM(selm);
-    CHECK_PADDING_VM(stam);
     CHECK_PADDING_VM(tm);
     CHECK_PADDING_VM(trpm);
